<commit_before>package main

import (
	"crypto/tls"
	"encoding/json"
	"flag"
	"koding/kontrol/kontrolhelper"
	"koding/tools/amqputil"
	"koding/tools/config"
	"koding/tools/lifecycle"
	"koding/tools/logger"
	"koding/tools/sockjs"
	"koding/tools/utils"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/fatih/set"
	"github.com/streadway/amqp"
)

const BROKER_NAME = "broker"

var (
	conf *config.Config
	log  = logger.New(BROKER_NAME)

	STORAGE_BACKEND = "redis"
	routeMap        = make(map[string]*set.Set)
	sessionsMap     = make(map[string]*sockjs.Session)
	globalMapMutex  sync.Mutex

	changeClientsGauge          = lifecycle.CreateClientsGauge()
	changeNewClientsGauge       = logger.CreateCounterGauge("newClients", logger.NoUnit, true)
	changeWebsocketClientsGauge = logger.CreateCounterGauge("websocketClients", logger.NoUnit, false)

	flagProfile      = flag.String("c", "", "Configuration profile from file")
	flagBrokerDomain = flag.String("a", "", "Send kontrol a custom domain istead of os.Hostname")
	flagKontrolUUID  = flag.String("u", "", "Enable Kontrol mode")
	flagBrokerType   = flag.String("b", "broker", "Define broker type. Available: broker and brokerKite. B")
	flagDebug        = flag.Bool("d", false, "Debug mode")
)

// Broker is a router/multiplexer that routes messages coming from a SockJS
// server to an AMQP exchange and vice versa. Broker basically listens to
// client messages (Koding users) from the SockJS server. The message is
// either passed to the appropriate exchange or a response is sent back to the
// client. Each message has an "action" field that defines how to act for a
// received message.
type Broker struct {
	Config            *config.Broker
	Hostname          string
	ServiceUniqueName string
	AuthAllExchange   string
	PublishConn       *amqp.Connection
	ConsumeConn       *amqp.Connection

	// Accepts SockJS connections
	listener net.Listener

	// Closed when SockJS server is ready to acccept connections
	ready chan struct{}
}

// NewBroker returns a new Broker instance with ServiceUniqueName and Hostname
// prepopulated. After creating a Broker instance, one has to call
// broker.Run() or broker.Start() to start the broker instance and call
// broker.Close() for a graceful stop.
func NewBroker() *Broker {
	// returns os.Hostname() if config.BrokerDomain is empty, otherwise it just
	// returns config.BrokerDomain back
	brokerHostname := kontrolhelper.CustomHostname(*flagBrokerDomain)
	sanitizedHostname := strings.Replace(brokerHostname, ".", "_", -1)
	serviceUniqueName := BROKER_NAME + "|" + sanitizedHostname

	return &Broker{
		Hostname:          brokerHostname,
		ServiceUniqueName: serviceUniqueName,
		ready:             make(chan struct{}),
	}
}

func main() {
	flag.Parse()
	if *flagProfile == "" {
		log.Fatal("Please specify profile via -c. Aborting.")
	}

	conf = config.MustConfig(*flagProfile)
	broker := NewBroker()

	switch *flagBrokerType {
	case "brokerKite":
		broker.Config = &conf.BrokerKite
	default:
		broker.Config = &conf.Broker
	}

	// update broker name
	log = logger.New(broker.Config.Name)
	var logLevel logger.Level
	if *flagDebug {
		logLevel = logger.DEBUG
	} else {
		logLevel = logger.GetLoggingLevelFromConfig(BROKER_NAME, *flagProfile)
	}

	log.SetLevel(logLevel)
	broker.Run()
}

// Run starts the broker.
func (b *Broker) Run() {
	lifecycle.Startup(BROKER_NAME, false)
	logger.RunGaugesLoop(log)

	if err := b.registerToKontrol(); err != nil {
		log.Critical("Couldnt register to kontrol, stopping... %v", err)
		return
	}

	if err := b.startAMQP(); err != nil {
		log.Critical("Couldnt create amqp bindings, stopping... %v", err)
		return
	}

	b.startSockJS() // blocking
}

// Start is like Run() but waits until the SockJS listener is ready to be
// used.
func (b *Broker) Start() {
	go b.Run()
	<-b.ready
}

// Close close all amqp connections and closes the SockJS server listener
func (b *Broker) Close() {
	b.PublishConn.Close()
	b.ConsumeConn.Close()
	b.listener.Close()
}

// registerToKontrol registers the broker to KontrolDaemon. This is needed to
// populate a list of brokers and show them to the client. The list is
// available at: https://koding.com/-/services/broker?all
func (b *Broker) registerToKontrol() {
	if err := kontrolhelper.RegisterToKontrol(
		conf,
		b.Config.Name,
		b.Config.Name, // servicGenericName
		b.ServiceUniqueName,
		*flagKontrolUUID,
		b.Hostname,
		b.Config.Port,
	); err != nil {
		log.Critical("Couldnt register to kontrol %v", err)
	}
}

// startAMQP setups the the neccesary publisher and consumer connections for
// the broker broker.
func (b *Broker) startAMQP() {
	b.PublishConn = amqputil.CreateConnection(conf, b.Config.Name)
	defer b.PublishConn.Close()

	b.ConsumeConn = amqputil.CreateConnection(conf, b.Config.Name)
	defer b.ConsumeConn.Close()

	consumeChannel := amqputil.CreateChannel(b.ConsumeConn)
	defer consumeChannel.Close()

	presenceQueue := amqputil.JoinPresenceExchange(
		consumeChannel,      // channel
		"services-presence", // exchange
		b.Config.Name,       // serviceType
		b.Config.Name,       // serviceGenericName
		b.ServiceUniqueName, // serviceUniqueName
		false,               // loadBalancing
	)

	go func() {
		sigusr1Channel := make(chan os.Signal)
		signal.Notify(sigusr1Channel, syscall.SIGUSR1)
		<-sigusr1Channel
		consumeChannel.QueueDelete(presenceQueue, false, false, false)
	}()

	stream := amqputil.DeclareBindConsumeQueue(consumeChannel, "topic", b.Config.Name, "#", false)

	if err := consumeChannel.ExchangeDeclare(
		"updateInstances", // name
		"fanout",          // kind
		false,             // durable
		false,             // autoDelete
		false,             // internal
		false,             // noWait
		nil,               // args
	); err != nil {
		log.Critical("Couldnt create updateInstances exchange  %v", err)
	}

	if err := consumeChannel.ExchangeBind(BROKER_NAME, "", "updateInstances", false, nil); err != nil {
		log.Critical("Couldnt bind to updateInstances exchange  %v", err)
	}

	// signal that we are ready now
	close(b.amqpReady)

	// start to listen from "broker" topic exchange
	for amqpMessage := range stream {
		routingKey := amqpMessage.RoutingKey
		payload := json.RawMessage(utils.FilterInvalidUTF8(amqpMessage.Body))

		pos := strings.IndexRune(routingKey, '.') // skip first dot, since we want at least two components to always include the secret
		for pos != -1 && pos < len(routingKey) {
			index := strings.IndexRune(routingKey[pos+1:], '.')
			pos += index + 1
			if index == -1 {
				pos = len(routingKey)
			}
			routingKeyPrefix := routingKey[:pos]
			globalMapMutex.Lock()

			if routes, ok := routeMap[routingKeyPrefix]; ok {
				routes.Each(func(sessionId interface{}) bool {
					if routeSession, ok := sessionsMap[sessionId.(string)]; ok {
						sendToClient(routeSession, routingKey, &payload)
					}
					return true
				})
			}
			globalMapMutex.Unlock()
		}
	}
}

// startSockJS starts a new HTTPS listener that implies the SockJS protocol.
func (b *Broker) startSockJS() {
	service := sockjs.NewService(
		conf.Client.StaticFilesBaseUrl+"/js/sock.js",
		10*time.Minute,
		b.sockjsSession,
	)
	defer service.Close()

	service.MaxReceivedPerSecond = 50
	service.ErrorHandler = log.LogError

	// TODO use http.Mux instead of sockjs.Mux.
	mux := &sockjs.Mux{
		Handlers: map[string]http.Handler{
			"/subscribe": service,
			"/buildnumber": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("Content-Type", "text/plain")
				w.Write([]byte(strconv.Itoa(conf.BuildNumber)))
			}),
		},
	}

	server := &http.Server{Handler: mux}

	var err error
	b.listener, err = net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP(b.Config.IP), Port: b.Config.Port})
	if err != nil {
		log.Fatal(err)
	}

	if b.Config.CertFile != "" {
		cert, err := tls.LoadX509KeyPair(b.Config.CertFile, b.Config.KeyFile)
		if err != nil {
			log.Fatal(err)
		}
		b.listener = tls.NewListener(b.listener, &tls.Config{
			NextProtos:   []string{"http/1.1"},
			Certificates: []tls.Certificate{cert},
		})
	}

	// signal that we are ready now
	close(b.ready)

	lastErrorTime := time.Now()
	for {
		err := server.Serve(b.listener)
		if err != nil {
			// comes when the broker is closed with Close() method. This error
			// is defined in net/net.go as "var errClosing", unfortunaly it's
			// not exported.
			if strings.Contains(err.Error(), "use of closed network connection") {
				return
			}

			log.Warning("Server error: %v", err)
			if time.Now().Sub(lastErrorTime) < time.Second {
				log.Fatal(nil)
			}
			lastErrorTime = time.Now()
		}
	}

}

// sockjsSession is called for every client connection and handles all the
// message trafic for a single client connection.
func (b *Broker) sockjsSession(session *sockjs.Session) {

	client := NewClient(session, b)
	sessionGaugeEnd := client.gaugeStart()

	defer sessionGaugeEnd()
	defer client.Close()

	err := client.ControlChannel.Publish(b.Config.AuthAllExchange, "broker.clientConnected", false, false, amqp.Publishing{Body: []byte(client.SocketId)})
	if err != nil {
		log.Critical("Couldnt publish to control channel %v", err)
	}

	sendToClient(session, "broker.connected", client.SocketId)

	for data := range session.ReceiveChan {
		if data == nil || session.Closed {
			break
		}

		client.handleSessionMessage(data)
	}
}

// sendToClient sends the given payload back to the client. It attachs the
// routintKey along with the payload. It closes the session if sending fails.
func sendToClient(session *sockjs.Session, routingKey string, payload interface{}) {
	var message struct {
		RoutingKey string      `json:"routingKey"`
		Payload    interface{} `json:"payload"`
	}
	message.RoutingKey = routingKey
	message.Payload = payload
	if !session.Send(message) {
		session.Close()
		log.Warning("Dropped session because of broker to client buffer overflow. %v", session.Tag)
	}
}
<commit_msg>Broker: return error while registering to kontrol if there is<commit_after>package main

import (
	"crypto/tls"
	"encoding/json"
	"flag"
	"koding/kontrol/kontrolhelper"
	"koding/tools/amqputil"
	"koding/tools/config"
	"koding/tools/lifecycle"
	"koding/tools/logger"
	"koding/tools/sockjs"
	"koding/tools/utils"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/fatih/set"
	"github.com/streadway/amqp"
)

const BROKER_NAME = "broker"

var (
	conf *config.Config
	log  = logger.New(BROKER_NAME)

	STORAGE_BACKEND = "redis"
	routeMap        = make(map[string]*set.Set)
	sessionsMap     = make(map[string]*sockjs.Session)
	globalMapMutex  sync.Mutex

	changeClientsGauge          = lifecycle.CreateClientsGauge()
	changeNewClientsGauge       = logger.CreateCounterGauge("newClients", logger.NoUnit, true)
	changeWebsocketClientsGauge = logger.CreateCounterGauge("websocketClients", logger.NoUnit, false)

	flagProfile      = flag.String("c", "", "Configuration profile from file")
	flagBrokerDomain = flag.String("a", "", "Send kontrol a custom domain istead of os.Hostname")
	flagKontrolUUID  = flag.String("u", "", "Enable Kontrol mode")
	flagBrokerType   = flag.String("b", "broker", "Define broker type. Available: broker and brokerKite. B")
	flagDebug        = flag.Bool("d", false, "Debug mode")
)

// Broker is a router/multiplexer that routes messages coming from a SockJS
// server to an AMQP exchange and vice versa. Broker basically listens to
// client messages (Koding users) from the SockJS server. The message is
// either passed to the appropriate exchange or a response is sent back to the
// client. Each message has an "action" field that defines how to act for a
// received message.
type Broker struct {
	Config            *config.Broker
	Hostname          string
	ServiceUniqueName string
	AuthAllExchange   string
	PublishConn       *amqp.Connection
	ConsumeConn       *amqp.Connection

	// Accepts SockJS connections
	listener net.Listener

	// Closed when SockJS server is ready to acccept connections
	ready chan struct{}
}

// NewBroker returns a new Broker instance with ServiceUniqueName and Hostname
// prepopulated. After creating a Broker instance, one has to call
// broker.Run() or broker.Start() to start the broker instance and call
// broker.Close() for a graceful stop.
func NewBroker() *Broker {
	// returns os.Hostname() if config.BrokerDomain is empty, otherwise it just
	// returns config.BrokerDomain back
	brokerHostname := kontrolhelper.CustomHostname(*flagBrokerDomain)
	sanitizedHostname := strings.Replace(brokerHostname, ".", "_", -1)
	serviceUniqueName := BROKER_NAME + "|" + sanitizedHostname

	return &Broker{
		Hostname:          brokerHostname,
		ServiceUniqueName: serviceUniqueName,
		ready:             make(chan struct{}),
	}
}

func main() {
	flag.Parse()
	if *flagProfile == "" {
		log.Fatal("Please specify profile via -c. Aborting.")
	}

	conf = config.MustConfig(*flagProfile)
	broker := NewBroker()

	switch *flagBrokerType {
	case "brokerKite":
		broker.Config = &conf.BrokerKite
	default:
		broker.Config = &conf.Broker
	}

	// update broker name
	log = logger.New(broker.Config.Name)
	var logLevel logger.Level
	if *flagDebug {
		logLevel = logger.DEBUG
	} else {
		logLevel = logger.GetLoggingLevelFromConfig(BROKER_NAME, *flagProfile)
	}

	log.SetLevel(logLevel)
	broker.Run()
}

// Run starts the broker.
func (b *Broker) Run() {
	lifecycle.Startup(BROKER_NAME, false)
	logger.RunGaugesLoop(log)

	if err := b.registerToKontrol(); err != nil {
		log.Critical("Couldnt register to kontrol, stopping... %v", err)
		return
	}

	if err := b.startAMQP(); err != nil {
		log.Critical("Couldnt create amqp bindings, stopping... %v", err)
		return
	}

	b.startSockJS() // blocking
}

// Start is like Run() but waits until the SockJS listener is ready to be
// used.
func (b *Broker) Start() {
	go b.Run()
	<-b.ready
}

// Close close all amqp connections and closes the SockJS server listener
func (b *Broker) Close() {
	b.PublishConn.Close()
	b.ConsumeConn.Close()
	b.listener.Close()
}

// registerToKontrol registers the broker to KontrolDaemon. This is needed to
// populate a list of brokers and show them to the client. The list is
// available at: https://koding.com/-/services/broker?all
func (b *Broker) registerToKontrol() error {
	if err := kontrolhelper.RegisterToKontrol(
		conf,
		b.Config.Name,
		b.Config.Name, // servicGenericName
		b.ServiceUniqueName,
		*flagKontrolUUID,
		b.Hostname,
		b.Config.Port,
	); err != nil {
		return err
	}
	return nil
}

// startAMQP setups the the neccesary publisher and consumer connections for
// the broker broker.
func (b *Broker) startAMQP() {
	b.PublishConn = amqputil.CreateConnection(conf, b.Config.Name)
	defer b.PublishConn.Close()

	b.ConsumeConn = amqputil.CreateConnection(conf, b.Config.Name)
	defer b.ConsumeConn.Close()

	consumeChannel := amqputil.CreateChannel(b.ConsumeConn)
	defer consumeChannel.Close()

	presenceQueue := amqputil.JoinPresenceExchange(
		consumeChannel,      // channel
		"services-presence", // exchange
		b.Config.Name,       // serviceType
		b.Config.Name,       // serviceGenericName
		b.ServiceUniqueName, // serviceUniqueName
		false,               // loadBalancing
	)

	go func() {
		sigusr1Channel := make(chan os.Signal)
		signal.Notify(sigusr1Channel, syscall.SIGUSR1)
		<-sigusr1Channel
		consumeChannel.QueueDelete(presenceQueue, false, false, false)
	}()

	stream := amqputil.DeclareBindConsumeQueue(consumeChannel, "topic", b.Config.Name, "#", false)

	if err := consumeChannel.ExchangeDeclare(
		"updateInstances", // name
		"fanout",          // kind
		false,             // durable
		false,             // autoDelete
		false,             // internal
		false,             // noWait
		nil,               // args
	); err != nil {
		log.Critical("Couldnt create updateInstances exchange  %v", err)
	}

	if err := consumeChannel.ExchangeBind(BROKER_NAME, "", "updateInstances", false, nil); err != nil {
		log.Critical("Couldnt bind to updateInstances exchange  %v", err)
	}

	// signal that we are ready now
	close(b.amqpReady)

	// start to listen from "broker" topic exchange
	for amqpMessage := range stream {
		routingKey := amqpMessage.RoutingKey
		payload := json.RawMessage(utils.FilterInvalidUTF8(amqpMessage.Body))

		pos := strings.IndexRune(routingKey, '.') // skip first dot, since we want at least two components to always include the secret
		for pos != -1 && pos < len(routingKey) {
			index := strings.IndexRune(routingKey[pos+1:], '.')
			pos += index + 1
			if index == -1 {
				pos = len(routingKey)
			}
			routingKeyPrefix := routingKey[:pos]
			globalMapMutex.Lock()

			if routes, ok := routeMap[routingKeyPrefix]; ok {
				routes.Each(func(sessionId interface{}) bool {
					if routeSession, ok := sessionsMap[sessionId.(string)]; ok {
						sendToClient(routeSession, routingKey, &payload)
					}
					return true
				})
			}
			globalMapMutex.Unlock()
		}
	}
}

// startSockJS starts a new HTTPS listener that implies the SockJS protocol.
func (b *Broker) startSockJS() {
	service := sockjs.NewService(
		conf.Client.StaticFilesBaseUrl+"/js/sock.js",
		10*time.Minute,
		b.sockjsSession,
	)
	defer service.Close()

	service.MaxReceivedPerSecond = 50
	service.ErrorHandler = log.LogError

	// TODO use http.Mux instead of sockjs.Mux.
	mux := &sockjs.Mux{
		Handlers: map[string]http.Handler{
			"/subscribe": service,
			"/buildnumber": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("Content-Type", "text/plain")
				w.Write([]byte(strconv.Itoa(conf.BuildNumber)))
			}),
		},
	}

	server := &http.Server{Handler: mux}

	var err error
	b.listener, err = net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP(b.Config.IP), Port: b.Config.Port})
	if err != nil {
		log.Fatal(err)
	}

	if b.Config.CertFile != "" {
		cert, err := tls.LoadX509KeyPair(b.Config.CertFile, b.Config.KeyFile)
		if err != nil {
			log.Fatal(err)
		}
		b.listener = tls.NewListener(b.listener, &tls.Config{
			NextProtos:   []string{"http/1.1"},
			Certificates: []tls.Certificate{cert},
		})
	}

	// signal that we are ready now
	close(b.ready)

	lastErrorTime := time.Now()
	for {
		err := server.Serve(b.listener)
		if err != nil {
			// comes when the broker is closed with Close() method. This error
			// is defined in net/net.go as "var errClosing", unfortunaly it's
			// not exported.
			if strings.Contains(err.Error(), "use of closed network connection") {
				return
			}

			log.Warning("Server error: %v", err)
			if time.Now().Sub(lastErrorTime) < time.Second {
				log.Fatal(nil)
			}
			lastErrorTime = time.Now()
		}
	}

}

// sockjsSession is called for every client connection and handles all the
// message trafic for a single client connection.
func (b *Broker) sockjsSession(session *sockjs.Session) {

	client := NewClient(session, b)
	sessionGaugeEnd := client.gaugeStart()

	defer sessionGaugeEnd()
	defer client.Close()

	err := client.ControlChannel.Publish(b.Config.AuthAllExchange, "broker.clientConnected", false, false, amqp.Publishing{Body: []byte(client.SocketId)})
	if err != nil {
		log.Critical("Couldnt publish to control channel %v", err)
	}

	sendToClient(session, "broker.connected", client.SocketId)

	for data := range session.ReceiveChan {
		if data == nil || session.Closed {
			break
		}

		client.handleSessionMessage(data)
	}
}

// sendToClient sends the given payload back to the client. It attachs the
// routintKey along with the payload. It closes the session if sending fails.
func sendToClient(session *sockjs.Session, routingKey string, payload interface{}) {
	var message struct {
		RoutingKey string      `json:"routingKey"`
		Payload    interface{} `json:"payload"`
	}
	message.RoutingKey = routingKey
	message.Payload = payload
	if !session.Send(message) {
		session.Close()
		log.Warning("Dropped session because of broker to client buffer overflow. %v", session.Tag)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
)

type empty struct{}

// Note: struct{} instead of int
var prereqs = map[string]map[string]empty{
	"algorithms":     {"data structures": empty{}},
	"calculus":       {"linear algebra": empty{}},
	"linear algebra": {"calculus": empty{}},

	"compilers": {
		"data structures":       empty{},
		"formal languages":      empty{},
		"computer organization": empty{},
	},

	"data structures":  {"discrete math": empty{}},
	"databases":        {"data structures": empty{}},
	"discrete math":    {"intro to programming": empty{}},
	"formal languages": {"discrete math": empty{}},
	"networks":         {"operating systems": empty{}},
	"operating systems": {
		"data structures":       empty{},
		"computer organization": empty{},
	},
	"programming languages": {
		"data structures":       empty{},
		"computer organization": empty{},
	},
}

const (
	white = iota
	gray
	black
)

func main() {
	courses, err := topoSort(prereqs)
	if err != nil {
		log.Fatal("cyclic dependency detected ")
	}
	for i, course := range courses {
		fmt.Printf("%d:\t%s\n", i+1, course)
	}
}

func topoSort(m map[string]map[string]empty) ([]string, error) {
	var order []string
	seen := make(map[string]int)

	var visitAll func(map[string]empty) error

	visitAll = func(items map[string]empty) error {
		for item := range items {
			if seen[item] == white {
				seen[item] = gray
				if err := visitAll(m[item]); err != nil {
					return err
				}
				order = append(order, item)
				seen[item] = black
			}
			if seen[item] == gray {
				// panic("cycled graph")
				return fmt.Errorf("%s", "cycled graph")
			}
		}
		return nil
	}
	keys := make(map[string]empty, len(m))
	for key := range m {
		keys[key] = empty{}
	}

	// sort.Strings(keys)
	if err := visitAll(keys); err != nil {
		return nil, err
	}

	return order, nil
}
<commit_msg>[5.11] Updated error massge.<commit_after>package main

import (
	"fmt"
	"log"
)

type empty struct{}

// Note: struct{} instead of int
var prereqs = map[string]map[string]empty{
	"algorithms":     {"data structures": empty{}},
	"calculus":       {"linear algebra": empty{}},
	"linear algebra": {"calculus": empty{}},

	"compilers": {
		"data structures":       empty{},
		"formal languages":      empty{},
		"computer organization": empty{},
	},

	"data structures":  {"discrete math": empty{}},
	"databases":        {"data structures": empty{}},
	"discrete math":    {"intro to programming": empty{}},
	"formal languages": {"discrete math": empty{}},
	"networks":         {"operating systems": empty{}},
	"operating systems": {
		"data structures":       empty{},
		"computer organization": empty{},
	},
	"programming languages": {
		"data structures":       empty{},
		"computer organization": empty{},
	},
}

const (
	white = iota
	gray
	black
)

func main() {
	courses, err := topoSort(prereqs)
	if err != nil {
		log.Fatalf("error: %v", err)
	}
	for i, course := range courses {
		fmt.Printf("%d:\t%s\n", i+1, course)
	}
}

func topoSort(m map[string]map[string]empty) ([]string, error) {
	var order []string
	seen := make(map[string]int)

	var visitAll func(map[string]empty) error

	visitAll = func(items map[string]empty) error {
		for item := range items {
			if seen[item] == white {
				seen[item] = gray
				if err := visitAll(m[item]); err != nil {
					return err
				}
				seen[item] = black
				order = append(order, item)
			}
			if seen[item] == gray {
				// panic("cycled graph")
				return fmt.Errorf("%s", "cycled graph")
			}
		}
		return nil
	}
	keys := make(map[string]empty, len(m))
	for key := range m {
		keys[key] = empty{}
	}

	// sort.Strings(keys)
	if err := visitAll(keys); err != nil {
		return nil, err
	}

	return order, nil
}
<|endoftext|>


<commit_before>package outflux

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"sync"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/net/context/ctxhttp"

	"github.com/nilium/dagr"
	"github.com/nilium/dagr/outflux/internal/dubb"
)

type BadStatusError int

func (e BadStatusError) Error() string {
	return fmt.Sprintf("outflux: bad status returned: %d", int(e))
}

type WriteFunc func(io.Writer) error

type Proxy struct {
	destURL *url.URL
	buffer  *dubb.Buffer
	client  *http.Client

	startOnce sync.Once
	cancel    context.CancelFunc
	ctx       context.Context
}

func NewURL(ctx context.Context, client *http.Client, destURL *url.URL) *Proxy {
	if destURL == nil {
		panic("outflux: destination url is nil")
	}

	if ctx == nil {
		ctx = context.Background()
	}
	ctx, cancel := context.WithCancel(ctx)

	if client == nil {
		client = http.DefaultClient
	}

	return &Proxy{
		destURL: destURL,
		buffer:  dubb.NewBuffer(64000),
		client:  client,
		cancel:  cancel,
		ctx:     ctx,
	}
}

func New(ctx context.Context, client *http.Client, destURL string) *Proxy {
	if destURL == "" {
		panic("outflux: destination url is nil")
	}

	du, err := url.Parse(destURL)
	if err != nil {
		panic(fmt.Sprintf("outflux: error parsing url: %v", err))
	}

	return NewURL(ctx, client, du)
}

// Close stops the Proxy's runloop, if it was ever started. The Proxy is no longer usable if closed.
func (w *Proxy) Close() error {
	if err := w.ctx.Err(); err != nil {
		return err
	}
	w.cancel()
	return nil
}

type nopWriteCloser struct{ io.Writer }

func (nopWriteCloser) Close() error { return nil }

// Write writes the byte slice b to the write buffer of the Proxy. WriteMeasurements should be preferred to ensure that
// the writer is correctly sending InfluxDB line protocol messages, but may be used as a raw writer to the underlying
// Proxy buffers.
func (w *Proxy) Write(b []byte) (int, error) {
	if err := w.ctx.Err(); err != nil {
		return 0, err
	}

	return w.buffer.Write(b)
}

// Writer returns a locked writer for the Proxy's write buffer. It must be closed to release the lock.
func (w *Proxy) Writer() io.WriteCloser {
	if err := w.ctx.Err(); err != nil {
		return nopWriteCloser{ioutil.Discard}
	}

	return w.buffer.Writer()
}

// Transaction locks the Proxy's write buffer and passes it to fn. Once fn completes, the lock is released. This is
// shorthand for just doing that yourself, in the event that you have a function to pass a writer to but want to avoid
// writing extra code. For example:
//
//      err := proxy.Transaction(func(w io.Writer) {
//              _, err := buf.WriteTo(w)
//              return err
//      })
//
//      if err != nil {
//              // ...
//      }
//
// The WriteFunc given may return an error. This has no effect on the outcome of the transaction and is entirely for
// convenience. If the Proxy is closed, it will return the context error for its closure.
func (w *Proxy) Transaction(fn WriteFunc) error {
	if err := w.ctx.Err(); err != nil {
		return err
	}

	wx := w.Writer()
	defer logclose(wx)
	return fn(wx)
}

// WriteMeasurements writes all measurements in measurements to the Proxy, effectively queueing them for delivery.
func (w *Proxy) WriteMeasurements(measurements ...dagr.Measurement) (n int64, err error) {
	if err := w.ctx.Err(); err != nil {
		return 0, err
	} else if len(measurements) == 0 {
		return 0, nil
	}

	return dagr.WriteMeasurements(w, measurements...)
}

func (w *Proxy) WriteMeasurement(measurement dagr.Measurement) (n int64, err error) {
	if err := w.ctx.Err(); err != nil {
		return 0, err
	}

	return dagr.WriteMeasurement(w, measurement)
}

// WritePoint writes a single point to the Proxy.
func (w *Proxy) WritePoint(key string, when time.Time, tags dagr.Tags, fields dagr.Fields) (n int64, err error) {
	if err := w.ctx.Err(); err != nil {
		return 0, err
	}

	if key == "" {
		logf("Empty key in point")
		return 0, dagr.ErrEmptyKey
	} else if len(fields) == 0 {
		logf("No fields in point %q", key)
		return 0, dagr.ErrNoFields
	}

	if when.IsZero() {
		when = time.Now()
	}

	return dagr.WriteMeasurement(w, dagr.RawPoint{key, tags, fields, when})
}

func (w *Proxy) Start(interval time.Duration) context.CancelFunc {
	w.startOnce.Do(func() {
		go w.sendEveryInterval(interval)
	})

	return w.cancel
}

func (w *Proxy) sendEveryInterval(interval time.Duration) {
	tick := time.NewTicker(interval)
	defer tick.Stop()

	for {
		select {
		case <-w.ctx.Done():
			return
		case <-tick.C:
			w.swapAndSend()
		}
	}
}

func (w *Proxy) swapAndSend() {
	buf := w.buffer

	buf.Swap()
	rd := buf.Reader()
	defer func() {
		rd.Truncate(0)
		logclose(rd)
	}()

	if rd.Len() == 0 {
		// Nothing to do.
		return
	}

	if err := w.send(rd); err != nil {

	}
}

func (w *Proxy) coder(rd dubb.Reader) (rc io.ReadCloser, encoding string) {
	if rd.Len() <= 1000 {
		// We truncate and close the dubb reader elsewhere, so make it a nop here.
		return ioutil.NopCloser(rd), ""
	}

	pr, pw := io.Pipe()
	enc := gzip.NewWriter(pw)
	go func() {
		if _, err := rd.WriteTo(enc); err != nil {
			logf("Error writing measurements: %v", err)
		}
		if err := enc.Close(); err != nil {
			logf("Error closing gzip encoder: %v", err)
		}
		if err := pw.Close(); err != nil {
			logf("Error closing pipe writer: %v", err)
		}
	}()

	return pr, "gzip"
}

func (w *Proxy) send(rd dubb.Reader) error {
	pr, encoding := w.coder(rd)
	req, err := http.NewRequest("POST", w.destURL.String(), pr)
	if err != nil {
		defer pr.Close()
		logf("Error creating request: %v", err)
		return err
	}

	req.Header.Set("Content-Type", "")
	if encoding != "" {
		req.Header.Set("Content-Encoding", encoding)
	}

	if ui := req.URL.User; ui != nil {
		pw, _ := ui.Password()
		req.SetBasicAuth(ui.Username(), pw)
		req.URL.User = nil
	}

	resp, err := ctxhttp.Do(w.ctx, w.client, req)
	if err != nil {
		logf("Error posting to InfluxDB: %v", err)
		return err
	}
	defer func() {
		// Discard response.
		if _, err := io.Copy(ioutil.Discard, resp.Body); err != nil {
			logf("Error discarding InfluxDB response body: %v", err)
		}
		logclose(resp.Body)
	}()

	// Ideally we'll get status 204, but we discard anything from InfluxDB that's a success.
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		var buf bytes.Buffer
		if _, err := io.Copy(&buf, resp.Body); err != nil {
			logf("Unable to copy body in measurement send: %v", err)
			return err
		}

		logf("Bad response from InfluxDB (%d): %s", resp.StatusCode, buf.Bytes())
		return BadStatusError(resp.StatusCode)
	}

	return nil
}
<commit_msg>Add optional context-based timeout to HTTP requests<commit_after>package outflux

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"sync"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/net/context/ctxhttp"

	"github.com/nilium/dagr"
	"github.com/nilium/dagr/outflux/internal/dubb"
)

type BadStatusError int

func (e BadStatusError) Error() string {
	return fmt.Sprintf("outflux: bad status returned: %d", int(e))
}

type WriteFunc func(io.Writer) error

type Proxy struct {
	destURL *url.URL
	buffer  *dubb.Buffer
	client  *http.Client
	timeout time.Duration

	startOnce sync.Once
	cancel    context.CancelFunc
	ctx       context.Context
}

func NewURL(ctx context.Context, timeout time.Duration, client *http.Client, destURL *url.URL) *Proxy {
	if destURL == nil {
		panic("outflux: destination url is nil")
	}

	if ctx == nil {
		ctx = context.Background()
	}
	ctx, cancel := context.WithCancel(ctx)

	if client == nil {
		client = http.DefaultClient
	}

	return &Proxy{
		destURL: destURL,
		buffer:  dubb.NewBuffer(64000),
		client:  client,
		timeout: timeout,
		cancel:  cancel,
		ctx:     ctx,
	}
}

func New(ctx context.Context, timeout time.Duration, client *http.Client, destURL string) *Proxy {
	if destURL == "" {
		panic("outflux: destination url is nil")
	}

	du, err := url.Parse(destURL)
	if err != nil {
		panic(fmt.Sprintf("outflux: error parsing url: %v", err))
	}

	return NewURL(ctx, timeout, client, du)
}

// Close stops the Proxy's runloop, if it was ever started. The Proxy is no longer usable if closed.
func (w *Proxy) Close() error {
	if err := w.ctx.Err(); err != nil {
		return err
	}
	w.cancel()
	return nil
}

type nopWriteCloser struct{ io.Writer }

func (nopWriteCloser) Close() error { return nil }

// Write writes the byte slice b to the write buffer of the Proxy. WriteMeasurements should be preferred to ensure that
// the writer is correctly sending InfluxDB line protocol messages, but may be used as a raw writer to the underlying
// Proxy buffers.
func (w *Proxy) Write(b []byte) (int, error) {
	if err := w.ctx.Err(); err != nil {
		return 0, err
	}

	return w.buffer.Write(b)
}

// Writer returns a locked writer for the Proxy's write buffer. It must be closed to release the lock.
func (w *Proxy) Writer() io.WriteCloser {
	if err := w.ctx.Err(); err != nil {
		return nopWriteCloser{ioutil.Discard}
	}

	return w.buffer.Writer()
}

// Transaction locks the Proxy's write buffer and passes it to fn. Once fn completes, the lock is released. This is
// shorthand for just doing that yourself, in the event that you have a function to pass a writer to but want to avoid
// writing extra code. For example:
//
//      err := proxy.Transaction(func(w io.Writer) {
//              _, err := buf.WriteTo(w)
//              return err
//      })
//
//      if err != nil {
//              // ...
//      }
//
// The WriteFunc given may return an error. This has no effect on the outcome of the transaction and is entirely for
// convenience. If the Proxy is closed, it will return the context error for its closure.
func (w *Proxy) Transaction(fn WriteFunc) error {
	if err := w.ctx.Err(); err != nil {
		return err
	}

	wx := w.Writer()
	defer logclose(wx)
	return fn(wx)
}

// WriteMeasurements writes all measurements in measurements to the Proxy, effectively queueing them for delivery.
func (w *Proxy) WriteMeasurements(measurements ...dagr.Measurement) (n int64, err error) {
	if err := w.ctx.Err(); err != nil {
		return 0, err
	} else if len(measurements) == 0 {
		return 0, nil
	}

	return dagr.WriteMeasurements(w, measurements...)
}

func (w *Proxy) WriteMeasurement(measurement dagr.Measurement) (n int64, err error) {
	if err := w.ctx.Err(); err != nil {
		return 0, err
	}

	return dagr.WriteMeasurement(w, measurement)
}

// WritePoint writes a single point to the Proxy.
func (w *Proxy) WritePoint(key string, when time.Time, tags dagr.Tags, fields dagr.Fields) (n int64, err error) {
	if err := w.ctx.Err(); err != nil {
		return 0, err
	}

	if key == "" {
		logf("Empty key in point")
		return 0, dagr.ErrEmptyKey
	} else if len(fields) == 0 {
		logf("No fields in point %q", key)
		return 0, dagr.ErrNoFields
	}

	if when.IsZero() {
		when = time.Now()
	}

	return dagr.WriteMeasurement(w, dagr.RawPoint{key, tags, fields, when})
}

func (w *Proxy) Start(interval time.Duration) context.CancelFunc {
	w.startOnce.Do(func() {
		go w.sendEveryInterval(interval)
	})

	return w.cancel
}

func (w *Proxy) sendEveryInterval(interval time.Duration) {
	tick := time.NewTicker(interval)
	defer tick.Stop()

	for {
		select {
		case <-w.ctx.Done():
			return
		case <-tick.C:
			w.swapAndSend()
		}
	}
}

func (w *Proxy) swapAndSend() {
	buf := w.buffer

	buf.Swap()
	rd := buf.Reader()
	defer func() {
		rd.Truncate(0)
		logclose(rd)
	}()

	if rd.Len() == 0 {
		// Nothing to do.
		return
	}

	if err := w.send(rd); err != nil {

	}
}

func (w *Proxy) coder(rd dubb.Reader) (rc io.ReadCloser, encoding string) {
	if rd.Len() <= 1000 {
		// We truncate and close the dubb reader elsewhere, so make it a nop here.
		return ioutil.NopCloser(rd), ""
	}

	pr, pw := io.Pipe()
	enc := gzip.NewWriter(pw)
	go func() {
		if _, err := rd.WriteTo(enc); err != nil {
			logf("Error writing measurements: %v", err)
		}
		if err := enc.Close(); err != nil {
			logf("Error closing gzip encoder: %v", err)
		}
		if err := pw.Close(); err != nil {
			logf("Error closing pipe writer: %v", err)
		}
	}()

	return pr, "gzip"
}

func (w *Proxy) send(rd dubb.Reader) error {
	if err := w.ctx.Err(); err != nil {
		return err
	}

	ctx := w.ctx
	if timeout := w.timeout; timeout > 0 {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, timeout)
		defer cancel()
	}
	pr, encoding := w.coder(rd)
	req, err := http.NewRequest("POST", w.destURL.String(), pr)
	if err != nil {
		defer pr.Close()
		logf("Error creating request: %v", err)
		return err
	}

	req.Header.Set("Content-Type", "")
	if encoding != "" {
		req.Header.Set("Content-Encoding", encoding)
	}

	if ui := req.URL.User; ui != nil {
		pw, _ := ui.Password()
		req.SetBasicAuth(ui.Username(), pw)
		req.URL.User = nil
	}

	resp, err := ctxhttp.Do(ctx, w.client, req)
	if err != nil {
		logf("Error posting to InfluxDB: %v", err)
		return err
	}
	defer func() {
		// Discard response.
		if _, err := io.Copy(ioutil.Discard, resp.Body); err != nil {
			logf("Error discarding InfluxDB response body: %v", err)
		}
		logclose(resp.Body)
	}()

	// Ideally we'll get status 204, but we discard anything from InfluxDB that's a success.
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		var buf bytes.Buffer
		if _, err := io.Copy(&buf, resp.Body); err != nil {
			logf("Unable to copy body in measurement send: %v", err)
			return err
		}

		logf("Bad response from InfluxDB (%d): %s", resp.StatusCode, buf.Bytes())
		return BadStatusError(resp.StatusCode)
	}

	return nil
}
<|endoftext|>


<commit_before>package cache

import (
	"fmt"
	"sync"
	"time"

	"github.com/johnny-morrice/godless/api"
	"github.com/johnny-morrice/godless/crdt"
	"github.com/johnny-morrice/godless/log"
	"github.com/johnny-morrice/godless/query"
	"github.com/pkg/errors"
)

// Memcache style key value store.
// Will drop oldest.
type residentIndexCache struct {
	sync.RWMutex
	buff  []indexCacheItem
	assoc map[crdt.IPFSPath]*indexCacheItem
}

func MakeResidentIndexCache(buffSize int) api.IndexCache {
	if buffSize <= 0 {
		buffSize = __DEFAULT_BUFFER_SIZE
	}

	return &residentIndexCache{
		buff:  make([]indexCacheItem, buffSize),
		assoc: map[crdt.IPFSPath]*indexCacheItem{},
	}
}

type indexCacheItem struct {
	key       crdt.IPFSPath
	index     crdt.Index
	timestamp int
}

func (cache *residentIndexCache) GetIndex(indexAddr crdt.IPFSPath) (crdt.Index, error) {
	cache.RLock()
	defer cache.RUnlock()

	item, present := cache.assoc[indexAddr]

	if !present {
		return crdt.EmptyIndex(), fmt.Errorf("No cached index for: %v", indexAddr)
	}

	return item.index, nil
}

func (cache *residentIndexCache) SetIndex(indexAddr crdt.IPFSPath, index crdt.Index) error {
	cache.Lock()
	defer cache.Unlock()

	item, present := cache.assoc[indexAddr]

	if present {
		item.timestamp = makeTimestamp()
		return nil
	}

	return cache.addNewItem(indexAddr, index)
}

func (cache *residentIndexCache) addNewItem(indexAddr crdt.IPFSPath, index crdt.Index) error {
	newItem := indexCacheItem{
		timestamp: makeTimestamp(),
		key:       indexAddr,
		index:     index,
	}

	bufferedItem := cache.findOldest()
	*bufferedItem = newItem
	cache.assoc[indexAddr] = bufferedItem
	return nil
}

func (cache *residentIndexCache) findOldest() *indexCacheItem {
	var oldest *indexCacheItem

	for _, item := range cache.buff {
		if oldest == nil {
			oldest = &item
			continue
		}

		if item.timestamp < oldest.timestamp {
			oldest = &item
		}
	}

	if oldest == nil {
		panic("Corrupt buffer")
	}

	return oldest
}

func makeTimestamp() int {
	return time.Now().Nanosecond()
}

type residentHeadCache struct {
	sync.RWMutex
	writing  bool
	written  bool
	current  crdt.IPFSPath
	previous crdt.IPFSPath
}

func (cache *residentHeadCache) BeginReadTransaction() error {
	cache.RLock()
	return nil
}

func (cache *residentHeadCache) BeginWriteTransaction() error {
	cache.Lock()
	cache.writing = true
	return nil
}

func (cache *residentHeadCache) SetHead(head crdt.IPFSPath) error {
	cache.previous = cache.current
	cache.current = head
	cache.written = true
	return nil
}

func (cache *residentHeadCache) Commit() error {
	if cache.writing {
		cache.previous = ""
		cache.writing = false
		cache.written = false
		cache.Unlock()
	} else {
		cache.RUnlock()
	}

	return nil
}

func (cache *residentHeadCache) Rollback() error {
	if !cache.writing {
		return errors.New("Cannot rollback without write")
	}

	if cache.written {
		cache.current = cache.previous
	}

	return cache.Commit()
}

func (cache *residentHeadCache) GetHead() (crdt.IPFSPath, error) {
	head := cache.current
	return head, nil
}

func MakeResidentHeadCache() api.HeadCache {
	return &residentHeadCache{}
}

type residentPriorityQueue struct {
	sync.Mutex
	semaphore chan struct{}
	buff      []residentQueueItem
	datach    chan interface{}
}

func MakeResidentBufferQueue(buffSize int) api.RequestPriorityQueue {
	if buffSize <= 0 {
		buffSize = __DEFAULT_BUFFER_SIZE
	}

	queue := &residentPriorityQueue{
		semaphore: make(chan struct{}, buffSize),
		buff:      make([]residentQueueItem, buffSize),
		datach:    make(chan interface{}),
	}

	return queue
}

func (queue *residentPriorityQueue) Enqueue(request api.APIRequest, data interface{}) error {
	item, err := makeResidentQueueItem(request, data)

	if err != nil {
		return errors.Wrap(err, "residentPriorityQueue.Enqueue failed")
	}

	queue.lockResource()
	queue.Lock()
	defer queue.Unlock()

	for i := 0; i < len(queue.buff); i++ {
		spot := &queue.buff[i]
		if !spot.populated {
			*spot = item
			return nil
		}
	}

	return corruptBuffer
}

func (queue *residentPriorityQueue) Drain() <-chan interface{} {
	go func() {
		for {
			data, err := queue.popFront()

			if err != nil {
				log.Error("Error draining residentPriorityQueue: %v", err)
				return
			}

			queue.datach <- data
		}
	}()

	return queue.datach
}

func (queue *residentPriorityQueue) Close() error {
	close(queue.datach)
	return nil
}

func (queue *residentPriorityQueue) popFront() (interface{}, error) {
	queue.unlockResource()
	queue.Lock()
	defer queue.Unlock()

	var best *residentQueueItem
	for i := 0; i < len(queue.buff); i++ {
		spot := &queue.buff[i]
		if !spot.populated {
			continue
		}

		if best == nil {
			best = spot
			continue
		}

		if spot.priority < best.priority {
			best = spot
		}
	}

	if best == nil {
		log.Error("Buffer is corrupt")
		return nil, corruptBuffer
	}

	best.populated = false
	return best.data, nil
}

func (queue *residentPriorityQueue) lockResource() {
	queue.semaphore <- struct{}{}
}

func (queue *residentPriorityQueue) unlockResource() {
	<-queue.semaphore
}

type residentQueueItem struct {
	populated bool
	data      interface{}
	priority  residentPriority
}

func makeResidentQueueItem(request api.APIRequest, data interface{}) (residentQueueItem, error) {
	priority, err := findRequestPriority(request)

	if err != nil {
		return residentQueueItem{}, err
	}

	item := residentQueueItem{
		data:      data,
		priority:  priority,
		populated: true,
	}

	return item, nil
}

func findRequestPriority(request api.APIRequest) (residentPriority, error) {
	switch request.Type {
	case api.API_QUERY:
		if request.Query.OpCode == query.JOIN {
			return __QUERY_JOIN_PRIORITY, nil
		} else {
			return __QUERY_SELECT_PRIORITY, nil
		}
	case api.API_REFLECT:
		return __QUERY_REFLECT_PRIORITY, nil
	case api.API_REPLICATE:
		return __QUERY_REPLICATE_PRIORITY, nil
	default:
		return __UNKNOWN_PRIORITY, fmt.Errorf("Unknown request.Type: %v", request.Type)
	}
}

var corruptBuffer error = errors.New("Corrupt residentPriorityQueue buffer")

type residentPriority uint8

const (
	__QUERY_JOIN_PRIORITY = residentPriority(iota)
	__QUERY_REFLECT_PRIORITY
	__QUERY_SELECT_PRIORITY
	__QUERY_REPLICATE_PRIORITY
	__UNKNOWN_PRIORITY
)

const __DEFAULT_BUFFER_SIZE = 512
<commit_msg>Larger buffer<commit_after>package cache

import (
	"fmt"
	"sync"
	"time"

	"github.com/johnny-morrice/godless/api"
	"github.com/johnny-morrice/godless/crdt"
	"github.com/johnny-morrice/godless/log"
	"github.com/johnny-morrice/godless/query"
	"github.com/pkg/errors"
)

// Memcache style key value store.
// Will drop oldest.
type residentIndexCache struct {
	sync.RWMutex
	buff  []indexCacheItem
	assoc map[crdt.IPFSPath]*indexCacheItem
}

func MakeResidentIndexCache(buffSize int) api.IndexCache {
	if buffSize <= 0 {
		buffSize = __DEFAULT_BUFFER_SIZE
	}

	return &residentIndexCache{
		buff:  make([]indexCacheItem, buffSize),
		assoc: map[crdt.IPFSPath]*indexCacheItem{},
	}
}

type indexCacheItem struct {
	key       crdt.IPFSPath
	index     crdt.Index
	timestamp int
}

func (cache *residentIndexCache) GetIndex(indexAddr crdt.IPFSPath) (crdt.Index, error) {
	cache.RLock()
	defer cache.RUnlock()

	item, present := cache.assoc[indexAddr]

	if !present {
		return crdt.EmptyIndex(), fmt.Errorf("No cached index for: %v", indexAddr)
	}

	return item.index, nil
}

func (cache *residentIndexCache) SetIndex(indexAddr crdt.IPFSPath, index crdt.Index) error {
	cache.Lock()
	defer cache.Unlock()

	item, present := cache.assoc[indexAddr]

	if present {
		item.timestamp = makeTimestamp()
		return nil
	}

	return cache.addNewItem(indexAddr, index)
}

func (cache *residentIndexCache) addNewItem(indexAddr crdt.IPFSPath, index crdt.Index) error {
	newItem := indexCacheItem{
		timestamp: makeTimestamp(),
		key:       indexAddr,
		index:     index,
	}

	bufferedItem := cache.findOldest()
	*bufferedItem = newItem
	cache.assoc[indexAddr] = bufferedItem
	return nil
}

func (cache *residentIndexCache) findOldest() *indexCacheItem {
	var oldest *indexCacheItem

	for _, item := range cache.buff {
		if oldest == nil {
			oldest = &item
			continue
		}

		if item.timestamp < oldest.timestamp {
			oldest = &item
		}
	}

	if oldest == nil {
		panic("Corrupt buffer")
	}

	return oldest
}

func makeTimestamp() int {
	return time.Now().Nanosecond()
}

type residentHeadCache struct {
	sync.RWMutex
	writing  bool
	written  bool
	current  crdt.IPFSPath
	previous crdt.IPFSPath
}

func (cache *residentHeadCache) BeginReadTransaction() error {
	cache.RLock()
	return nil
}

func (cache *residentHeadCache) BeginWriteTransaction() error {
	cache.Lock()
	cache.writing = true
	return nil
}

func (cache *residentHeadCache) SetHead(head crdt.IPFSPath) error {
	cache.previous = cache.current
	cache.current = head
	cache.written = true
	return nil
}

func (cache *residentHeadCache) Commit() error {
	if cache.writing {
		cache.previous = ""
		cache.writing = false
		cache.written = false
		cache.Unlock()
	} else {
		cache.RUnlock()
	}

	return nil
}

func (cache *residentHeadCache) Rollback() error {
	if !cache.writing {
		return errors.New("Cannot rollback without write")
	}

	if cache.written {
		cache.current = cache.previous
	}

	return cache.Commit()
}

func (cache *residentHeadCache) GetHead() (crdt.IPFSPath, error) {
	head := cache.current
	return head, nil
}

func MakeResidentHeadCache() api.HeadCache {
	return &residentHeadCache{}
}

type residentPriorityQueue struct {
	sync.Mutex
	semaphore chan struct{}
	buff      []residentQueueItem
	datach    chan interface{}
}

func MakeResidentBufferQueue(buffSize int) api.RequestPriorityQueue {
	if buffSize <= 0 {
		buffSize = __DEFAULT_BUFFER_SIZE
	}

	queue := &residentPriorityQueue{
		semaphore: make(chan struct{}, buffSize),
		buff:      make([]residentQueueItem, buffSize),
		datach:    make(chan interface{}),
	}

	return queue
}

func (queue *residentPriorityQueue) Enqueue(request api.APIRequest, data interface{}) error {
	item, err := makeResidentQueueItem(request, data)

	if err != nil {
		return errors.Wrap(err, "residentPriorityQueue.Enqueue failed")
	}

	queue.lockResource()
	queue.Lock()
	defer queue.Unlock()

	for i := 0; i < len(queue.buff); i++ {
		spot := &queue.buff[i]
		if !spot.populated {
			*spot = item
			return nil
		}
	}

	return corruptBuffer
}

func (queue *residentPriorityQueue) Drain() <-chan interface{} {
	go func() {
		for {
			data, err := queue.popFront()

			if err != nil {
				log.Error("Error draining residentPriorityQueue: %v", err)
				return
			}

			queue.datach <- data
		}
	}()

	return queue.datach
}

func (queue *residentPriorityQueue) Close() error {
	close(queue.datach)
	return nil
}

func (queue *residentPriorityQueue) popFront() (interface{}, error) {
	queue.unlockResource()
	queue.Lock()
	defer queue.Unlock()

	var best *residentQueueItem
	for i := 0; i < len(queue.buff); i++ {
		spot := &queue.buff[i]
		if !spot.populated {
			continue
		}

		if best == nil {
			best = spot
			continue
		}

		if spot.priority < best.priority {
			best = spot
		}
	}

	if best == nil {
		log.Error("Buffer is corrupt")
		return nil, corruptBuffer
	}

	best.populated = false
	return best.data, nil
}

func (queue *residentPriorityQueue) lockResource() {
	queue.semaphore <- struct{}{}
}

func (queue *residentPriorityQueue) unlockResource() {
	<-queue.semaphore
}

type residentQueueItem struct {
	populated bool
	data      interface{}
	priority  residentPriority
}

func makeResidentQueueItem(request api.APIRequest, data interface{}) (residentQueueItem, error) {
	priority, err := findRequestPriority(request)

	if err != nil {
		return residentQueueItem{}, err
	}

	item := residentQueueItem{
		data:      data,
		priority:  priority,
		populated: true,
	}

	return item, nil
}

func findRequestPriority(request api.APIRequest) (residentPriority, error) {
	switch request.Type {
	case api.API_QUERY:
		if request.Query.OpCode == query.JOIN {
			return __QUERY_JOIN_PRIORITY, nil
		} else {
			return __QUERY_SELECT_PRIORITY, nil
		}
	case api.API_REFLECT:
		return __QUERY_REFLECT_PRIORITY, nil
	case api.API_REPLICATE:
		return __QUERY_REPLICATE_PRIORITY, nil
	default:
		return __UNKNOWN_PRIORITY, fmt.Errorf("Unknown request.Type: %v", request.Type)
	}
}

var corruptBuffer error = errors.New("Corrupt residentPriorityQueue buffer")

type residentPriority uint8

const (
	__QUERY_JOIN_PRIORITY = residentPriority(iota)
	__QUERY_REFLECT_PRIORITY
	__QUERY_SELECT_PRIORITY
	__QUERY_REPLICATE_PRIORITY
	__UNKNOWN_PRIORITY
)

const __DEFAULT_BUFFER_SIZE = 1024
<|endoftext|>


<commit_before>package debugstream

import (
	"bufio"
	"fmt"
	"io"
	"strconv"
	"strings"
	"sync"

	"github.com/oakmound/oak/v3/oakerr"
)

// ScopedCommands for the debug stream commands.
// Contains a set of scopes that align with oak.Controller.
// Currently can only be attached to a single stream
type ScopedCommands struct {
	sync.Mutex
	attachOnce   sync.Once
	assumedScope int32
	scopes       []int32
	commands     map[int32]map[string]func([]string)
}

// NewScopedCommands creates set of standard help functions.
func NewScopedCommands() *ScopedCommands {
	sc := &ScopedCommands{commands: map[int32]map[string]func([]string){}}
	sc.AddCommand("help", sc.printHelp)
	sc.AddCommand("scope", sc.assumeScope)
	sc.AddCommand("fade", fadeCommands)
	return sc
}

// AttachToStream and start executing the registered commands on input to said stream.
// Currently a given set of scoped commands may be attached once and only once.
func (sc *ScopedCommands) AttachToStream(input io.Reader) {
	if sc == nil {
		return
	}
	sc.attachOnce.Do(
		func() {
			scanner := bufio.NewScanner(input)
			go func() {
				for {
					for scanner.Scan() {

						// TODO: accept interrupts

						tokenString := strings.Fields(scanner.Text())
						if len(tokenString) == 0 {
							continue
						}
						// Attempt to parse the first arg as a scope
						scopeID, err := strToInt32(tokenString[0])

						tokenIDX := 0
						// if there was a scope specified then increment what we care about
						if err == nil {
							_, ok := sc.commands[scopeID]
							if !ok {
								fmt.Printf("unknown scopeID %d see correct usage via help\n", scopeID)
								continue
							}
							if len(tokenString) == 1 {
								fmt.Printf("Only provided scopeID %d without see usage via help\n", scopeID)
								continue
							}

							tokenIDX++
							// see if specified
							if fn, ok := sc.commands[scopeID][tokenString[tokenIDX]]; ok {
								fn(tokenString[tokenIDX+1:])
								continue
							}
						}

						// assumedscope
						if fn, ok := sc.commands[sc.assumedScope][tokenString[0]]; ok {
							fn(tokenString[1:])
							continue
						}

						// fallback to scope 0
						if fn, ok := sc.commands[0][tokenString[0]]; ok {
							fn(tokenString[1:])
							continue
						}

						fmt.Printf("Unknown command '%s' for scopeID %d see correct usage via help or help %d\n", tokenString[tokenIDX], scopeID, scopeID)

					}
				}
			}()
		})
}

// AddCommand adds a console command to call fn when
// '<s> <args>' is input to the console. fn will be called
// with args split on whitespace.
func (sc *ScopedCommands) AddCommand(s string, fn func([]string)) error {
	// We tightly link to controllerIDs here for better or for worse and controllerIDs shall always be > 0
	// This means our unscoped commands are safe when set as 0.
	return sc.AddScopedCommand(0, s, fn)
}

// AddScopedCommand adds a console command for a given window scope to call fn when
// '<s> <args>' is input to the console. fn will be called
// with args split on whitespace.
func (sc *ScopedCommands) AddScopedCommand(scopeID int32, s string, fn func([]string)) error {
	return sc.addCommand(scopeID, s, fn, false)
}

// addCommand for future executions.
func (sc *ScopedCommands) addCommand(scopeID int32, s string, fn func([]string), force bool) error {

	sc.Lock()
	defer sc.Unlock()

	if _, ok := sc.commands[scopeID]; !ok {

		sc.commands[scopeID] = map[string]func([]string){}
		sc.scopes = append(sc.scopes, scopeID)
	}

	if _, ok := sc.commands[scopeID][s]; ok {
		if !force {
			return oakerr.ExistingElement{
				InputName:   s,
				InputType:   "string",
				Overwritten: false,
			}
		}
	}
	sc.commands[scopeID][s] = fn

	return nil
}

// ClearCommand clears an existing debug command for scope with key: <s>
func (sc *ScopedCommands) ClearCommand(scopeID int32, s string) {
	_, ok := sc.commands[scopeID]
	if !ok {
		return
	}
	delete(sc.commands[scopeID], s)
}

// ResetCommands will throw out all existing debug commands from the
// debug console.
func (sc *ScopedCommands) ResetCommands() {
	sc.commands = map[int32]map[string]func([]string){}
}

// ResetCommandsForScope will throw out all existing debug commands from the
// debug console for hte given scope.
func (sc *ScopedCommands) ResetCommandsForScope(scope int32) {
	sc.commands[scope] = map[string]func([]string){}
}

// RemoveScope from the command set.
// Usually done on the close of a scope.
func (sc *ScopedCommands) RemoveScope(scope int32) {
	delete(sc.commands, scope)
	for i := 0; i < len(sc.scopes); i++ {
		if sc.scopes[i] == scope {
			sc.scopes = append(sc.scopes[:i], sc.scopes[i+1:]...)
			return
		}
	}
}

// GetDebugKeys returns the current debug console commands as a string array
func (sc *ScopedCommands) CommandsInScope(scope int32) []string {

	cmds, ok := sc.commands[scope]
	if !ok {
		return []string{}
	}

	dkeys := make([]string, len(cmds))
	i := 0
	for k := range cmds {
		dkeys[i] = k
		i++
	}
	return dkeys
}

func (sc *ScopedCommands) printHelp(tokenString []string) {
	if len(tokenString) == 0 {
		fmt.Println("help <scopeID> to see commands linked to a window or help 0 to see general commands")
		fmt.Printf("Active Scopes: %v\n", sc.scopes)
		return
	}

	scopeID, err := strToInt32(tokenString[0])
	if err != nil {
		fmt.Println("help <scopeID> expects a valid int scope")
		fmt.Printf("you provided %s which errored with %v \n", tokenString[0], err)
		return
	}
	if _, ok := sc.commands[scopeID]; !ok {
		fmt.Printf("inactive scope %d see correct usage via help\n", scopeID)
	}
	fmt.Println("Current Assumed Scope:", sc.assumedScope)
	fmt.Printf("Commands: %s\n", strings.Join(sc.CommandsInScope(scopeID), ","))
}

// assumeScope of the given windowID if possible
// This allows for easier usage of windows when multiple windows exist.
func (sc *ScopedCommands) assumeScope(tokenString []string) {
	if len(tokenString) == 0 {
		fmt.Println("assume scope requires a scopeID or -")
		fmt.Printf("Active Scopes: %v\n", sc.scopes)
		return
	}

	scopeID, err := strToInt32(tokenString[0])
	if err != nil {
		fmt.Println("assume scope <scopeID> expects a valid int32 scope")
		fmt.Printf("you provided %s which errored with %v \n", tokenString[0], err)
		return
	}
	if _, ok := sc.commands[scopeID]; !ok {
		fmt.Printf("inactive scope %d see correct usage via help\n", scopeID)
	}
	sc.assumedScope = scopeID
	fmt.Println("assumed scope ", scopeID)
}

func strToInt32(potentialInt string) (int32, error) {
	i64, err := strconv.ParseInt(potentialInt, 10, 32)
	return int32(i64), err
}

func parseTokenAsInt(tokenString []string, arrIndex int, defaultVal int) int {
	if len(tokenString) > arrIndex {
		tmp, err := strconv.Atoi(tokenString[arrIndex])
		if err == nil {
			return tmp
		}
	}
	return defaultVal
}
<commit_msg>debugstream/commands.go update for interim help cleaning<commit_after>package debugstream

import (
	"bufio"
	"fmt"
	"io"
	"strconv"
	"strings"
	"sync"

	"github.com/oakmound/oak/v3/oakerr"
)

// ScopedCommands for the debug stream commands.
// Contains a set of scopes that align with oak.Controller.
// Currently can only be attached to a single stream
type ScopedCommands struct {
	sync.Mutex
	attachOnce   sync.Once
	assumedScope int32
	scopes       []int32
	commands     map[int32]map[string]func([]string)
}

// NewScopedCommands creates set of standard help functions.
func NewScopedCommands() *ScopedCommands {
	sc := &ScopedCommands{commands: map[int32]map[string]func([]string){}}
	sc.AddCommand("help", sc.printHelp)
	sc.AddCommand("scope", sc.assumeScope)
	sc.AddCommand("fade", fadeCommands)
	return sc
}

// AttachToStream and start executing the registered commands on input to said stream.
// Currently a given set of scoped commands may be attached once and only once.
func (sc *ScopedCommands) AttachToStream(input io.Reader) {
	if sc == nil {
		return
	}
	sc.attachOnce.Do(
		func() {
			scanner := bufio.NewScanner(input)
			go func() {
				for {
					for scanner.Scan() {

						// TODO: accept interrupts

						tokenString := strings.Fields(scanner.Text())
						if len(tokenString) == 0 {
							continue
						}
						// Attempt to parse the first arg as a scope
						scopeID, err := strToInt32(tokenString[0])

						tokenIDX := 0
						// if there was a scope specified then increment what we care about
						if err == nil {
							_, ok := sc.commands[scopeID]
							if !ok {
								fmt.Printf("unknown scopeID %d see correct usage via help\n", scopeID)
								continue
							}
							if len(tokenString) == 1 {
								fmt.Printf("Only provided scopeID %d without see usage via help\n", scopeID)
								continue
							}

							tokenIDX++
							// see if specified
							if fn, ok := sc.commands[scopeID][tokenString[tokenIDX]]; ok {
								fn(tokenString[tokenIDX+1:])
								continue
							}
						}

						// assumedscope
						if fn, ok := sc.commands[sc.assumedScope][tokenString[0]]; ok {
							fn(tokenString[1:])
							continue
						}

						// fallback to scope 0
						if fn, ok := sc.commands[0][tokenString[0]]; ok {
							fn(tokenString[1:])
							continue
						}

						fmt.Printf("Unknown command '%s' for scopeID %d see correct usage via help or help %d\n", tokenString[tokenIDX], scopeID, scopeID)

					}
				}
			}()
		})
}

// AddCommand adds a console command to call fn when
// '<s> <args>' is input to the console. fn will be called
// with args split on whitespace.
func (sc *ScopedCommands) AddCommand(s string, fn func([]string)) error {
	// We tightly link to controllerIDs here for better or for worse and controllerIDs shall always be > 0
	// This means our unscoped commands are safe when set as 0.
	return sc.AddScopedCommand(0, s, fn)
}

// AddScopedCommand adds a console command for a given window scope to call fn when
// '<s> <args>' is input to the console. fn will be called
// with args split on whitespace.
func (sc *ScopedCommands) AddScopedCommand(scopeID int32, s string, fn func([]string)) error {
	return sc.addCommand(scopeID, s, fn, false)
}

// addCommand for future executions.
func (sc *ScopedCommands) addCommand(scopeID int32, s string, fn func([]string), force bool) error {

	sc.Lock()
	defer sc.Unlock()

	if _, ok := sc.commands[scopeID]; !ok {

		sc.commands[scopeID] = map[string]func([]string){}
		sc.scopes = append(sc.scopes, scopeID)
	}

	if _, ok := sc.commands[scopeID][s]; ok {
		if !force {
			return oakerr.ExistingElement{
				InputName:   s,
				InputType:   "string",
				Overwritten: false,
			}
		}
	}
	sc.commands[scopeID][s] = fn

	return nil
}

// ClearCommand clears an existing debug command for scope with key: <s>
func (sc *ScopedCommands) ClearCommand(scopeID int32, s string) {
	_, ok := sc.commands[scopeID]
	if !ok {
		return
	}
	delete(sc.commands[scopeID], s)
}

// ResetCommands will throw out all existing debug commands from the
// debug console.
func (sc *ScopedCommands) ResetCommands() {
	sc.commands = map[int32]map[string]func([]string){}
}

// ResetCommandsForScope will throw out all existing debug commands from the
// debug console for hte given scope.
func (sc *ScopedCommands) ResetCommandsForScope(scope int32) {
	sc.commands[scope] = map[string]func([]string){}
}

// RemoveScope from the command set.
// Usually done on the close of a scope.
func (sc *ScopedCommands) RemoveScope(scope int32) {
	delete(sc.commands, scope)
	for i := 0; i < len(sc.scopes); i++ {
		if sc.scopes[i] == scope {
			sc.scopes = append(sc.scopes[:i], sc.scopes[i+1:]...)
			return
		}
	}
}

// GetDebugKeys returns the current debug console commands as a string array
func (sc *ScopedCommands) CommandsInScope(scope int32) []string {

	cmds, ok := sc.commands[scope]
	if !ok {
		return []string{}
	}

	dkeys := make([]string, len(cmds))
	i := 0
	for k := range cmds {
		dkeys[i] = k
		i++
	}
	return dkeys
}

func (sc *ScopedCommands) printHelp(tokenString []string) {

	scopeID := sc.assumedScope
	var err error
	if len(tokenString) != 0 {
		scopeID, err = strToInt32(tokenString[0])
		if err != nil {
			fmt.Println("help <scopeID> expects a valid int scope")
			fmt.Printf("you provided %s which errored with %v \n", tokenString[0], err)
			fmt.Println("try using help without arguments for an overview")
			return
		}
	}

	fmt.Println("help <scopeID> to see commands linked to a given window")
	fmt.Printf("Active Scopes: %v\n", sc.scopes)
	if _, ok := sc.commands[scopeID]; !ok {
		fmt.Printf("inactive scope %d see correct usage via help\n", scopeID)
	}

	fmt.Println("Current Assumed Scope:", sc.assumedScope)
	// TODO: if in a verbose mode present usage.

	fmt.Printf("General Commands:\n%s%s\n", indent, strings.Join(sc.CommandsInScope(scopeID), "\n"+indent))
	fmt.Printf("Current Window Commands:\n%s%s\n", indent, strings.Join(sc.CommandsInScope(scopeID), "\n"+indent))
}

const indent = "  "

// assumeScope of the given windowID if possible
// This allows for easier usage of windows when multiple windows exist.
func (sc *ScopedCommands) assumeScope(tokenString []string) {
	if len(tokenString) == 0 {
		fmt.Println("assume scope requires a scopeID or -")
		fmt.Printf("Active Scopes: %v\n", sc.scopes)
		return
	}

	scopeID, err := strToInt32(tokenString[0])
	if err != nil {
		fmt.Println("assume scope <scopeID> expects a valid int32 scope")
		fmt.Printf("you provided %s which errored with %v \n", tokenString[0], err)
		return
	}
	if _, ok := sc.commands[scopeID]; !ok {
		fmt.Printf("inactive scope %d see correct usage via help\n", scopeID)
	}
	sc.assumedScope = scopeID
	fmt.Println("assumed scope ", scopeID)
}

func strToInt32(potentialInt string) (int32, error) {
	i64, err := strconv.ParseInt(potentialInt, 10, 32)
	return int32(i64), err
}

func parseTokenAsInt(tokenString []string, arrIndex int, defaultVal int) int {
	if len(tokenString) > arrIndex {
		tmp, err := strconv.Atoi(tokenString[arrIndex])
		if err == nil {
			return tmp
		}
	}
	return defaultVal
}
<|endoftext|>


<commit_before>package overlay

import (
	"github.com/MJKWoolnough/gopherjs/style"
	"github.com/MJKWoolnough/gopherjs/xjs"
	"honnef.co/go/js/dom"
)

const css = `.mw-overlay {
	position : fixed;
	top : 0;
	left : 0;
	right : 0;
	bottom : 0;
	background : rgba(0, 0, 0, 0.25);
}

.mw-overlay > * {
	position : absolute;
	top : 10px;
	left : 10px;
	right : 10px;
	background : #fff;
}

.mw-overlay .closer {
	color : #fff;
	font-family : serif;
	right : 2px;
	position : absolute;
	text-shadow : #000 -1px -1px 0, #000 1px -1px 0, #000 1px 1px 0, #000 -1px 1px 0, #000 -1px 0 0, #000 1px 0 0;
	top : 0;
	transition : color 0.25s;
	transition-timing-function : ease;
	user-select : none;
	z-index : 1;
	-moz-user-select : none;
	-webkit-user-select : none;
}

.mw-overlay .closer:hover {
	color : #f00;
	cursor : pointer;
}`

func init() {
	style.Add(css)
}

func Overlay(e dom.Element) dom.Element {
	o := xjs.CreateElement("div")
	o.SetAttribute("class", "mw-overlay")
	c := xjs.CreateElement("div")
	xjs.SetInnerText(c, "X")
	c.SetAttribute("class", "closer")
	c.AddEventListener("click", false, func(dom.Event) {
		p := o.ParentNode()
		if p != nil {
			p.RemoveChild(o)
		}
	})
	e.InsertBefore(c, e.FirstChild())
	o.AppendChild(e)
	return o
}
<commit_msg>Added missing window/body CSS<commit_after>package overlay

import (
	"github.com/MJKWoolnough/gopherjs/style"
	"github.com/MJKWoolnough/gopherjs/xjs"
	"honnef.co/go/js/dom"
)

const css = `
window, body {
	width : 100%;
	margin : 0;
	overflow-x : hidden;
}

.mw-overlay {
	position : fixed;
	top : 0;
	left : 0;
	right : 0;
	bottom : 0;
	background : rgba(0, 0, 0, 0.25);
}

.mw-overlay > * {
	position : absolute;
	top : 10px;
	left : 10px;
	right : 10px;
	background : #fff;
}

.mw-overlay .closer {
	color : #fff;
	font-family : serif;
	right : 2px;
	position : absolute;
	text-shadow : #000 -1px -1px 0, #000 1px -1px 0, #000 1px 1px 0, #000 -1px 1px 0, #000 -1px 0 0, #000 1px 0 0;
	top : 0;
	transition : color 0.25s;
	transition-timing-function : ease;
	user-select : none;
	z-index : 1;
	-moz-user-select : none;
	-webkit-user-select : none;
}

.mw-overlay .closer:hover {
	color : #f00;
	cursor : pointer;
}`

func init() {
	style.Add(css)
}

func Overlay(e dom.Element) dom.Element {
	o := xjs.CreateElement("div")
	o.SetAttribute("class", "mw-overlay")
	c := xjs.CreateElement("div")
	xjs.SetInnerText(c, "X")
	c.SetAttribute("class", "closer")
	c.AddEventListener("click", false, func(dom.Event) {
		p := o.ParentNode()
		if p != nil {
			p.RemoveChild(o)
		}
	})
	e.InsertBefore(c, e.FirstChild())
	o.AppendChild(e)
	return o
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kube

import (
	"github.com/prometheus/client_golang/prometheus"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
)

var (
	metricLabels = []string{
		// name of the job
		"job_name",
		// type of the prowjob: presubmit, postsubmit, periodic, batch
		"type",
		// state of the prowjob: triggered, pending, success, failure, aborted, error
		"state",
		// the org of the prowjob's repo
		"org",
		// the prowjob's repo
		"repo",
		// the base_ref of the prowjob's repo
		"base_ref",
	}
	prowJobs = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "prowjobs",
		Help: "Number of prowjobs in the system",
	}, metricLabels)
	prowJobTransitions = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "prowjob_state_transitions",
		Help: "Number of prowjobs transitioning states",
	}, metricLabels)
)

type jobLabel struct {
	jobName string
	jobType string
	state   string
	org     string
	repo    string
	baseRef string
}

func (jl *jobLabel) values() []string {
	return []string{jl.jobName, jl.jobType, jl.state, jl.org, jl.repo, jl.baseRef}
}

func init() {
	prometheus.MustRegister(prowJobs)
	prometheus.MustRegister(prowJobTransitions)
}

func getJobLabelMap(pjs []prowapi.ProwJob) map[jobLabel]float64 {
	jobLabelMap := make(map[jobLabel]float64)

	for _, pj := range pjs {
		jobLabelMap[getJobLabel(pj)]++
	}
	return jobLabelMap
}

func getJobLabel(pj prowapi.ProwJob) jobLabel {
	jl := jobLabel{jobName: pj.Spec.Job, jobType: string(pj.Spec.Type), state: string(pj.Status.State)}

	if pj.Spec.Refs != nil {
		jl.org = pj.Spec.Refs.Org
		jl.repo = pj.Spec.Refs.Repo
		jl.baseRef = pj.Spec.Refs.BaseRef
	} else if len(pj.Spec.ExtraRefs) > 0 {
		jl.org = pj.Spec.ExtraRefs[0].Org
		jl.repo = pj.Spec.ExtraRefs[0].Repo
		jl.baseRef = pj.Spec.ExtraRefs[0].BaseRef
	}

	return jl
}

type jobIdentifier struct {
	jobLabel
	buildId string
}

func getJobIdentifier(pj prowapi.ProwJob) jobIdentifier {
	return jobIdentifier{
		jobLabel: getJobLabel(pj),
		buildId:  pj.Status.BuildID,
	}
}

// previousStates records the prowJobs we were called with previously
var previousStates map[jobIdentifier]prowapi.ProwJobState

// GatherProwJobMetrics gathers prometheus metrics for prowjobs.
// Not threadsafe, ensure this is called serially.
func GatherProwJobMetrics(current []prowapi.ProwJob) {
	// This may be racing with the prometheus server but we need to remove
	// stale metrics like triggered or pending jobs that are now complete.
	prowJobs.Reset()

	// record the current state of ProwJob CRs on the system
	for jl, count := range getJobLabelMap(current) {
		prowJobs.WithLabelValues(jl.values()...).Set(count)
	}

	// record state transitions since the last time we were called
	currentStates := map[jobIdentifier]prowapi.ProwJobState{}
	for _, pj := range current {
		ji := getJobIdentifier(pj)
		state := prowapi.ProwJobState(ji.state)
		currentStates[ji] = state

		if previousState, seen := previousStates[ji]; !seen || previousState != state {
			prowJobTransitions.WithLabelValues(ji.values()...).Inc()
		}
	}

	previousStates = currentStates
}
<commit_msg>Add job_namespace label into prow job metrics<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kube

import (
	"github.com/prometheus/client_golang/prometheus"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
)

var (
	metricLabels = []string{
		// namespace of the job
		"job_namespace",
		// name of the job
		"job_name",
		// type of the prowjob: presubmit, postsubmit, periodic, batch
		"type",
		// state of the prowjob: triggered, pending, success, failure, aborted, error
		"state",
		// the org of the prowjob's repo
		"org",
		// the prowjob's repo
		"repo",
		// the base_ref of the prowjob's repo
		"base_ref",
	}
	prowJobs = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "prowjobs",
		Help: "Number of prowjobs in the system",
	}, metricLabels)
	prowJobTransitions = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "prowjob_state_transitions",
		Help: "Number of prowjobs transitioning states",
	}, metricLabels)
)

type jobLabel struct {
	jobNamespace string
	jobName      string
	jobType      string
	state        string
	org          string
	repo         string
	baseRef      string
}

func (jl *jobLabel) values() []string {
	return []string{jl.jobNamespace, jl.jobName, jl.jobType, jl.state, jl.org, jl.repo, jl.baseRef}
}

func init() {
	prometheus.MustRegister(prowJobs)
	prometheus.MustRegister(prowJobTransitions)
}

func getJobLabelMap(pjs []prowapi.ProwJob) map[jobLabel]float64 {
	jobLabelMap := make(map[jobLabel]float64)

	for _, pj := range pjs {
		jobLabelMap[getJobLabel(pj)]++
	}
	return jobLabelMap
}

func getJobLabel(pj prowapi.ProwJob) jobLabel {
	jl := jobLabel{jobName: pj.Spec.Job, jobType: string(pj.Spec.Type), state: string(pj.Status.State)}

	if pj.Spec.Refs != nil {
		jl.jobNamespace = pj.Namespace
		jl.org = pj.Spec.Refs.Org
		jl.repo = pj.Spec.Refs.Repo
		jl.baseRef = pj.Spec.Refs.BaseRef
	} else if len(pj.Spec.ExtraRefs) > 0 {
		jl.org = pj.Spec.ExtraRefs[0].Org
		jl.repo = pj.Spec.ExtraRefs[0].Repo
		jl.baseRef = pj.Spec.ExtraRefs[0].BaseRef
	}

	return jl
}

type jobIdentifier struct {
	jobLabel
	buildId string
}

func getJobIdentifier(pj prowapi.ProwJob) jobIdentifier {
	return jobIdentifier{
		jobLabel: getJobLabel(pj),
		buildId:  pj.Status.BuildID,
	}
}

// previousStates records the prowJobs we were called with previously
var previousStates map[jobIdentifier]prowapi.ProwJobState

// GatherProwJobMetrics gathers prometheus metrics for prowjobs.
// Not threadsafe, ensure this is called serially.
func GatherProwJobMetrics(current []prowapi.ProwJob) {
	// This may be racing with the prometheus server but we need to remove
	// stale metrics like triggered or pending jobs that are now complete.
	prowJobs.Reset()

	// record the current state of ProwJob CRs on the system
	for jl, count := range getJobLabelMap(current) {
		prowJobs.WithLabelValues(jl.values()...).Set(count)
	}

	// record state transitions since the last time we were called
	currentStates := map[jobIdentifier]prowapi.ProwJobState{}
	for _, pj := range current {
		ji := getJobIdentifier(pj)
		state := prowapi.ProwJobState(ji.state)
		currentStates[ji] = state

		if previousState, seen := previousStates[ji]; !seen || previousState != state {
			prowJobTransitions.WithLabelValues(ji.values()...).Inc()
		}
	}

	previousStates = currentStates
}
<|endoftext|>


<commit_before>package strategy

import (
	"fmt"
	"testing"

	"github.com/docker/swarm/cluster"
	"github.com/samalba/dockerclient"
	"github.com/stretchr/testify/assert"
)

func createNode(ID string, memory int64, cpus int64) *cluster.Node {
	node := cluster.NewNode(ID, 0.05)
	node.ID = ID
	node.Memory = memory * 1024 * 1024 * 1024
	node.Cpus = cpus
	return node
}

func createConfig(memory int64, cpus int64) *dockerclient.ContainerConfig {
	return &dockerclient.ContainerConfig{Memory: memory * 1024 * 1024 * 1024, CpuShares: cpus}
}

func createContainer(ID string, config *dockerclient.ContainerConfig) *cluster.Container {
	return &cluster.Container{Container: dockerclient.Container{Id: ID}, Info: dockerclient.ContainerInfo{Config: config}}
}

func TestPlaceContainerMemory(t *testing.T) {
	s := &BinPackingPlacementStrategy{}

	nodes := []*cluster.Node{}
	for i := 0; i < 2; i++ {
		nodes = append(nodes, createNode(fmt.Sprintf("node-%d", i), 2, 1))
	}

	// add 1 container 1G
	config := createConfig(1, 0)
	node1, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node1.AddContainer(createContainer("c1", config)))

	assert.Equal(t, node1.ReservedMemory(), 1024*1024*1024)

	// add another container 1G
	config = createConfig(1, 1)
	node2, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node2.AddContainer(createContainer("c2", config)))

	assert.Equal(t, node2.ReservedMemory(), int64(2*1024*1024*1024))

	// check that both containers ended on the same node
	assert.Equal(t, node1.ID, node2.ID, "")
	assert.Equal(t, len(node1.Containers()), len(node2.Containers()), "")
}

func TestPlaceContainerCPU(t *testing.T) {
	s := &BinPackingPlacementStrategy{}

	nodes := []*cluster.Node{}
	for i := 0; i < 2; i++ {
		nodes = append(nodes, createNode(fmt.Sprintf("node-%d", i), 1, 2))
	}

	// add 1 container 1CPU
	config := createConfig(0, 1)
	node1, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node1.AddContainer(createContainer("c1", config)))

	assert.Equal(t, node1.ReservedCpus(), 1)

	// add another container 1CPU
	config = createConfig(0, 1)
	node2, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node2.AddContainer(createContainer("c2", config)))
	assert.Equal(t, node2.ReservedCpus(), 2)

	// check that both containers ended on the same node
	assert.Equal(t, node1.ID, node2.ID, "")
	assert.Equal(t, len(node1.Containers()), len(node2.Containers()), "")
}

func TestPlaceContainerHuge(t *testing.T) {
	s := &BinPackingPlacementStrategy{}

	nodes := []*cluster.Node{}
	for i := 0; i < 100; i++ {
		nodes = append(nodes, createNode(fmt.Sprintf("node-%d", i), 1, 1))
	}

	// add 100 container 1CPU
	for i := 0; i < 100; i++ {
		node, err := s.PlaceContainer(createConfig(0, 1), nodes)
		assert.NoError(t, err)
		assert.NoError(t, node.AddContainer(createContainer(fmt.Sprintf("c%d", i), createConfig(0, 100))))
	}

	// try to add another container 1CPU
	_, err := s.PlaceContainer(createConfig(0, 1), nodes)
	assert.Error(t, err)

	// add 100 container 1G
	for i := 100; i < 200; i++ {
		node, err := s.PlaceContainer(createConfig(1, 0), nodes)
		assert.NoError(t, err)
		assert.NoError(t, node.AddContainer(createContainer(fmt.Sprintf("c%d", i), createConfig(1, 0))))
	}

	// try to add another container 1G
	_, err = s.PlaceContainer(createConfig(1, 0), nodes)
	assert.Error(t, err)
}

func TestPlaceContainerOvercommit(t *testing.T) {
	s, err := New("binpacking")
	assert.NoError(t, err)

	nodes := []*cluster.Node{createNode("node-1", 0, 1)}
	nodes[0].Memory = 100

	config := createConfig(0, 0)

	// Below limit should still work.
	config.Memory = 90
	node, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.Equal(t, node, nodes[0])

	// At memory limit should still work.
	config.Memory = 100
	node, err = s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.Equal(t, node, nodes[0])

	// Up to 105% it should still work.
	config.Memory = 105
	node, err = s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.Equal(t, node, nodes[0])

	// Above it should return an error.
	config.Memory = 106
	node, err = s.PlaceContainer(config, nodes)
	assert.Error(t, err)
}

// The demo
func TestPlaceContainerDemo(t *testing.T) {
	s := &BinPackingPlacementStrategy{}

	nodes := []*cluster.Node{}
	for i := 0; i < 3; i++ {
		nodes = append(nodes, createNode(fmt.Sprintf("node-%d", i), 2, 4))
	}

	// try to place a 10G container
	config := createConfig(10, 0)
	_, err := s.PlaceContainer(config, nodes)

	// check that it refuses because the cluster is full
	assert.Error(t, err)

	// add one container 1G
	config = createConfig(1, 0)
	node1, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node1.AddContainer(createContainer("c1", config)))

	// add another container 1G
	config = createConfig(1, 0)
	node1bis, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node1bis.AddContainer(createContainer("c2", config)))

	// check that both containers ended on the same node
	assert.Equal(t, node1.ID, node1bis.ID, "")
	assert.Equal(t, len(node1.Containers()), len(node1bis.Containers()), "")

	// add another container 2G
	config = createConfig(2, 0)
	node2, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node2.AddContainer(createContainer("c3", config)))

	// check that it ends up on another node
	assert.NotEqual(t, node1.ID, node2.ID, "")

	// add another container 1G
	config = createConfig(1, 0)
	node3, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node3.AddContainer(createContainer("c4", config)))

	// check that it ends up on another node
	assert.NotEqual(t, node1.ID, node3.ID, "")
	assert.NotEqual(t, node2.ID, node3.ID, "")

	// add another container 1G
	config = createConfig(1, 0)
	node3bis, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node3bis.AddContainer(createContainer("c5", config)))

	// check that it ends up on the same node
	assert.Equal(t, node3.ID, node3bis.ID, "")

	// try to add another container
	config = createConfig(1, 0)
	_, err = s.PlaceContainer(config, nodes)

	// check that it refuses because the cluster is full
	assert.Error(t, err)

	// remove container in the middle
	node2.CleanupContainers()

	// add another container
	config = createConfig(1, 0)
	node2bis, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node2bis.AddContainer(createContainer("c6", config)))

	// check it ends up on `node3`
	assert.Equal(t, node2.ID, node2bis.ID, "")
	assert.Equal(t, len(node2.Containers()), len(node2bis.Containers()), "")
}
<commit_msg>Add test that fails to execute the proper strategy<commit_after>package strategy

import (
	"fmt"
	"testing"

	"github.com/docker/swarm/cluster"
	"github.com/samalba/dockerclient"
	"github.com/stretchr/testify/assert"
)

func createNode(ID string, memory int64, cpus int64) *cluster.Node {
	node := cluster.NewNode(ID, 0.05)
	node.ID = ID
	node.Memory = memory * 1024 * 1024 * 1024
	node.Cpus = cpus
	return node
}

func createConfig(memory int64, cpus int64) *dockerclient.ContainerConfig {
	return &dockerclient.ContainerConfig{Memory: memory * 1024 * 1024 * 1024, CpuShares: cpus}
}

func createContainer(ID string, config *dockerclient.ContainerConfig) *cluster.Container {
	return &cluster.Container{Container: dockerclient.Container{Id: ID}, Info: dockerclient.ContainerInfo{Config: config}}
}

func TestPlaceContainerMemory(t *testing.T) {
	s := &BinPackingPlacementStrategy{}

	nodes := []*cluster.Node{}
	for i := 0; i < 2; i++ {
		nodes = append(nodes, createNode(fmt.Sprintf("node-%d", i), 2, 1))
	}

	// add 1 container 1G
	config := createConfig(1, 0)
	node1, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node1.AddContainer(createContainer("c1", config)))

	assert.Equal(t, node1.ReservedMemory(), 1024*1024*1024)

	// add another container 1G
	config = createConfig(1, 1)
	node2, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node2.AddContainer(createContainer("c2", config)))

	assert.Equal(t, node2.ReservedMemory(), int64(2*1024*1024*1024))

	// check that both containers ended on the same node
	assert.Equal(t, node1.ID, node2.ID, "")
	assert.Equal(t, len(node1.Containers()), len(node2.Containers()), "")
}

func TestPlaceContainerCPU(t *testing.T) {
	s := &BinPackingPlacementStrategy{}

	nodes := []*cluster.Node{}
	for i := 0; i < 2; i++ {
		nodes = append(nodes, createNode(fmt.Sprintf("node-%d", i), 1, 2))
	}

	// add 1 container 1CPU
	config := createConfig(0, 1)
	node1, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node1.AddContainer(createContainer("c1", config)))

	assert.Equal(t, node1.ReservedCpus(), 1)

	// add another container 1CPU
	config = createConfig(0, 1)
	node2, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node2.AddContainer(createContainer("c2", config)))
	assert.Equal(t, node2.ReservedCpus(), 2)

	// check that both containers ended on the same node
	assert.Equal(t, node1.ID, node2.ID, "")
	assert.Equal(t, len(node1.Containers()), len(node2.Containers()), "")
}

func TestPlaceContainerHuge(t *testing.T) {
	s := &BinPackingPlacementStrategy{}

	nodes := []*cluster.Node{}
	for i := 0; i < 100; i++ {
		nodes = append(nodes, createNode(fmt.Sprintf("node-%d", i), 1, 1))
	}

	// add 100 container 1CPU
	for i := 0; i < 100; i++ {
		node, err := s.PlaceContainer(createConfig(0, 1), nodes)
		assert.NoError(t, err)
		assert.NoError(t, node.AddContainer(createContainer(fmt.Sprintf("c%d", i), createConfig(0, 100))))
	}

	// try to add another container 1CPU
	_, err := s.PlaceContainer(createConfig(0, 1), nodes)
	assert.Error(t, err)

	// add 100 container 1G
	for i := 100; i < 200; i++ {
		node, err := s.PlaceContainer(createConfig(1, 0), nodes)
		assert.NoError(t, err)
		assert.NoError(t, node.AddContainer(createContainer(fmt.Sprintf("c%d", i), createConfig(1, 0))))
	}

	// try to add another container 1G
	_, err = s.PlaceContainer(createConfig(1, 0), nodes)
	assert.Error(t, err)
}

func TestPlaceContainerOvercommit(t *testing.T) {
	s, err := New("binpacking")
	assert.NoError(t, err)

	nodes := []*cluster.Node{createNode("node-1", 0, 1)}
	nodes[0].Memory = 100

	config := createConfig(0, 0)

	// Below limit should still work.
	config.Memory = 90
	node, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.Equal(t, node, nodes[0])

	// At memory limit should still work.
	config.Memory = 100
	node, err = s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.Equal(t, node, nodes[0])

	// Up to 105% it should still work.
	config.Memory = 105
	node, err = s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.Equal(t, node, nodes[0])

	// Above it should return an error.
	config.Memory = 106
	node, err = s.PlaceContainer(config, nodes)
	assert.Error(t, err)
}

// The demo
func TestPlaceContainerDemo(t *testing.T) {
	s := &BinPackingPlacementStrategy{}

	nodes := []*cluster.Node{}
	for i := 0; i < 3; i++ {
		nodes = append(nodes, createNode(fmt.Sprintf("node-%d", i), 2, 4))
	}

	// try to place a 10G container
	config := createConfig(10, 0)
	_, err := s.PlaceContainer(config, nodes)

	// check that it refuses because the cluster is full
	assert.Error(t, err)

	// add one container 1G
	config = createConfig(1, 0)
	node1, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node1.AddContainer(createContainer("c1", config)))

	// add another container 1G
	config = createConfig(1, 0)
	node1bis, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node1bis.AddContainer(createContainer("c2", config)))

	// check that both containers ended on the same node
	assert.Equal(t, node1.ID, node1bis.ID, "")
	assert.Equal(t, len(node1.Containers()), len(node1bis.Containers()), "")

	// add another container 2G
	config = createConfig(2, 0)
	node2, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node2.AddContainer(createContainer("c3", config)))

	// check that it ends up on another node
	assert.NotEqual(t, node1.ID, node2.ID, "")

	// add another container 1G
	config = createConfig(1, 0)
	node3, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node3.AddContainer(createContainer("c4", config)))

	// check that it ends up on another node
	assert.NotEqual(t, node1.ID, node3.ID, "")
	assert.NotEqual(t, node2.ID, node3.ID, "")

	// add another container 1G
	config = createConfig(1, 0)
	node3bis, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node3bis.AddContainer(createContainer("c5", config)))

	// check that it ends up on the same node
	assert.Equal(t, node3.ID, node3bis.ID, "")

	// try to add another container
	config = createConfig(1, 0)
	_, err = s.PlaceContainer(config, nodes)

	// check that it refuses because the cluster is full
	assert.Error(t, err)

	// remove container in the middle
	node2.CleanupContainers()

	// add another container
	config = createConfig(1, 0)
	node2bis, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node2bis.AddContainer(createContainer("c6", config)))

	// check it ends up on `node3`
	assert.Equal(t, node2.ID, node2bis.ID, "")
	assert.Equal(t, len(node2.Containers()), len(node2bis.Containers()), "")
}

func TestComplexPlacement(t *testing.T) {
	s := &BinPackingPlacementStrategy{}

	nodes := []*cluster.Node{}
	for i := 0; i < 2; i++ {
		nodes = append(nodes, createNode(fmt.Sprintf("node-%d", i), 4, 4))
	}

	// add one container 2G
	config := createConfig(2, 0)
	node1, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node1.AddContainer(createContainer("c1", config)))

	// add one container 3G
	config = createConfig(3, 0)
	node2, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node2.AddContainer(createContainer("c2", config)))

	// check that they end up on separate nodes
	assert.NotEqual(t, node1.ID, node2.ID, "")

	// add one container 1G
	config = createConfig(1, 0)
	node3, err := s.PlaceContainer(config, nodes)
	assert.NoError(t, err)
	assert.NoError(t, node3.AddContainer(createContainer("c3", config)))

	// check that it ends up on the same node as the 3G
	assert.Equal(t, node2.ID, node3.ID, "")
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"os/exec"

	"github.com/herman-rogers/gogeta/config"
	"github.com/herman-rogers/gogeta/logger"
	git "github.com/libgit2/git2go"
	"github.com/satori/go.uuid"
)

const UP_TO_DATE = "UP_TO_DATE"
const NORMAL_MERGE = "NORMAL_MERGE"
const FAST_FORWARD = "FAST_FORWARD"

func GitProcessSQSMessages(gitReq scmServiceRequest) error {
	if gitReq.Usr == "" || gitReq.Project == "" || gitReq.Repo == "" {
		return errors.New("Missing Git Service Properties")
	}
	go GitShallowClone(gitReq)
	return nil
}

func GitShallowClone(data scmServiceRequest) {
	repoPath, relativePath := GetRepoPath(data.Project)
	cmd := exec.Command("git", "clone", "--depth", "2", data.Repo, repoPath)

	logfile := logger.GetLogFile()
	defer logfile.Close()

	cmd.Stdout = logfile
	cmd.Stderr = logfile

	commandErr := cmd.Start()
	logger.LogError(commandErr, "Git Clone")

	cloneErr := cmd.Wait()
	logger.LogError(cloneErr, "Git Clone")
	if cloneErr == nil {
		gitData := &GogetaRepo{
			data.Id,
			data.Usr,
			data.Repo,
			relativePath,
			data.SCMType,
			data.Engine,
			data.Platform,
		}
		CreateGitCredentials(repoPath)
		go SaveRepo(*gitData)
		go TriggerMrRobotBuild(*gitData)
	}
}

func GetRepoPath(project string) (string, string) {
	var uuid uuid.UUID = uuid.NewV4()
	relativePath := project + "_" + uuid.String()
	repoPath := config.File.RepoPath + relativePath
	return repoPath, relativePath
}

func CreateGitCredentials(repoLocation string) {
	gitConfigUser := exec.Command("sudo", "cd", repoLocation, "&&", "git", "config", "user.name", "Gamebuildr")
	gitConfigEmail := exec.Command("sudo", "cd", repoLocation, "&&", "git", "config", "user.email", "contact@gamebuildr.io")

	gitConfigUserErr := gitConfigUser.Start()
	if gitConfigUserErr != nil {
		logger.LogError(gitConfigUserErr, "Git Config Set User.Name: ")
	}
	gitConfigEmailErr := gitConfigEmail.Start()
	if gitConfigEmailErr != nil {
		logger.LogError(gitConfigEmailErr, "Git Config Email")
	}
}

func UpdateGitRepositories() {
	repos := FindAllRepos()
	for i := 0; i < len(repos); i++ {
		folder := config.File.RepoPath + repos[i].Folder
		repo, err := git.OpenRepository(folder)
		if err != nil {
			logger.Error(err.Error())
			continue
		}
		if repo.IsBare() {
			continue
		}
		go GitPull(repos[i], repo)
	}
}

func GitPull(data GogetaRepo, repo *git.Repository) {
	FetchLatestsUpdates(data)
	remoteBranch := GetRemoteBranch(repo)
	remoteBranchID := remoteBranch.Target()

	analysis, err := AnalyzeLatestUpdates(repo, remoteBranch)
	if err != nil {
		logger.Error(err.Error())
	}
	mergeType := CheckMergeAnalysis(analysis)
	msg := "Git Merge " + mergeType

	switch mergeType {
	case UP_TO_DATE:
		break
	case NORMAL_MERGE:
		err := MergeNormal(repo, remoteBranch, remoteBranchID)
		BuildAfterMerge(err, msg, data)
		break
	case FAST_FORWARD:
		err := MergeFastForward(repo, remoteBranchID)
		BuildAfterMerge(err, msg, data)
		break
	}
}

func GetRemoteBranch(repo *git.Repository) *git.Reference {
	remoteBranch, err := repo.References.Lookup("refs/remotes/origin/master")
	logger.LogError(err, "Git Remote Branch")
	return remoteBranch
}

func FetchLatestsUpdates(data GogetaRepo) {
	cmd := exec.Command("git", "fetch", "--depth", "1", "origin", "master")
	cmd.Dir = config.File.RepoPath + data.Folder

	commandErr := cmd.Start()
	logger.LogError(commandErr, "Git Fetch")

	fetchErr := cmd.Wait()
	logger.LogError(fetchErr, "Git Fetch")
}

func AnalyzeLatestUpdates(repo *git.Repository, remoteBranch *git.Reference) (git.MergeAnalysis, error) {
	annotatedCommit, err := repo.AnnotatedCommitFromRef(remoteBranch)
	if err != nil {
		var nullAnalysis git.MergeAnalysis
		return nullAnalysis, err
	}
	mergeHeads := make([]*git.AnnotatedCommit, 1)
	mergeHeads[0] = annotatedCommit
	analysis, _, err := repo.MergeAnalysis(mergeHeads)
	logger.LogError(err, "Git Merge Analysis")
	return analysis, nil
}

func CheckMergeAnalysis(analysis git.MergeAnalysis) string {
	if analysis&git.MergeAnalysisUpToDate != 0 {
		return UP_TO_DATE
	}
	if analysis&git.MergeAnalysisNormal != 0 {
		return NORMAL_MERGE
	}
	return FAST_FORWARD
}

func MergeNormal(repo *git.Repository, remoteBranch *git.Reference, remoteBranchID *git.Oid) error {
	head, err := repo.Head()
	if err != nil {
		return err
	}
	annotatedCommit, err := repo.AnnotatedCommitFromRef(remoteBranch)
	if err != nil {
		return err
	}
	if err := repo.Merge([]*git.AnnotatedCommit{annotatedCommit}, nil, nil); err != nil {
		return err
	}
	index, err := repo.Index()
	if err != nil {
		return err
	}
	if index.HasConflicts() {
		return errors.New("Conflicts encountered, Please Resolve Them")
	}
	sig, err := repo.DefaultSignature()
	if err != nil {
		return err
	}
	treeId, err := index.WriteTree()
	if err != nil {
		return err
	}
	tree, err := repo.LookupTree(treeId)
	if err != nil {
		return err
	}
	localCommit, err := repo.LookupCommit(head.Target())
	if err != nil {
		return err
	}
	remoteCommit, err := repo.LookupCommit(remoteBranchID)
	if err != nil {
		return err
	}
	repo.CreateCommit("HEAD", sig, sig, "", tree, localCommit, remoteCommit)
	repo.StateCleanup()
	return nil
}

func MergeFastForward(repo *git.Repository, remoteBranchID *git.Oid) error {
	head, err := repo.Head()
	if err != nil {
		return err
	}
	remoteTree, err := repo.LookupTree(remoteBranchID)
	if err != nil {
		return err
	}
	if err := repo.CheckoutTree(remoteTree, nil); err != nil {
		return err
	}
	branchRef, err := repo.References.Lookup("refs/heads/master")
	if err != nil {
		return err
	}
	branchRef.SetTarget(remoteBranchID, "")
	if _, err := head.SetTarget(remoteBranchID, ""); err != nil {
		return err
	}
	return nil
}
<commit_msg>Add ability to add git credentials on clone<commit_after>package main

import (
	"errors"
	"os/exec"

	"github.com/herman-rogers/gogeta/config"
	"github.com/herman-rogers/gogeta/logger"
	git "github.com/libgit2/git2go"
	"github.com/satori/go.uuid"
)

const UP_TO_DATE = "UP_TO_DATE"
const NORMAL_MERGE = "NORMAL_MERGE"
const FAST_FORWARD = "FAST_FORWARD"

func GitProcessSQSMessages(gitReq scmServiceRequest) error {
	if gitReq.Usr == "" || gitReq.Project == "" || gitReq.Repo == "" {
		return errors.New("Missing Git Service Properties")
	}
	go GitShallowClone(gitReq)
	return nil
}

func GitShallowClone(data scmServiceRequest) {
	repoPath, relativePath := GetRepoPath(data.Project)
	cmd := exec.Command("git", "clone", "--depth", "2", data.Repo, repoPath)

	logfile := logger.GetLogFile()
	defer logfile.Close()

	cmd.Stdout = logfile
	cmd.Stderr = logfile

	commandErr := cmd.Start()
	logger.LogError(commandErr, "Git Clone")

	cloneErr := cmd.Wait()
	logger.LogError(cloneErr, "Git Clone")
	if cloneErr == nil {
		gitData := &GogetaRepo{
			data.Id,
			data.Usr,
			data.Repo,
			relativePath,
			data.SCMType,
			data.Engine,
			data.Platform,
		}
		CreateGitCredentials(repoPath)
		go SaveRepo(*gitData)
		go TriggerMrRobotBuild(*gitData)
	}
}

func GetRepoPath(project string) (string, string) {
	var uuid uuid.UUID = uuid.NewV4()
	relativePath := project + "_" + uuid.String()
	repoPath := config.File.RepoPath + relativePath
	return repoPath, relativePath
}

func CreateGitCredentials(repo string) {
	openRepo, err := git.OpenRepository(repo)
	if err != nil {
		logger.LogError(err, "Git Create Credentials")
	}
	config, configErr := openRepo.Config()
	if configErr != nil {
		logger.LogError(configErr, "Git Config Set")
	}
	configNameErr := config.SetString("user.name", "gamebuildr")
	if configNameErr != nil {
		logger.LogError(configNameErr, "Git Config Name")
	}
	configEmailErr := config.SetString("user.email", "contact@gamebuildr.io")
	if configEmailErr != nil {
		logger.LogError(configEmailErr, "Git Config Email")
	}
}

func UpdateGitRepositories() {
	repos := FindAllRepos()
	for i := 0; i < len(repos); i++ {
		folder := config.File.RepoPath + repos[i].Folder
		repo, err := git.OpenRepository(folder)
		if err != nil {
			logger.Error(err.Error())
			continue
		}
		if repo.IsBare() {
			continue
		}
		go GitPull(repos[i], repo)
	}
}

func GitPull(data GogetaRepo, repo *git.Repository) {
	FetchLatestsUpdates(data)
	remoteBranch := GetRemoteBranch(repo)
	remoteBranchID := remoteBranch.Target()

	analysis, err := AnalyzeLatestUpdates(repo, remoteBranch)
	if err != nil {
		logger.Error(err.Error())
	}
	mergeType := CheckMergeAnalysis(analysis)
	msg := "Git Merge " + mergeType

	switch mergeType {
	case UP_TO_DATE:
		break
	case NORMAL_MERGE:
		err := MergeNormal(repo, remoteBranch, remoteBranchID)
		BuildAfterMerge(err, msg, data)
		break
	case FAST_FORWARD:
		err := MergeFastForward(repo, remoteBranchID)
		BuildAfterMerge(err, msg, data)
		break
	}
}

func GetRemoteBranch(repo *git.Repository) *git.Reference {
	remoteBranch, err := repo.References.Lookup("refs/remotes/origin/master")
	logger.LogError(err, "Git Remote Branch")
	return remoteBranch
}

func FetchLatestsUpdates(data GogetaRepo) {
	cmd := exec.Command("git", "fetch", "--depth", "1", "origin", "master")
	cmd.Dir = config.File.RepoPath + data.Folder

	commandErr := cmd.Start()
	logger.LogError(commandErr, "Git Fetch")

	fetchErr := cmd.Wait()
	logger.LogError(fetchErr, "Git Fetch")
}

func AnalyzeLatestUpdates(repo *git.Repository, remoteBranch *git.Reference) (git.MergeAnalysis, error) {
	annotatedCommit, err := repo.AnnotatedCommitFromRef(remoteBranch)
	if err != nil {
		var nullAnalysis git.MergeAnalysis
		return nullAnalysis, err
	}
	mergeHeads := make([]*git.AnnotatedCommit, 1)
	mergeHeads[0] = annotatedCommit
	analysis, _, err := repo.MergeAnalysis(mergeHeads)
	logger.LogError(err, "Git Merge Analysis")
	return analysis, nil
}

func CheckMergeAnalysis(analysis git.MergeAnalysis) string {
	if analysis&git.MergeAnalysisUpToDate != 0 {
		return UP_TO_DATE
	}
	if analysis&git.MergeAnalysisNormal != 0 {
		return NORMAL_MERGE
	}
	return FAST_FORWARD
}

func MergeNormal(repo *git.Repository, remoteBranch *git.Reference, remoteBranchID *git.Oid) error {
	head, err := repo.Head()
	if err != nil {
		return err
	}
	annotatedCommit, err := repo.AnnotatedCommitFromRef(remoteBranch)
	if err != nil {
		return err
	}
	if err := repo.Merge([]*git.AnnotatedCommit{annotatedCommit}, nil, nil); err != nil {
		return err
	}
	index, err := repo.Index()
	if err != nil {
		return err
	}
	if index.HasConflicts() {
		return errors.New("Conflicts encountered, Please Resolve Them")
	}
	sig, err := repo.DefaultSignature()
	if err != nil {
		return err
	}
	treeId, err := index.WriteTree()
	if err != nil {
		return err
	}
	tree, err := repo.LookupTree(treeId)
	if err != nil {
		return err
	}
	localCommit, err := repo.LookupCommit(head.Target())
	if err != nil {
		return err
	}
	remoteCommit, err := repo.LookupCommit(remoteBranchID)
	if err != nil {
		return err
	}
	repo.CreateCommit("HEAD", sig, sig, "", tree, localCommit, remoteCommit)
	repo.StateCleanup()
	return nil
}

func MergeFastForward(repo *git.Repository, remoteBranchID *git.Oid) error {
	head, err := repo.Head()
	if err != nil {
		return err
	}
	remoteTree, err := repo.LookupTree(remoteBranchID)
	if err != nil {
		return err
	}
	if err := repo.CheckoutTree(remoteTree, nil); err != nil {
		return err
	}
	branchRef, err := repo.References.Lookup("refs/heads/master")
	if err != nil {
		return err
	}
	branchRef.SetTarget(remoteBranchID, "")
	if _, err := head.SetTarget(remoteBranchID, ""); err != nil {
		return err
	}
	return nil
}
<|endoftext|>