<commit_before>package m_etcd

import (
	"encoding/json"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/coreos/go-etcd/etcd"
	"github.com/lytics/metafora"
)

type ownerValue struct {
	Node string `json:"node"`
}

type watcher struct {
	cordCtx      metafora.CoordinatorContext
	path         string
	responseChan chan *etcd.Response // coordinator watches for changes
	errorChan    chan error          // coordinator watches for errors
	stopChan     chan bool           // closed to signal etcd's Watch to stop
	client       *etcd.Client
	running      bool       // only set in watch(); only read by watching()
	m            sync.Mutex // running requires synchronization
}

// watch emits etcd responses over responseChan until stopChan is closed.
func (w *watcher) watch() {
	// Setup watch state
	w.m.Lock()
	w.running = true
	w.stopChan = make(chan bool)
	w.m.Unlock()

	// Teardown watch state when watch() returns
	defer func() {
		w.m.Lock()
		w.running = false
		w.m.Unlock()
	}()

	var index uint64

	// Get existing tasks
	const sorted = true
	const recursive = true
	resp, err := w.client.Get(w.path, sorted, recursive)
	if err != nil {
		w.cordCtx.Log(metafora.LogLevelError, "Error getting the existing tasks from the path:%s error:%v", w.path, err)
		goto done
	}

	for _, node := range resp.Node.Nodes {
		// Act like these are newly created nodes
		select {
		case <-w.stopChan:
			goto done
		case w.responseChan <- &etcd.Response{Action: "create", Node: node, EtcdIndex: resp.EtcdIndex}:
		}
	}

	index = resp.EtcdIndex

	// Start blocking watch
	for {
		// Make a new inner response channel on each loop since Watch() closes the
		// one passed in.
		innerRespChan := make(chan *etcd.Response)
		go func() {
			for {
				select {
				case r, ok := <-innerRespChan:
					if !ok {
						// Don't close w.responseChan when the Watch exits
						return
					}
					// Update the last seen index
					index = r.EtcdIndex
					w.responseChan <- r
				case <-w.stopChan:
				}
			}
		}()
		// Start the blocking watch.
		_, err = w.client.Watch(w.path, index, recursive, innerRespChan, w.stopChan)
		if err != nil {
			if err == etcd.ErrWatchStoppedByUser {
				// This isn't actually an error, return nil
				err = nil
			} else if jsonErr, ok := err.(*json.SyntaxError); ok && jsonErr.Offset == 0 {
				// This is a bug in Go's HTTP transport + go-etcd which causes the
				// connection to timeout perdiocally and need to be restarted *after*
				// closing idle connections.
				w.cordCtx.Log(metafora.LogLevelDebug, "Watch timed out; restarting")
				transport.CloseIdleConnections()
				err = nil
				continue
			}
		}
		goto done
	}
done:
	w.errorChan <- err
}

// watching is a safe way for concurrent goroutines to check if the watcher is
// running.
func (w *watcher) watching() bool {
	w.m.Lock()
	r := w.running
	w.m.Unlock()
	return r
}

// stops the watching goroutine.
func (w *watcher) stop() {
	if w.watching() {
		select {
		case <-w.stopChan:
			// already stopped, let's avoid panic()s
		default:
			close(w.stopChan)
		}
	}
}

type EtcdCoordinator struct {
	Client    *etcd.Client
	cordCtx   metafora.CoordinatorContext
	Namespace string
	TaskPath  string

	taskWatcher *watcher
	ClaimTTL    uint64 // seconds

	NodeID         string
	CommandPath    string
	commandWatcher *watcher

	taskManager *taskManager
}

func NewEtcdCoordinator(nodeId, namespace string, client *etcd.Client) metafora.Coordinator {
	namespace = strings.Trim(namespace, "/ ")

	if nodeId == "" {
		hn, _ := os.Hostname()
		//Adding the UUID incase we run two nodes on the same box.
		// TODO lets move this to the Readme as part of the example of calling NewEtcdCoordinator.
		// Then just remove the Autocreated nodeId.
		nodeId = hn + uuid.NewRandom().String()
	}

	nodeId = strings.Trim(nodeId, "/ ")

	client.SetTransport(transport)
	client.SetConsistency(etcd.STRONG_CONSISTENCY)
	return &EtcdCoordinator{
		Client:    client,
		Namespace: namespace,

		TaskPath: fmt.Sprintf("/%s/%s", namespace, TasksPath),
		ClaimTTL: ClaimTTL, //default to the package constant, but allow it to be overwritten

		NodeID:      nodeId,
		CommandPath: fmt.Sprintf("/%s/%s/%s/%s", namespace, NodesPath, nodeId, CommandsPath),
	}
}

// Init is called once by the consumer to provide a Logger to Coordinator
// implementations.
func (ec *EtcdCoordinator) Init(cordCtx metafora.CoordinatorContext) {
	cordCtx.Log(metafora.LogLevelDebug, "Initializing coordinator with namespace: %s and etcd cluster: %s",
		ec.Namespace, strings.Join(ec.Client.GetCluster(), ", "))

	ec.cordCtx = cordCtx

	ec.upsertDir("/"+ec.Namespace, ForeverTTL)
	ec.upsertDir(ec.TaskPath, ForeverTTL)
	ec.upsertDir(ec.CommandPath, ForeverTTL)

	ec.taskWatcher = &watcher{
		cordCtx:      cordCtx,
		path:         ec.TaskPath,
		responseChan: make(chan *etcd.Response),
		errorChan:    make(chan error),
		client:       ec.Client,
	}

	ec.commandWatcher = &watcher{
		cordCtx:      cordCtx,
		path:         ec.CommandPath,
		responseChan: make(chan *etcd.Response),
		errorChan:    make(chan error),
		client:       ec.Client,
	}

	ec.taskManager = newManager(cordCtx, ec.Client, ec.TaskPath, ec.NodeID, ec.ClaimTTL)
}

func (ec *EtcdCoordinator) upsertDir(path string, ttl uint64) {

	//hidden etcd key that isn't visible to ls commands on the directory,
	//  you have to know about it to find it :).  I'm using it to add some
	//  info about when the cluster's schema was setup.
	pathMarker := path + "/" + MetadataKey
	const sorted = false
	const recursive = false

	_, err := ec.Client.Get(path, sorted, recursive)
	if err == nil {
		return
	}

	etcdErr, ok := err.(*etcd.EtcdError)
	if ok && etcdErr.ErrorCode == EcodeKeyNotFound {
		_, err := ec.Client.CreateDir(path, ttl)
		if err != nil {
			ec.cordCtx.Log(metafora.LogLevelDebug, "Error trying to create directory. path:[%s] error:[ %v ]", path, err)
		}
		host, _ := os.Hostname()

		metadata := struct {
			Host        string `json:"host"`
			CreatedTime string `json:"created"`
			ownerValue
		}{
			Host:        host,
			CreatedTime: time.Now().String(),
			ownerValue:  ownerValue{Node: ec.NodeID},
		}
		metadataB, _ := json.Marshal(metadata)
		metadataStr := string(metadataB)
		ec.Client.Create(pathMarker, metadataStr, ttl)
	}
}

// Watch will do a blocking etcd watch() on taskPath until a taskId is returned.
// The return taskId isn't guaranteed to be claimable.
//
func (ec *EtcdCoordinator) Watch() (taskID string, err error) {
	if !ec.taskWatcher.watching() {
		// Watcher hasn't been started, so start it now
		go ec.taskWatcher.watch()
	}

	for {
		select {
		case resp, ok := <-ec.taskWatcher.responseChan:
			if !ok {
				return "", nil
			}

			//The etcd watcher may have received a new task signal.
			if resp.Action == "create" {
				ec.cordCtx.Log(metafora.LogLevelDebug, "New task signaled while watching %s: response[action:%v etcdIndex:%v nodeKey:%v]",
					ec.TaskPath, resp.Action, resp.EtcdIndex, resp.Node.Key)
				taskId, skip := ec.parseTaskIdFromTaskNode(resp.Node)
				if !skip {
					return taskId, nil
				}
			}

			//The etcd watcher may have received a released task or an owner may have left the cluster.
			//Ref: etcd event types : https://github.com/coreos/etcd/blob/master/store/event.go#L4
			if (resp.Action == "expire" || resp.Action == "delete" || resp.Action == "compareAndDelete") &&
				ec.nodeIsTheOwnerMarker(resp.Node) {

				ec.cordCtx.Log(metafora.LogLevelDebug, "Released task signaled while watching %s: response[action:%v etcdIndex:%v nodeKey:%v]",
					ec.TaskPath, resp.Action, resp.EtcdIndex, resp.Node.Key)

				taskId, skip := ec.parseTaskIdFromOwnerNode(resp.Node.Key)
				if !skip {
					return taskId, nil
				}
			}

		case err := <-ec.taskWatcher.errorChan:
			return "", err
		}
	}
}

func (ec *EtcdCoordinator) parseTaskIdFromOwnerNode(nodePath string) (taskID string, skip bool) {
	//remove ec.TaskPath
	res := strings.Replace(nodePath, ec.TaskPath+"/", "", 1)
	//remove OwnerMarker
	res2 := strings.Replace(res, "/"+OwnerMarker, "", 1)
	//if remainder doens't contain "/", then we've found the taskid
	if !strings.Contains(res2, "/") {
		return res2, false
	} else {
		return "", true
	}
}

//Determines if its a task by removing the [ec.TaskPath + "/"] from the path
// and if there are still slashes then we know its a child of the the task
// not the task it's self.
func (ec *EtcdCoordinator) isATaskNode(nodePath string) bool {
	possibleTaskId := strings.Replace(nodePath, ec.TaskPath+"/", "", 1)
	if strings.Contains(possibleTaskId, "/") {
		return false
	} else {
		return true
	}
}

func (ec *EtcdCoordinator) parseTaskIdFromTaskNode(node *etcd.Node) (taskID string, skip bool) {
	taskId := ""

	if !ec.isATaskNode(node.Key) {
		return "", true
	}

	if ec.nodeHasOwnerMarker(node) {
		return "", true
	}

	taskpath := strings.Split(node.Key, "/")
	if len(taskpath) == 0 {
		//TODO log
		return "", true
	}
	if !node.Dir {
		return "", true
	}

	taskId = taskpath[len(taskpath)-1]

	if taskId == "tasks" {
		return "", true
	}

	ec.cordCtx.Log(metafora.LogLevelDebug, "A claimable task was found. task:%s key:%s", taskId, node.Key)
	return taskId, false
}

func (ec *EtcdCoordinator) nodeIsTheOwnerMarker(node *etcd.Node) bool {
	//If the node is the owner marker, it most likely means the recursive watch picked
	// this node's creation up.
	if strings.Contains(node.Key, OwnerMarker) {
		return true
	}
	return false
}

func (ec *EtcdCoordinator) nodeHasOwnerMarker(node *etcd.Node) bool {
	//If its a task with an owner marker (Child), then most likely this node came from
	// the prefetch code which found an existing task.
	if node.Nodes != nil && len(node.Nodes) > 0 {
		for _, n := range node.Nodes {
			if strings.Contains(n.Key, OwnerMarker) {
				return true
			}
		}
	}
	return false
}

// Claim is called by the Consumer when a Balancer has determined that a task
// ID can be claimed. Claim returns false if another consumer has already
// claimed the ID.
func (ec *EtcdCoordinator) Claim(taskID string) bool {
	return ec.taskManager.add(taskID)
}

// Release deletes the claim file.
func (ec *EtcdCoordinator) Release(taskID string) {
	const done = false
	ec.taskManager.remove(taskID, done)
}

// Done deletes the task.
func (ec *EtcdCoordinator) Done(taskID string) {
	const done = true
	ec.taskManager.remove(taskID, done)
}

// Command blocks until a command for this node is received from the broker
// by the coordinator.
func (ec *EtcdCoordinator) Command() (metafora.Command, error) {
	if !ec.commandWatcher.watching() {
		go ec.commandWatcher.watch()
	}

	for {
		select {
		case resp, ok := <-ec.commandWatcher.responseChan:
			if !ok {
				return nil, nil
			}

			if strings.HasSuffix(resp.Node.Key, MetadataKey) {
				// Skip metadata marker
				continue
			}

			const recurse = false
			if _, err := ec.Client.Delete(resp.Node.Key, recurse); err != nil {
				ec.cordCtx.Log(metafora.LogLevelError, "Error deleting handled command %s: %v", resp.Node.Key, err)
			}

			return metafora.UnmarshalCommand([]byte(resp.Node.Value))
		case err := <-ec.commandWatcher.errorChan:
			return nil, err
		}
	}
}

// Close stops the coordinator and causes blocking Watch and Command methods to
// return zero values.
func (ec *EtcdCoordinator) Close() {
	ec.taskWatcher.stop()
	ec.commandWatcher.stop()
	ec.taskManager.stop()
}
<commit_msg>Add comment linking to issue #61<commit_after>package m_etcd

import (
	"encoding/json"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/coreos/go-etcd/etcd"
	"github.com/lytics/metafora"
)

type ownerValue struct {
	Node string `json:"node"`
}

type watcher struct {
	cordCtx      metafora.CoordinatorContext
	path         string
	responseChan chan *etcd.Response // coordinator watches for changes
	errorChan    chan error          // coordinator watches for errors
	stopChan     chan bool           // closed to signal etcd's Watch to stop
	client       *etcd.Client
	running      bool       // only set in watch(); only read by watching()
	m            sync.Mutex // running requires synchronization
}

// watch emits etcd responses over responseChan until stopChan is closed.
func (w *watcher) watch() {
	// Setup watch state
	w.m.Lock()
	w.running = true
	w.stopChan = make(chan bool)
	w.m.Unlock()

	// Teardown watch state when watch() returns
	defer func() {
		w.m.Lock()
		w.running = false
		w.m.Unlock()
	}()

	var index uint64

	// Get existing tasks
	const sorted = true
	const recursive = true
	resp, err := w.client.Get(w.path, sorted, recursive)
	if err != nil {
		w.cordCtx.Log(metafora.LogLevelError, "Error getting the existing tasks from the path:%s error:%v", w.path, err)
		goto done
	}

	for _, node := range resp.Node.Nodes {
		// Act like these are newly created nodes
		select {
		case <-w.stopChan:
			goto done
		case w.responseChan <- &etcd.Response{Action: "create", Node: node, EtcdIndex: resp.EtcdIndex}:
		}
	}

	index = resp.EtcdIndex

	// Start blocking watch
	for {
		// Make a new inner response channel on each loop since Watch() closes the
		// one passed in.
		innerRespChan := make(chan *etcd.Response)
		go func() {
			for {
				select {
				case r, ok := <-innerRespChan:
					if !ok {
						// Don't close w.responseChan when the Watch exits
						return
					}
					// Update the last seen index
					index = r.EtcdIndex
					w.responseChan <- r
				case <-w.stopChan:
				}
			}
		}()
		// Start the blocking watch.
		_, err = w.client.Watch(w.path, index, recursive, innerRespChan, w.stopChan)
		if err != nil {
			if err == etcd.ErrWatchStoppedByUser {
				// This isn't actually an error, return nil
				err = nil
			} else if jsonErr, ok := err.(*json.SyntaxError); ok && jsonErr.Offset == 0 {
				// This is a bug in Go's HTTP transport + go-etcd which causes the
				// connection to timeout perdiocally and need to be restarted *after*
				// closing idle connections.
				w.cordCtx.Log(metafora.LogLevelDebug, "Watch timed out; restarting")
				transport.CloseIdleConnections()
				err = nil
				continue
			}
		}
		goto done
	}
done:
	w.errorChan <- err
}

// watching is a safe way for concurrent goroutines to check if the watcher is
// running.
func (w *watcher) watching() bool {
	w.m.Lock()
	r := w.running
	w.m.Unlock()
	return r
}

// stops the watching goroutine.
func (w *watcher) stop() {
	if w.watching() {
		select {
		case <-w.stopChan:
			// already stopped, let's avoid panic()s
		default:
			close(w.stopChan)
		}
	}
}

type EtcdCoordinator struct {
	Client    *etcd.Client
	cordCtx   metafora.CoordinatorContext
	Namespace string
	TaskPath  string

	taskWatcher *watcher
	ClaimTTL    uint64 // seconds

	NodeID         string
	CommandPath    string
	commandWatcher *watcher

	taskManager *taskManager
}

func NewEtcdCoordinator(nodeId, namespace string, client *etcd.Client) metafora.Coordinator {
	namespace = strings.Trim(namespace, "/ ")

	if nodeId == "" {
		hn, _ := os.Hostname()
		//Adding the UUID incase we run two nodes on the same box.
		// TODO lets move this to the Readme as part of the example of calling NewEtcdCoordinator.
		// Then just remove the Autocreated nodeId.
		nodeId = hn + uuid.NewRandom().String()
	}

	nodeId = strings.Trim(nodeId, "/ ")

	client.SetTransport(transport)
	client.SetConsistency(etcd.STRONG_CONSISTENCY)
	return &EtcdCoordinator{
		Client:    client,
		Namespace: namespace,

		TaskPath: fmt.Sprintf("/%s/%s", namespace, TasksPath),
		ClaimTTL: ClaimTTL, //default to the package constant, but allow it to be overwritten

		NodeID:      nodeId,
		CommandPath: fmt.Sprintf("/%s/%s/%s/%s", namespace, NodesPath, nodeId, CommandsPath),
	}
}

// Init is called once by the consumer to provide a Logger to Coordinator
// implementations.
func (ec *EtcdCoordinator) Init(cordCtx metafora.CoordinatorContext) {
	cordCtx.Log(metafora.LogLevelDebug, "Initializing coordinator with namespace: %s and etcd cluster: %s",
		ec.Namespace, strings.Join(ec.Client.GetCluster(), ", "))

	ec.cordCtx = cordCtx

	ec.upsertDir("/"+ec.Namespace, ForeverTTL)
	ec.upsertDir(ec.TaskPath, ForeverTTL)
	//FIXME Should get cleaned up on shutdown and have a TTL - #61
	ec.upsertDir(ec.CommandPath, ForeverTTL)

	ec.taskWatcher = &watcher{
		cordCtx:      cordCtx,
		path:         ec.TaskPath,
		responseChan: make(chan *etcd.Response),
		errorChan:    make(chan error),
		client:       ec.Client,
	}

	ec.commandWatcher = &watcher{
		cordCtx:      cordCtx,
		path:         ec.CommandPath,
		responseChan: make(chan *etcd.Response),
		errorChan:    make(chan error),
		client:       ec.Client,
	}

	ec.taskManager = newManager(cordCtx, ec.Client, ec.TaskPath, ec.NodeID, ec.ClaimTTL)
}

func (ec *EtcdCoordinator) upsertDir(path string, ttl uint64) {

	//hidden etcd key that isn't visible to ls commands on the directory,
	//  you have to know about it to find it :).  I'm using it to add some
	//  info about when the cluster's schema was setup.
	pathMarker := path + "/" + MetadataKey
	const sorted = false
	const recursive = false

	_, err := ec.Client.Get(path, sorted, recursive)
	if err == nil {
		return
	}

	etcdErr, ok := err.(*etcd.EtcdError)
	if ok && etcdErr.ErrorCode == EcodeKeyNotFound {
		_, err := ec.Client.CreateDir(path, ttl)
		if err != nil {
			ec.cordCtx.Log(metafora.LogLevelDebug, "Error trying to create directory. path:[%s] error:[ %v ]", path, err)
		}
		host, _ := os.Hostname()

		metadata := struct {
			Host        string `json:"host"`
			CreatedTime string `json:"created"`
			ownerValue
		}{
			Host:        host,
			CreatedTime: time.Now().String(),
			ownerValue:  ownerValue{Node: ec.NodeID},
		}
		metadataB, _ := json.Marshal(metadata)
		metadataStr := string(metadataB)
		ec.Client.Create(pathMarker, metadataStr, ttl)
	}
}

// Watch will do a blocking etcd watch() on taskPath until a taskId is returned.
// The return taskId isn't guaranteed to be claimable.
//
func (ec *EtcdCoordinator) Watch() (taskID string, err error) {
	if !ec.taskWatcher.watching() {
		// Watcher hasn't been started, so start it now
		go ec.taskWatcher.watch()
	}

	for {
		select {
		case resp, ok := <-ec.taskWatcher.responseChan:
			if !ok {
				return "", nil
			}

			//The etcd watcher may have received a new task signal.
			if resp.Action == "create" {
				ec.cordCtx.Log(metafora.LogLevelDebug, "New task signaled while watching %s: response[action:%v etcdIndex:%v nodeKey:%v]",
					ec.TaskPath, resp.Action, resp.EtcdIndex, resp.Node.Key)
				taskId, skip := ec.parseTaskIdFromTaskNode(resp.Node)
				if !skip {
					return taskId, nil
				}
			}

			//The etcd watcher may have received a released task or an owner may have left the cluster.
			//Ref: etcd event types : https://github.com/coreos/etcd/blob/master/store/event.go#L4
			if (resp.Action == "expire" || resp.Action == "delete" || resp.Action == "compareAndDelete") &&
				ec.nodeIsTheOwnerMarker(resp.Node) {

				ec.cordCtx.Log(metafora.LogLevelDebug, "Released task signaled while watching %s: response[action:%v etcdIndex:%v nodeKey:%v]",
					ec.TaskPath, resp.Action, resp.EtcdIndex, resp.Node.Key)

				taskId, skip := ec.parseTaskIdFromOwnerNode(resp.Node.Key)
				if !skip {
					return taskId, nil
				}
			}

		case err := <-ec.taskWatcher.errorChan:
			return "", err
		}
	}
}

func (ec *EtcdCoordinator) parseTaskIdFromOwnerNode(nodePath string) (taskID string, skip bool) {
	//remove ec.TaskPath
	res := strings.Replace(nodePath, ec.TaskPath+"/", "", 1)
	//remove OwnerMarker
	res2 := strings.Replace(res, "/"+OwnerMarker, "", 1)
	//if remainder doens't contain "/", then we've found the taskid
	if !strings.Contains(res2, "/") {
		return res2, false
	} else {
		return "", true
	}
}

//Determines if its a task by removing the [ec.TaskPath + "/"] from the path
// and if there are still slashes then we know its a child of the the task
// not the task it's self.
func (ec *EtcdCoordinator) isATaskNode(nodePath string) bool {
	possibleTaskId := strings.Replace(nodePath, ec.TaskPath+"/", "", 1)
	if strings.Contains(possibleTaskId, "/") {
		return false
	} else {
		return true
	}
}

func (ec *EtcdCoordinator) parseTaskIdFromTaskNode(node *etcd.Node) (taskID string, skip bool) {
	taskId := ""

	if !ec.isATaskNode(node.Key) {
		return "", true
	}

	if ec.nodeHasOwnerMarker(node) {
		return "", true
	}

	taskpath := strings.Split(node.Key, "/")
	if len(taskpath) == 0 {
		//TODO log
		return "", true
	}
	if !node.Dir {
		return "", true
	}

	taskId = taskpath[len(taskpath)-1]

	if taskId == "tasks" {
		return "", true
	}

	ec.cordCtx.Log(metafora.LogLevelDebug, "A claimable task was found. task:%s key:%s", taskId, node.Key)
	return taskId, false
}

func (ec *EtcdCoordinator) nodeIsTheOwnerMarker(node *etcd.Node) bool {
	//If the node is the owner marker, it most likely means the recursive watch picked
	// this node's creation up.
	if strings.Contains(node.Key, OwnerMarker) {
		return true
	}
	return false
}

func (ec *EtcdCoordinator) nodeHasOwnerMarker(node *etcd.Node) bool {
	//If its a task with an owner marker (Child), then most likely this node came from
	// the prefetch code which found an existing task.
	if node.Nodes != nil && len(node.Nodes) > 0 {
		for _, n := range node.Nodes {
			if strings.Contains(n.Key, OwnerMarker) {
				return true
			}
		}
	}
	return false
}

// Claim is called by the Consumer when a Balancer has determined that a task
// ID can be claimed. Claim returns false if another consumer has already
// claimed the ID.
func (ec *EtcdCoordinator) Claim(taskID string) bool {
	return ec.taskManager.add(taskID)
}

// Release deletes the claim file.
func (ec *EtcdCoordinator) Release(taskID string) {
	const done = false
	ec.taskManager.remove(taskID, done)
}

// Done deletes the task.
func (ec *EtcdCoordinator) Done(taskID string) {
	const done = true
	ec.taskManager.remove(taskID, done)
}

// Command blocks until a command for this node is received from the broker
// by the coordinator.
func (ec *EtcdCoordinator) Command() (metafora.Command, error) {
	if !ec.commandWatcher.watching() {
		go ec.commandWatcher.watch()
	}

	for {
		select {
		case resp, ok := <-ec.commandWatcher.responseChan:
			if !ok {
				return nil, nil
			}

			if strings.HasSuffix(resp.Node.Key, MetadataKey) {
				// Skip metadata marker
				continue
			}

			const recurse = false
			if _, err := ec.Client.Delete(resp.Node.Key, recurse); err != nil {
				ec.cordCtx.Log(metafora.LogLevelError, "Error deleting handled command %s: %v", resp.Node.Key, err)
			}

			return metafora.UnmarshalCommand([]byte(resp.Node.Value))
		case err := <-ec.commandWatcher.errorChan:
			return nil, err
		}
	}
}

// Close stops the coordinator and causes blocking Watch and Command methods to
// return zero values.
func (ec *EtcdCoordinator) Close() {
	ec.taskWatcher.stop()
	ec.commandWatcher.stop()
	ec.taskManager.stop()
}
<|endoftext|>


<commit_before>package vkapi

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"
)

const (
	defaultHTTPTimeout        = 60 * time.Second
	defaultRequestTimeout     = 120 * time.Second
	defaultKeepAliveInterval  = 60 * time.Second
	defaultHTTPHeadersTimeout = defaultRequestTimeout
)

type Response struct {
	Errors   Errors `json:"execute_errors,omitempty"`
	Error    Error  `json:"error,omitempty"`
	Response Raw    `json:"response,omitempty"`
}

type Request struct {
	Method string     `json:"method"`
	Token  string     `json:"token"`
	Values url.Values `json:"values"`
}

type Raw []byte

func (r Raw) Bytes() []byte {
	return []byte(r)
}

func (r Raw) String() string {
	return bytes.NewBuffer(r).String()
}

func (m Raw) MarshalJSON() ([]byte, error) {
	log.Println("marshalling to", m)
	return m, nil
}

func (m *Raw) UnmarshalJSON(data []byte) error {
	*m = data
	return nil
}

type HTTPClient interface {
	Do(*http.Request) (*http.Response, error)
}

func must(err error) {
	if err != nil {
		panic(err)
	}
}

func (api *ApiClient) Do(request Request) (response *Response, error *Error) {
	for i, v := range api.Values() {
		if request.Values.Get(i) == "" {
			request.Values.Add(i, v[0])
		}
	}

	req := request.HTTP()
	start := time.Now()
	log.Println("DO", request.Method)

	var res *http.Response
	var err interface {
		Error() string
	}

	for attempt := 1; attempt < 5; attempt++ {
		res, err = api.httpClient.Do(req)
		if err == nil {
			break
		}

		log.Println("HTTP attempt", err, attempt)
		time.Sleep(time.Second * 3)
	}

	if err != nil {
		log.Println("HTTP fatal", err)
		return nil, NewError(ErrBadCode, "HTTP fatal "+err.Error())
	}

	log.Println("HTTP", res.Status, time.Now().Sub(start))
	if res.StatusCode != http.StatusOK {
		return nil, NewError(ErrBadResponseCode, res.Status)
	}

	return Process(res.Body)
}

func (r Request) HTTP() (req *http.Request) {
	values := r.Values

	if r.Token != "" {
		values.Set(paramToken, r.Token)
	}

	u := ApiUrl()
	u.Path = path.Join(u.Path, r.Method)
	u.RawQuery = values.Encode()

	log.Print("URL: ", u.String())
	req, err := http.NewRequest(defaultMethod, u.String(), nil)

	must(err)

	return req
}

func (r Request) JS() string {
	args := make(map[string]string)
	for k := range r.Values {
		args[k] = r.Values.Get(k)
	}
	js := new(bytes.Buffer)
	encoder := json.NewEncoder(js)

	must(encoder.Encode(args))

	jsString := js.String()
	jsString = strings.TrimSpace(jsString)

	return fmt.Sprintf("API.%s(%s)", r.Method, jsString)
}

type vkResponseProcessor struct {
	input io.Reader
}

func (r Response) To(v interface{}) error {
	return json.Unmarshal(r.Response.Bytes(), v)
}

func (r Response) ServerError() error {
	if r.Errors != nil {
		return r.Errors
	}
	if r.Error.Code == ErrZero {
		return nil
	}
	return r.Error
}

func (d vkResponseProcessor) To(response *Response) *Error {
	if rc, ok := d.input.(io.ReadCloser); ok {
		defer rc.Close()
	}

	decoder := json.NewDecoder(d.input)

	if err := decoder.Decode(response); err != nil {
		return NewError(ErrBadCode, err.Error())
	}

	return &response.Error
}

func Process(input io.Reader) (response *Response, err *Error) {
	response = new(Response)
	return response, vkResponseProcessor{input}.To(response)
}

type Encoder struct {
	response *Response
	err      error
}

func (e Encoder) To(v interface{}) error {
	if e.err != nil {
		return e.err
	}
	if e.response == nil {
		return errors.New("wtf")
	}
	return e.response.To(v)
}

func Encode(input io.Reader) Encoder {
	res, err := Process(input)
	return Encoder{res, err}
}

func defaultHTTPClient() HTTPClient {
	client := &http.Client{
		Timeout: defaultRequestTimeout,
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   defaultHTTPTimeout,
				KeepAlive: defaultKeepAliveInterval,
			}).Dial,
			TLSHandshakeTimeout:   defaultHTTPTimeout,
			ResponseHeaderTimeout: defaultHTTPHeadersTimeout,
		},
	}
	return client
}
<commit_msg>Fixed errors<commit_after>package vkapi

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"
)

const (
	defaultHTTPTimeout        = 60 * time.Second
	defaultRequestTimeout     = 120 * time.Second
	defaultKeepAliveInterval  = 60 * time.Second
	defaultHTTPHeadersTimeout = defaultRequestTimeout
)

type Response struct {
	Errors   *Errors `json:"execute_errors,omitempty"`
	Error    *Error  `json:"error,omitempty"`
	Response Raw     `json:"response,omitempty"`
}

type Request struct {
	Method string     `json:"method"`
	Token  string     `json:"token"`
	Values url.Values `json:"values"`
}

type Raw []byte

func (r Raw) Bytes() []byte {
	return []byte(r)
}

func (r Raw) String() string {
	return bytes.NewBuffer(r).String()
}

func (m Raw) MarshalJSON() ([]byte, error) {
	log.Println("marshalling to", m)
	return m, nil
}

func (m *Raw) UnmarshalJSON(data []byte) error {
	*m = data
	return nil
}

type HTTPClient interface {
	Do(*http.Request) (*http.Response, error)
}

func must(err error) {
	if err != nil {
		panic(err)
	}
}

func (api *ApiClient) Do(request Request) (response *Response, error *Error) {
	for i, v := range api.Values() {
		if request.Values.Get(i) == "" {
			request.Values.Add(i, v[0])
		}
	}

	req := request.HTTP()
	start := time.Now()
	log.Println("DO", request.Method)

	var res *http.Response
	var err interface {
		Error() string
	}

	for attempt := 1; attempt < 5; attempt++ {
		res, err = api.httpClient.Do(req)
		if err == nil {
			break
		}

		log.Println("HTTP attempt", err, attempt)
		time.Sleep(time.Second * 3)
	}

	if err != nil {
		log.Println("HTTP fatal", err)
		return nil, NewError(ErrBadCode, "HTTP fatal "+err.Error())
	}

	log.Println("HTTP", res.Status, time.Now().Sub(start))
	if res.StatusCode != http.StatusOK {
		return nil, NewError(ErrBadResponseCode, res.Status)
	}

	return Process(res.Body)
}

func (r Request) HTTP() (req *http.Request) {
	values := r.Values

	if r.Token != "" {
		values.Set(paramToken, r.Token)
	}

	u := ApiUrl()
	u.Path = path.Join(u.Path, r.Method)
	u.RawQuery = values.Encode()

	log.Print("URL: ", u.String())
	req, err := http.NewRequest(defaultMethod, u.String(), nil)

	must(err)

	return req
}

func (r Request) JS() string {
	args := make(map[string]string)
	for k := range r.Values {
		args[k] = r.Values.Get(k)
	}
	js := new(bytes.Buffer)
	encoder := json.NewEncoder(js)

	must(encoder.Encode(args))

	jsString := js.String()
	jsString = strings.TrimSpace(jsString)

	return fmt.Sprintf("API.%s(%s)", r.Method, jsString)
}

type vkResponseProcessor struct {
	input io.Reader
}

func (r Response) To(v interface{}) error {
	return json.Unmarshal(r.Response.Bytes(), v)
}

func (r Response) ServerError() error {
	if r.Errors != nil {
		return r.Errors
	}
	if r.Error.Code == ErrZero {
		return nil
	}
	return r.Error
}

func (d vkResponseProcessor) To(response *Response) *Error {
	if rc, ok := d.input.(io.ReadCloser); ok {
		defer rc.Close()
	}

	decoder := json.NewDecoder(d.input)

	if err := decoder.Decode(response); err != nil {
		return NewError(ErrBadCode, err.Error())
	}

	return response.Error
}

func Process(input io.Reader) (response *Response, err *Error) {
	response = new(Response)
	return response, vkResponseProcessor{input}.To(response)
}

type Encoder struct {
	response *Response
	err      error
}

func (e Encoder) To(v interface{}) error {
	if e.err != nil {
		return e.err
	}
	if e.response == nil {
		return errors.New("wtf")
	}
	return e.response.To(v)
}

func Encode(input io.Reader) Encoder {
	res, err := Process(input)
	return Encoder{res, err}
}

func defaultHTTPClient() HTTPClient {
	client := &http.Client{
		Timeout: defaultRequestTimeout,
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   defaultHTTPTimeout,
				KeepAlive: defaultKeepAliveInterval,
			}).Dial,
			TLSHandshakeTimeout:   defaultHTTPTimeout,
			ResponseHeaderTimeout: defaultHTTPHeadersTimeout,
		},
	}
	return client
}
<|endoftext|>


<commit_before>package db

import (
	"path"

	// Allows connecting to mysql
	_ "github.com/go-sql-driver/mysql"
	"github.com/golang/glog"
)

// NewMySQLVolumeDatabase creates a new SQLVolumeDatabase, connectiong to a mysql host.
func NewMySQLVolumeDatabase(host string, username string, password string, schema string) SQLVolumeDatabase {

	var queries VolumeDatabaseQueries
	queries.volumesCreateTableSQL = `CREATE TABLE IF NOT EXISTS volumes (
        id INTEGER NOT NULL PRIMARY KEY AUTO_INCREMENT,
        name VARCHAR(256) NOT NULL UNIQUE,
        mountpoint TEXT
    )`

	// [username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
	var connection string
	var printedConnection string

	if username == "" {
		username = "root"
	}
	connection = username
	printedConnection = username

	if password != "" {
		connection += ":" + password
		printedConnection += ":<supplied password>"
	}

	connection += "@" + host
	printedConnection += "@" + host

	if schema == "" {
		glog.Fatal("A database schema must be specified with -dbschema")
	}
	connection += path.Join("/", schema)
	printedConnection += path.Join("/", schema)

	// Create the connection
	glog.Info("Connecting to", printedConnection)
	return NewSQLVolumeDatabase("mysql", connection, queries)
}
<commit_msg>Use better join in mysql config<commit_after>package db

import (
	"path"

	// Allows connecting to mysql
	_ "github.com/go-sql-driver/mysql"
	"github.com/golang/glog"
)

// NewMySQLVolumeDatabase creates a new SQLVolumeDatabase, connectiong to a mysql host.
func NewMySQLVolumeDatabase(host string, username string, password string, schema string) SQLVolumeDatabase {

	var queries VolumeDatabaseQueries
	queries.volumesCreateTableSQL = `CREATE TABLE IF NOT EXISTS volumes (
        id INTEGER NOT NULL PRIMARY KEY AUTO_INCREMENT,
        name VARCHAR(256) NOT NULL UNIQUE,
        mountpoint TEXT
    )`

	// [username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
	var connection string
	var printedConnection string

	if username == "" {
		username = "root"
	}
	connection = username
	printedConnection = username

	if password != "" {
		connection += ":" + password
		printedConnection += ":<supplied password>"
	}

	connection += "@" + host
	printedConnection += "@" + host

	if schema == "" {
		glog.Fatal("A database schema must be specified with -dbschema")
	}
	connection = path.Join(connection, schema)
	printedConnection = path.Join(printedConnection, schema)

	// Create the connection
	glog.Info("Connecting to", printedConnection)
	return NewSQLVolumeDatabase("mysql", connection, queries)
}
<|endoftext|>


<commit_before>/*
Package decimals is a small library of functions for rounding and
formatting base ten numbers. These are functions that are either
missing from the standard libraries or are more convenient for
presenting numbers in a human-readable format.
*/
package decimals

import (
	"strconv"
	"math"
	"fmt"
)

// RoundInt rounds a base ten int64 to the given precision. Precision is a
// negative number that represents the nearest power of ten to which the
// integer should be rounded. It is expressed as a negative number to be
// consistent with the decimal precision arguments used in rounding floats.
// If the rounded number falls outside the minimum and maximum for int64
// the minimum or maximum will be returned instead.
func RoundInt(x int64, precision int) int64 {

	var(
		xstr string = strconv.FormatInt(x, 10)
		xslice = []byte(xstr)
		zeroFrom int = -1
		roundFrom int
	)

	// Map for converting decimal bytes to int64
	decimalInts := map[byte]int64{
		'0': 0, '1': 1, '2': 2, '3': 3, '4': 4,
		'5': 5, '6': 6, '7': 7, '8': 8, '9': 9,
	}

	// Array for converting decimal ints to bytes
	decimalBytes := []byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',}

	// If precision is not negative return x
	if precision > -1 {

		return x
	}

	// If x is negative remove the sign
	if x < 0 {

		xslice = xslice[1:]
	}

	// Set the index of the digit to round from
	roundFrom = len(xslice) + precision

	// If rounding to more than one order of magnitude larger than x return 0
	if roundFrom < 0 {

		return 0
	}

	// If rounding to one order of magnitude larger than x round from first digit
	if roundFrom == 0 {

		firstDigit := decimalInts[xslice[0]]

		if firstDigit < 5 {

			return 0

		} else {

			xslice = append([]byte{'1'}, xslice...)
			zeroFrom = 1
		}

	// Otherwise round through the slice from right to left
	} else {

		// Start rounding from the round digit
		roundDigit := decimalInts[xslice[roundFrom]]

		// If less than five round from there
		if roundDigit < 5 {

			zeroFrom = roundFrom

		// Otherwise keep moving left to find the rounding point
		} else {

			for i := roundFrom; i > 0; i-- {

				j := i - 1
				nextDigit := decimalInts[xslice[j]]

				if nextDigit < 9 {

					xslice[j] = decimalBytes[nextDigit + 1]
					zeroFrom = i
					break
				}
			}

			// If not found add a leading one and round from there
			if zeroFrom == -1 {

				xslice = append([]byte{'1'}, xslice...)
				zeroFrom = 1
			}
		}
	}

	// Zero all digits after the rounding point
	for i := zeroFrom; i < len(xslice); i++ {

		xslice[i] = '0'
	}

	// If x is negative add the sign back
	if x < 0 {

		xslice = append([]byte("-"), xslice...)
	}

	// Convert the slice back to an int64
	rstr := string(xslice)
	r, _ := strconv.ParseInt(rstr, 10, 64)

	return r
}

// parseIntFromDigit takes a byte representing a decimal character and returns
// the value as an int64. The function will return an error if the byte cannot
// be cast to a decimal integer with with strconv.ParseInt().
func parseIntFromDigit(b byte) (int64, error) {

	x, err  := strconv.ParseInt(string(b), 10, 64)

	if err != nil {
		return 0, err
	}

	return x, nil
}

// getDigitFromInt takes an int64 between 0-9 and returns the byte for
// its string representation. The function will return an error if the
// integer is outside this range.
func getDigitFromInt(x int64) (byte, error) {

	if x < 0 || x > 9 {
		return []byte(strconv.FormatInt(0, 10))[0],
			fmt.Errorf("decimals: %d is not a digit between 0 and 9", x)
	}

	return []byte(strconv.FormatInt(x, 10))[0], nil
}

// RoundFloat rounds a base ten float64 to the given decimal precision.
// Precision may be positive, representing the number of decimal places,
// or negative, representing the nearest power of ten to which the float
// should be rounded.
func RoundFloat(x float64, precision int) float64 {

	// Handle negative precision with integer rounding
	if precision < 0 {

		i, _ := math.Modf(x)
		return float64(RoundInt(int64(i), precision))
	}

	// Handle positive precision with strconv.FormatFloat()
	rstr := strconv.FormatFloat(x, 'f', precision, 64)
	r, _ := strconv.ParseFloat(rstr, 64)

	return r
}

// FormatThousands converts an int64 into a string formatted using a comma
// separator for thousands.
func FormatThousands(x int64) string {

	var (
		xstr string
		xslice []byte
		fslice []byte
		lenx int
		lenf int
		commas int
	)

	// Get the number as a byte slice
	xstr = strconv.FormatInt(x, 10)
	xslice = []byte(xstr)
	lenx = len(xslice)

	// Determine the number of commas depending on the sign of x
	if x < 0 {

		commas = (lenx -2) / 3
		lenf = lenx + commas

	} else {

		commas = (lenx -1) / 3
		lenf = lenx + commas

	}

	// Create an empty byte slice for the formatted number
	fslice = make([]byte, lenf)

	// Copy the digits from right to left, adding commas
	i := lenx - 1
	j := lenf - 1

	// Copy the digits in batches of three
	for k := 0; k < commas; k++ {

		for l := 0; l < 3; l++ {

			fslice[j] = xslice[i]
			i--
			j--
		}

		// Add the comma
		fslice[j] = []byte(",")[0]
		j--
	}

	// Copy the remaining digits
	for ; i >= 0; i, j = i - 1, j - 1 {

		fslice[j] = xslice[i]
	}

	return string(fslice)
}

// FormatInt converts an int64 to a formatted string. The int is rounded
// to the given precision and formatted using a comma separator for thousands.
func FormatInt(x int64, precision int) string {

	return FormatThousands(RoundInt(x, precision))
}

// FormatFloat converts a float64 to a formatted string. The float is rounded
// to the given precision and formatted using a comma separator for thousands.
func FormatFloat(x float64, precision int) string {

	// Round the float and get the decimal and fractional parts
	r := RoundFloat(x, precision)
	i, f := math.Modf(r)
	is := FormatThousands(int64(i))

	// If precision is less than one return the formatted integer part
	if precision <= 0 {

		return is
	}

	// Otherwise convert the fractional part to a string
	fs := strconv.FormatFloat(f, 'f', precision, 64)

	// And get the digits after the decimal point
	if x < 0 {

		fs = fs[3:]

	} else {

		fs = fs[2:]
	}

	// Concatenate the decimal and fractional parts and return
	return is + "." + fs
}
<commit_msg>2.0.0<commit_after>/*
Package decimals is a small library of functions for rounding and
formatting base ten numbers. These are functions that are either
missing from the standard libraries or are more convenient for
presenting numbers in a human-readable format.
*/
package decimals

import (
	"strconv"
	"math"
)

// RoundInt rounds a base ten int64 to the given precision. Precision is a
// negative number that represents the nearest power of ten to which the
// integer should be rounded. It is expressed as a negative number to be
// consistent with the decimal precision arguments used in rounding floats.
// If the rounded number falls outside the minimum and maximum for int64
// the minimum or maximum will be returned instead.
func RoundInt(x int64, precision int) int64 {

	var(
		xstr string = strconv.FormatInt(x, 10)
		xslice = []byte(xstr)
		zeroFrom int = -1
		roundFrom int
	)

	// Map for converting decimal bytes to int64
	decimalInts := map[byte]int64{
		'0': 0, '1': 1, '2': 2, '3': 3, '4': 4,
		'5': 5, '6': 6, '7': 7, '8': 8, '9': 9,
	}

	// Array for converting decimal ints to bytes
	decimalBytes := []byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',}

	// If precision is not negative return x
	if precision > -1 {

		return x
	}

	// If x is negative remove the sign
	if x < 0 {

		xslice = xslice[1:]
	}

	// Set the index of the digit to round from
	roundFrom = len(xslice) + precision

	// If rounding to more than one order of magnitude larger than x return 0
	if roundFrom < 0 {

		return 0
	}

	// If rounding to one order of magnitude larger than x round from first digit
	if roundFrom == 0 {

		firstDigit := decimalInts[xslice[0]]

		if firstDigit < 5 {

			return 0

		} else {

			xslice = append([]byte{'1'}, xslice...)
			zeroFrom = 1
		}

	// Otherwise round through the slice from right to left
	} else {

		// Start rounding from the round digit
		roundDigit := decimalInts[xslice[roundFrom]]

		// If less than five round from there
		if roundDigit < 5 {

			zeroFrom = roundFrom

		// Otherwise keep moving left to find the rounding point
		} else {

			for i := roundFrom; i > 0; i-- {

				j := i - 1
				nextDigit := decimalInts[xslice[j]]

				if nextDigit < 9 {

					xslice[j] = decimalBytes[nextDigit + 1]
					zeroFrom = i
					break
				}
			}

			// If not found add a leading one and round from there
			if zeroFrom == -1 {

				xslice = append([]byte{'1'}, xslice...)
				zeroFrom = 1
			}
		}
	}

	// Zero all digits after the rounding point
	for i := zeroFrom; i < len(xslice); i++ {

		xslice[i] = '0'
	}

	// If x is negative add the sign back
	if x < 0 {

		xslice = append([]byte("-"), xslice...)
	}

	// Convert the slice back to an int64
	rstr := string(xslice)
	r, _ := strconv.ParseInt(rstr, 10, 64)

	return r
}

// RoundFloat rounds a base ten float64 to the given decimal precision.
// Precision may be positive, representing the number of decimal places,
// or negative, representing the nearest power of ten to which the float
// should be rounded.
func RoundFloat(x float64, precision int) float64 {

	// Handle negative precision with integer rounding
	if precision < 0 {

		i, _ := math.Modf(x)
		return float64(RoundInt(int64(i), precision))
	}

	// Handle positive precision with strconv.FormatFloat()
	rstr := strconv.FormatFloat(x, 'f', precision, 64)
	r, _ := strconv.ParseFloat(rstr, 64)

	return r
}

// FormatThousands converts an int64 into a string formatted using a comma
// separator for thousands.
func FormatThousands(x int64) string {

	var (
		xstr string
		xslice []byte
		fslice []byte
		lenx int
		lenf int
		commas int
	)

	// Get the number as a byte slice
	xstr = strconv.FormatInt(x, 10)
	xslice = []byte(xstr)
	lenx = len(xslice)

	// Determine the number of commas depending on the sign of x
	if x < 0 {

		commas = (lenx -2) / 3
		lenf = lenx + commas

	} else {

		commas = (lenx -1) / 3
		lenf = lenx + commas

	}

	// Create an empty byte slice for the formatted number
	fslice = make([]byte, lenf)

	// Copy the digits from right to left, adding commas
	i := lenx - 1
	j := lenf - 1

	// Copy the digits in batches of three
	for k := 0; k < commas; k++ {

		for l := 0; l < 3; l++ {

			fslice[j] = xslice[i]
			i--
			j--
		}

		// Add the comma
		fslice[j] = []byte(",")[0]
		j--
	}

	// Copy the remaining digits
	for ; i >= 0; i, j = i - 1, j - 1 {

		fslice[j] = xslice[i]
	}

	return string(fslice)
}

// FormatInt converts an int64 to a formatted string. The int is rounded
// to the given precision and formatted using a comma separator for thousands.
func FormatInt(x int64, precision int) string {

	return FormatThousands(RoundInt(x, precision))
}

// FormatFloat converts a float64 to a formatted string. The float is rounded
// to the given precision and formatted using a comma separator for thousands.
func FormatFloat(x float64, precision int) string {

	// Round the float and get the decimal and fractional parts
	r := RoundFloat(x, precision)
	i, f := math.Modf(r)
	is := FormatThousands(int64(i))

	// If precision is less than one return the formatted integer part
	if precision <= 0 {

		return is
	}

	// Otherwise convert the fractional part to a string
	fs := strconv.FormatFloat(f, 'f', precision, 64)

	// And get the digits after the decimal point
	if x < 0 {

		fs = fs[3:]

	} else {

		fs = fs[2:]
	}

	// Concatenate the decimal and fractional parts and return
	return is + "." + fs
}
<|endoftext|>


<commit_before>package youtube

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"log"
	"net/url"
	"regexp"
	"strconv"

	"github.com/dop251/goja"
)

func (c *Client) decipherURL(ctx context.Context, videoID string, cipher string) (string, error) {
	params, err := url.ParseQuery(cipher)
	if err != nil {
		return "", err
	}

	uri, err := url.Parse(params.Get("url"))
	if err != nil {
		return "", err
	}
	query := uri.Query()

	config, err := c.getPlayerConfig(ctx, videoID)
	if err != nil {
		return "", err
	}

	// decrypt s-parameter
	bs, err := config.decrypt([]byte(params.Get("s")))
	if err != nil {
		return "", err
	}
	query.Add(params.Get("sp"), string(bs))

	query, err = c.decryptNParam(ctx, config, query)
	if err != nil {
		return "", err
	}

	uri.RawQuery = query.Encode()

	return uri.String(), nil
}

// see https://github.com/kkdai/youtube/pull/244
func (c *Client) unThrottle(ctx context.Context, videoID string, urlString string) (string, error) {
	config, err := c.getPlayerConfig(ctx, videoID)
	if err != nil {
		return "", err
	}

	uri, err := url.Parse(urlString)
	if err != nil {
		return "", err
	}

	query, err := c.decryptNParam(ctx, config, uri.Query())
	if err != nil {
		return "", err
	}

	uri.RawQuery = query.Encode()
	return uri.String(), nil
}

func (c *Client) decryptNParam(ctx context.Context, config playerConfig, query url.Values) (url.Values, error) {
	// decrypt n-parameter
	nSig := query.Get("n")
	if nSig != "" {
		nDecoded, err := config.decodeNsig(nSig)
		if err != nil {
			return nil, fmt.Errorf("unable to decode nSig: %w", err)
		}
		query.Set("n", nDecoded)
	}

	if c.Debug {
		log.Printf("[nParam] n: %s; nDecoded: %s\nQuery: %v\n", nSig, query.Get("n"), query)
	}

	return query, nil
}

const (
	jsvarStr   = "[a-zA-Z_\\$][a-zA-Z_0-9]*"
	reverseStr = ":function\\(a\\)\\{" +
		"(?:return )?a\\.reverse\\(\\)" +
		"\\}"
	spliceStr = ":function\\(a,b\\)\\{" +
		"a\\.splice\\(0,b\\)" +
		"\\}"
	swapStr = ":function\\(a,b\\)\\{" +
		"var c=a\\[0\\];a\\[0\\]=a\\[b(?:%a\\.length)?\\];a\\[b(?:%a\\.length)?\\]=c(?:;return a)?" +
		"\\}"
)

var (
	nFunctionNameRegexp = regexp.MustCompile("\\.get\\(\"n\"\\)\\)&&\\(b=([a-zA-Z0-9$]{0,3})\\[(\\d+)\\](.+)\\|\\|([a-zA-Z0-9]{0,3})")
	actionsObjRegexp    = regexp.MustCompile(fmt.Sprintf(
		"var (%s)=\\{((?:(?:%s%s|%s%s|%s%s),?\\n?)+)\\};", jsvarStr, jsvarStr, swapStr, jsvarStr, spliceStr, jsvarStr, reverseStr))

	actionsFuncRegexp = regexp.MustCompile(fmt.Sprintf(
		"function(?: %s)?\\(a\\)\\{"+
			"a=a\\.split\\(\"\"\\);\\s*"+
			"((?:(?:a=)?%s\\.%s\\(a,\\d+\\);)+)"+
			"return a\\.join\\(\"\"\\)"+
			"\\}", jsvarStr, jsvarStr, jsvarStr))

	reverseRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, reverseStr))
	spliceRegexp  = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, spliceStr))
	swapRegexp    = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, swapStr))
)

func (config playerConfig) decodeNsig(encoded string) (string, error) {
	fBody, err := config.getNFunction()
	if err != nil {
		return "", err
	}

	return evalJavascript(fBody, encoded)
}

func evalJavascript(jsFunction, arg string) (string, error) {
	const myName = "myFunction"

	vm := goja.New()
	_, err := vm.RunString(myName + jsFunction)
	if err != nil {
		return "", err
	}

	var output func(string) string
	err = vm.ExportTo(vm.Get(myName), &output)
	if err != nil {
		return "", err
	}

	return output(arg), nil
}

func (config playerConfig) getNFunction() (string, error) {
	nameResult := nFunctionNameRegexp.FindSubmatch(config)
	if len(nameResult) == 0 {
		log.Println("playerConfig:", string(config))
		return "", errors.New("unable to extract n-function name")
	}

	var name string
	if idx, _ := strconv.Atoi(string(nameResult[2])); idx == 0 {
		name = string(nameResult[4])
	} else {
		name = string(nameResult[1])
	}

	// find the beginning of the function
	def := []byte(name + "=function(")
	start := bytes.Index(config, def)
	if start < 1 {
		return "", fmt.Errorf("unable to extract n-function body: looking for '%s'", def)
	}

	// start after the first curly bracket
	pos := start + bytes.IndexByte(config[start:], '{') + 1

	// find the bracket closing the function
	for brackets := 1; brackets > 0; pos++ {
		switch config[pos] {
		case '{':
			brackets++
		case '}':
			brackets--
		}
	}

	return string(config[start:pos]), nil
}

func (config playerConfig) decrypt(cyphertext []byte) ([]byte, error) {
	operations, err := config.parseDecipherOps()
	if err != nil {
		return nil, err
	}

	// apply operations
	bs := []byte(cyphertext)
	for _, op := range operations {
		bs = op(bs)
	}

	return bs, nil
}

/*
	parses decipher operations from https://youtube.com/s/player/4fbb4d5b/player_ias.vflset/en_US/base.js

	var Mt={
	splice:function(a,b){a.splice(0,b)},
	reverse:function(a){a.reverse()},
	EQ:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}};

	a=a.split("");
	Mt.splice(a,3);
	Mt.EQ(a,39);
	Mt.splice(a,2);
	Mt.EQ(a,1);
	Mt.splice(a,1);
	Mt.EQ(a,35);
	Mt.EQ(a,51);
	Mt.splice(a,2);
	Mt.reverse(a,52);
	return a.join("")
*/
func (config playerConfig) parseDecipherOps() (operations []DecipherOperation, err error) {
	objResult := actionsObjRegexp.FindSubmatch(config)
	funcResult := actionsFuncRegexp.FindSubmatch(config)
	if len(objResult) < 3 || len(funcResult) < 2 {
		return nil, fmt.Errorf("error parsing signature tokens (#obj=%d, #func=%d)", len(objResult), len(funcResult))
	}

	obj := objResult[1]
	objBody := objResult[2]
	funcBody := funcResult[1]

	var reverseKey, spliceKey, swapKey string

	if result := reverseRegexp.FindSubmatch(objBody); len(result) > 1 {
		reverseKey = string(result[1])
	}
	if result := spliceRegexp.FindSubmatch(objBody); len(result) > 1 {
		spliceKey = string(result[1])
	}
	if result := swapRegexp.FindSubmatch(objBody); len(result) > 1 {
		swapKey = string(result[1])
	}

	regex, err := regexp.Compile(fmt.Sprintf("(?:a=)?%s\\.(%s|%s|%s)\\(a,(\\d+)\\)", regexp.QuoteMeta(string(obj)), regexp.QuoteMeta(reverseKey), regexp.QuoteMeta(spliceKey), regexp.QuoteMeta(swapKey)))
	if err != nil {
		return nil, err
	}

	var ops []DecipherOperation
	for _, s := range regex.FindAllSubmatch(funcBody, -1) {
		switch string(s[1]) {
		case reverseKey:
			ops = append(ops, reverseFunc)
		case swapKey:
			arg, _ := strconv.Atoi(string(s[2]))
			ops = append(ops, newSwapFunc(arg))
		case spliceKey:
			arg, _ := strconv.Atoi(string(s[2]))
			ops = append(ops, newSpliceFunc(arg))
		}
	}
	return ops, nil
}

func (config playerConfig) getSignatureTimestamp() (string, error) {
	result := signatureRegexp.FindSubmatch(config)
	if result == nil {
		return "", ErrSignatureTimestampNotFound
	}

	return string(result[1]), nil
}
<commit_msg>Revert "fix unable to decode nSig: ReferenceError: myFunction is not defined at <eval>:1:1(0)"<commit_after>package youtube

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"log"
	"net/url"
	"regexp"
	"strconv"

	"github.com/dop251/goja"
)

func (c *Client) decipherURL(ctx context.Context, videoID string, cipher string) (string, error) {
	params, err := url.ParseQuery(cipher)
	if err != nil {
		return "", err
	}

	uri, err := url.Parse(params.Get("url"))
	if err != nil {
		return "", err
	}
	query := uri.Query()

	config, err := c.getPlayerConfig(ctx, videoID)
	if err != nil {
		return "", err
	}

	// decrypt s-parameter
	bs, err := config.decrypt([]byte(params.Get("s")))
	if err != nil {
		return "", err
	}
	query.Add(params.Get("sp"), string(bs))

	query, err = c.decryptNParam(ctx, config, query)
	if err != nil {
		return "", err
	}

	uri.RawQuery = query.Encode()

	return uri.String(), nil
}

// see https://github.com/kkdai/youtube/pull/244
func (c *Client) unThrottle(ctx context.Context, videoID string, urlString string) (string, error) {
	config, err := c.getPlayerConfig(ctx, videoID)
	if err != nil {
		return "", err
	}

	uri, err := url.Parse(urlString)
	if err != nil {
		return "", err
	}

	query, err := c.decryptNParam(ctx, config, uri.Query())
	if err != nil {
		return "", err
	}

	uri.RawQuery = query.Encode()
	return uri.String(), nil
}

func (c *Client) decryptNParam(ctx context.Context, config playerConfig, query url.Values) (url.Values, error) {
	// decrypt n-parameter
	nSig := query.Get("n")
	if nSig != "" {
		nDecoded, err := config.decodeNsig(nSig)
		if err != nil {
			return nil, fmt.Errorf("unable to decode nSig: %w", err)
		}
		query.Set("n", nDecoded)
	}

	if c.Debug {
		log.Printf("[nParam] n: %s; nDecoded: %s\nQuery: %v\n", nSig, query.Get("n"), query)
	}

	return query, nil
}

const (
	jsvarStr   = "[a-zA-Z_\\$][a-zA-Z_0-9]*"
	reverseStr = ":function\\(a\\)\\{" +
		"(?:return )?a\\.reverse\\(\\)" +
		"\\}"
	spliceStr = ":function\\(a,b\\)\\{" +
		"a\\.splice\\(0,b\\)" +
		"\\}"
	swapStr = ":function\\(a,b\\)\\{" +
		"var c=a\\[0\\];a\\[0\\]=a\\[b(?:%a\\.length)?\\];a\\[b(?:%a\\.length)?\\]=c(?:;return a)?" +
		"\\}"
)

var (
	nFunctionNameRegexp = regexp.MustCompile("\\.get\\(\"n\"\\)\\)&&\\(b=([a-zA-Z0-9$]{0,3})\\[(\\d+)\\](.+)\\|\\|([a-zA-Z0-9]{0,3})")
	actionsObjRegexp    = regexp.MustCompile(fmt.Sprintf(
		"var (%s)=\\{((?:(?:%s%s|%s%s|%s%s),?\\n?)+)\\};", jsvarStr, jsvarStr, swapStr, jsvarStr, spliceStr, jsvarStr, reverseStr))

	actionsFuncRegexp = regexp.MustCompile(fmt.Sprintf(
		"function(?: %s)?\\(a\\)\\{"+
			"a=a\\.split\\(\"\"\\);\\s*"+
			"((?:(?:a=)?%s\\.%s\\(a,\\d+\\);)+)"+
			"return a\\.join\\(\"\"\\)"+
			"\\}", jsvarStr, jsvarStr, jsvarStr))

	reverseRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, reverseStr))
	spliceRegexp  = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, spliceStr))
	swapRegexp    = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, swapStr))
)

func (config playerConfig) decodeNsig(encoded string) (string, error) {
	fBody, err := config.getNFunction()
	if err != nil {
		return "", err
	}

	return evalJavascript(fBody, encoded)
}

func evalJavascript(jsFunction, arg string) (string, error) {
	const myName = "myFunction"

	vm := goja.New()
	_, err := vm.RunString(myName + "=" + jsFunction)
	if err != nil {
		return "", err
	}

	var output func(string) string
	err = vm.ExportTo(vm.Get(myName), &output)
	if err != nil {
		return "", err
	}

	return output(arg), nil
}

func (config playerConfig) getNFunction() (string, error) {
	nameResult := nFunctionNameRegexp.FindSubmatch(config)
	if len(nameResult) == 0 {
		log.Println("playerConfig:", string(config))
		return "", errors.New("unable to extract n-function name")
	}

	var name string
	if idx, _ := strconv.Atoi(string(nameResult[2])); idx == 0 {
		name = string(nameResult[4])
	} else {
		name = string(nameResult[1])
	}

	// find the beginning of the function
	def := []byte(name + "=function(")
	start := bytes.Index(config, def)
	if start < 1 {
		return "", fmt.Errorf("unable to extract n-function body: looking for '%s'", def)
	}

	// start after the first curly bracket
	pos := start + bytes.IndexByte(config[start:], '{') + 1

	// find the bracket closing the function
	for brackets := 1; brackets > 0; pos++ {
		switch config[pos] {
		case '{':
			brackets++
		case '}':
			brackets--
		}
	}

	return string(config[start:pos]), nil
}

func (config playerConfig) decrypt(cyphertext []byte) ([]byte, error) {
	operations, err := config.parseDecipherOps()
	if err != nil {
		return nil, err
	}

	// apply operations
	bs := []byte(cyphertext)
	for _, op := range operations {
		bs = op(bs)
	}

	return bs, nil
}

/*
	parses decipher operations from https://youtube.com/s/player/4fbb4d5b/player_ias.vflset/en_US/base.js

	var Mt={
	splice:function(a,b){a.splice(0,b)},
	reverse:function(a){a.reverse()},
	EQ:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}};

	a=a.split("");
	Mt.splice(a,3);
	Mt.EQ(a,39);
	Mt.splice(a,2);
	Mt.EQ(a,1);
	Mt.splice(a,1);
	Mt.EQ(a,35);
	Mt.EQ(a,51);
	Mt.splice(a,2);
	Mt.reverse(a,52);
	return a.join("")
*/
func (config playerConfig) parseDecipherOps() (operations []DecipherOperation, err error) {
	objResult := actionsObjRegexp.FindSubmatch(config)
	funcResult := actionsFuncRegexp.FindSubmatch(config)
	if len(objResult) < 3 || len(funcResult) < 2 {
		return nil, fmt.Errorf("error parsing signature tokens (#obj=%d, #func=%d)", len(objResult), len(funcResult))
	}

	obj := objResult[1]
	objBody := objResult[2]
	funcBody := funcResult[1]

	var reverseKey, spliceKey, swapKey string

	if result := reverseRegexp.FindSubmatch(objBody); len(result) > 1 {
		reverseKey = string(result[1])
	}
	if result := spliceRegexp.FindSubmatch(objBody); len(result) > 1 {
		spliceKey = string(result[1])
	}
	if result := swapRegexp.FindSubmatch(objBody); len(result) > 1 {
		swapKey = string(result[1])
	}

	regex, err := regexp.Compile(fmt.Sprintf("(?:a=)?%s\\.(%s|%s|%s)\\(a,(\\d+)\\)", regexp.QuoteMeta(string(obj)), regexp.QuoteMeta(reverseKey), regexp.QuoteMeta(spliceKey), regexp.QuoteMeta(swapKey)))
	if err != nil {
		return nil, err
	}

	var ops []DecipherOperation
	for _, s := range regex.FindAllSubmatch(funcBody, -1) {
		switch string(s[1]) {
		case reverseKey:
			ops = append(ops, reverseFunc)
		case swapKey:
			arg, _ := strconv.Atoi(string(s[2]))
			ops = append(ops, newSwapFunc(arg))
		case spliceKey:
			arg, _ := strconv.Atoi(string(s[2]))
			ops = append(ops, newSpliceFunc(arg))
		}
	}
	return ops, nil
}

func (config playerConfig) getSignatureTimestamp() (string, error) {
	result := signatureRegexp.FindSubmatch(config)
	if result == nil {
		return "", ErrSignatureTimestampNotFound
	}

	return string(result[1]), nil
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"log"
	"strconv"
	"time"

	"github.com/brotherlogic/goserver"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildmaster/proto"
	pbs "github.com/brotherlogic/gobuildslave/proto"
)

const (
	intentWait = 1000
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	config  *pb.Config
	serving bool
}

type mainChecker struct{}

func getIP(servertype, servername string) (string, int) {
	conn, _ := grpc.Dial("192.168.86.34:50055", grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	r, err := registry.ListAllServices(context.Background(), &pbd.Empty{})
	if err != nil {
		return "", -1
	}
	for _, s := range r.Services {
		log.Printf("Does %v = %v and %v = %v?", s.Name, servertype, s.Identifier, servername)
		if s.Name == servertype && s.Identifier == servername {
			return s.Ip, int(s.Port)
		}
	}

	return "", -1
}

func (t *mainChecker) assess(server string) (*pbs.JobList, *pbs.Config) {
	list := &pbs.JobList{}
	conf := &pbs.Config{}

	log.Printf("Assessing server %v", server)
	ip, port := getIP("gobuildslave", server)
	conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(context.Background(), &pbs.Empty{})
	if err != nil {
		log.Printf("Err %v", err)
		return list, conf
	}

	r2, err := slave.GetConfig(context.Background(), &pbs.Empty{})
	if err != nil {
		log.Printf("Err %v", err)
		return list, conf
	}

	return r, r2
}

func runJob(job *pbs.JobSpec, server string) {
	log.Printf("RUNNING: %v on %v", job, server)
	ip, port := getIP("gobuildslave", server)
	conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()

	slave := pbs.NewGoBuildSlaveClient(conn)
	slave.Run(context.Background(), job)
	log.Printf("RUN COMMAND SENT %v", job)
}

func (t *mainChecker) discover() *pbd.ServiceList {
	ret := &pbd.ServiceList{}

	conn, _ := grpc.Dial("192.168.86.34:50055", grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	r, err := registry.ListAllServices(context.Background(), &pbd.Empty{})
	log.Printf("DISCOVERED: %v", r)
	if err == nil {
		for _, s := range r.Services {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	// Do nothing
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

func getConfig(c checker) *pb.Config {
	list, _ := getFleetStatus(c)
	config := &pb.Config{}

	for _, jlist := range list {
		for _, job := range jlist.Details {
			found := false
			for _, ij := range config.Intents {
				if job.Spec.Name == ij.Spec.Name {
					ij.Masters++
					found = true
				}
			}

			if !found {
				config.Intents = append(config.Intents, &pb.Intent{Spec: &pbs.JobSpec{Name: job.Spec.Name}, Masters: 1})
			}
		}
	}

	return config
}

// MatchIntent tries to match the intent with the state of production
func (s Server) MatchIntent() {
	//for s.serving {
	time.Sleep(intentWait)

	state := getConfig(&mainChecker{})
	diff := configDiff(s.config, state)
	joblist := runJobs(diff)
	log.Printf("FOUND %v from %v and %v", joblist, state, s.config)
	for _, job := range joblist {
		runJob(job, chooseServer(job, &mainChecker{}))
	}
	//}
}

func main() {
	config, err := loadConfig("config.pb")
	if err != nil {
		log.Fatalf("Fatal loading of config: %v", err)
	}
	log.Printf("READ: %v", config)

	var sync = flag.Bool("once", true, "One pass intent match")
	s := Server{&goserver.GoServer{}, config, true}

	flag.Parse()
	if *sync {
		s.MatchIntent()
	} else {
		s.Register = s
		s.PrepServer()
		s.RegisterServer("gobuildmaster", false)
		s.Serve()
	}
}
<commit_msg>Avoids running commands on an empty server. This closes #21<commit_after>package main

import (
	"context"
	"flag"
	"log"
	"strconv"
	"time"

	"github.com/brotherlogic/goserver"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildmaster/proto"
	pbs "github.com/brotherlogic/gobuildslave/proto"
)

const (
	intentWait = 1000
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	config  *pb.Config
	serving bool
}

type mainChecker struct{}

func getIP(servertype, servername string) (string, int) {
	conn, _ := grpc.Dial("192.168.86.34:50055", grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	r, err := registry.ListAllServices(context.Background(), &pbd.Empty{})
	if err != nil {
		return "", -1
	}
	for _, s := range r.Services {
		log.Printf("Does %v = %v and %v = %v?", s.Name, servertype, s.Identifier, servername)
		if s.Name == servertype && s.Identifier == servername {
			return s.Ip, int(s.Port)
		}
	}

	return "", -1
}

func (t *mainChecker) assess(server string) (*pbs.JobList, *pbs.Config) {
	list := &pbs.JobList{}
	conf := &pbs.Config{}

	log.Printf("Assessing server %v", server)
	ip, port := getIP("gobuildslave", server)
	conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(context.Background(), &pbs.Empty{})
	if err != nil {
		log.Printf("Err %v", err)
		return list, conf
	}

	r2, err := slave.GetConfig(context.Background(), &pbs.Empty{})
	if err != nil {
		log.Printf("Err %v", err)
		return list, conf
	}

	return r, r2
}

func runJob(job *pbs.JobSpec, server string) {
	log.Printf("RUNNING: %v on %v", job, server)
	if server != "" {
		ip, port := getIP("gobuildslave", server)
		conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
		defer conn.Close()

		slave := pbs.NewGoBuildSlaveClient(conn)
		slave.Run(context.Background(), job)
		log.Printf("RUN COMMAND SENT %v", job)
	}
}

func (t *mainChecker) discover() *pbd.ServiceList {
	ret := &pbd.ServiceList{}

	conn, _ := grpc.Dial("192.168.86.34:50055", grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	r, err := registry.ListAllServices(context.Background(), &pbd.Empty{})
	log.Printf("DISCOVERED: %v", r)
	if err == nil {
		for _, s := range r.Services {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	// Do nothing
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

func getConfig(c checker) *pb.Config {
	list, _ := getFleetStatus(c)
	config := &pb.Config{}

	for _, jlist := range list {
		for _, job := range jlist.Details {
			found := false
			for _, ij := range config.Intents {
				if job.Spec.Name == ij.Spec.Name {
					ij.Masters++
					found = true
				}
			}

			if !found {
				config.Intents = append(config.Intents, &pb.Intent{Spec: &pbs.JobSpec{Name: job.Spec.Name}, Masters: 1})
			}
		}
	}

	return config
}

// MatchIntent tries to match the intent with the state of production
func (s Server) MatchIntent() {
	//for s.serving {
	time.Sleep(intentWait)

	state := getConfig(&mainChecker{})
	diff := configDiff(s.config, state)
	joblist := runJobs(diff)
	log.Printf("FOUND %v from %v and %v", joblist, state, s.config)
	for _, job := range joblist {
		runJob(job, chooseServer(job, &mainChecker{}))
	}
	//}
}

func main() {
	config, err := loadConfig("config.pb")
	if err != nil {
		log.Fatalf("Fatal loading of config: %v", err)
	}
	log.Printf("READ: %v", config)

	var sync = flag.Bool("once", true, "One pass intent match")
	s := Server{&goserver.GoServer{}, config, true}

	flag.Parse()
	if *sync {
		s.MatchIntent()
	} else {
		s.Register = s
		s.PrepServer()
		s.RegisterServer("gobuildmaster", false)
		s.Serve()
	}
}
<|endoftext|>


<commit_before>package ss

import (
	"bytes"
	"encoding/json"
	"golang.org/x/net/websocket"
	"github.com/raz-varren/sacrificial-socket/log"
	"github.com/raz-varren/sacrificial-socket/tools"
	"sync"
)

//Socket represents a websocket connection
type Socket struct {
	l      *sync.RWMutex
	id     string
	ws     *websocket.Conn
	closed bool
	serv   *SocketServer
	roomsl *sync.RWMutex
	rooms  map[string]bool
}

const (
	idLen int = 32

	typeJSON string = "J"
	typeBin         = "B"
	typeStr         = "S"
)

var (
	idChars = []string{
		"0", "1", "2", "3", "4",
		"5", "6", "7", "8", "9",
		"A", "B", "C", "D", "E",
		"F", "G", "H", "I", "J",
		"K", "L", "M", "N", "O",
		"P", "Q", "R", "S", "T",
		"U", "V", "W", "X", "Y",
		"Z", "a", "b", "c", "d",
		"e", "f", "g", "h", "i",
		"j", "k", "l", "m", "n",
		"o", "p", "q", "r", "s",
		"t", "u", "v", "w", "x",
		"y", "z", "=", "_", "-",
		"#", ".",
	}

	idCharLen int = len(idChars) - 1
)

func newSocket(serv *SocketServer, ws *websocket.Conn) *Socket {
	buf := bytes.NewBuffer(nil)
	for i := 0; i < idLen; i++ {
		buf.WriteString(idChars[tools.RandomInt(0, idCharLen)])
	}
	s := &Socket{
		l:      &sync.RWMutex{},
		id:     buf.String(),
		ws:     ws,
		closed: false,
		serv:   serv,
		roomsl: &sync.RWMutex{},
		rooms:  make(map[string]bool),
	}
	serv.hub.addSocket(s)
	return s
}

func (s *Socket) receive(v interface{}) error {
	return websocket.Message.Receive(s.ws, v)
}

func (s *Socket) send(data interface{}) error {
	return websocket.Message.Send(s.ws, data)
}

func (s *Socket) InRoom(roomName string) bool {
	s.roomsl.RLock()
	defer s.roomsl.RUnlock()
	inRoom := s.rooms[roomName]
	return inRoom
}

//GetRooms returns a list of rooms that s is a member of
func (s *Socket) GetRooms() []string {
	s.roomsl.RLock()
	defer s.roomsl.RUnlock()

	var roomList []string
	for room, _ := range s.rooms {
		roomList = append(roomList, room)
	}
	return roomList
}

//Join adds s to the specified room. If the room does
//not exist, it will be created
func (s *Socket) Join(roomName string) {
	s.roomsl.Lock()
	defer s.roomsl.Unlock()
	s.serv.hub.joinRoom(&joinRequest{roomName, s})
	s.rooms[roomName] = true
}

//Leave removes s from the specified room. If s
//is not a member of the room, nothing will happen. If the room is
//empty upon removal of s, the room will be closed
func (s *Socket) Leave(roomName string) {
	s.roomsl.Lock()
	defer s.roomsl.Unlock()
	s.serv.hub.leaveRoom(&leaveRequest{roomName, s})
	delete(s.rooms, roomName)
}

//Roomcast dispatches an event to all Sockets in the specified room.
func (s *Socket) Roomcast(roomName, eventName string, data interface{}) {
	s.serv.hub.roomcast(&RoomMsg{roomName, eventName, data})
}

//Broadcast dispatches an event to all Sockets on the SocketServer.
func (s *Socket) Broadcast(eventName string, data interface{}) {
	s.serv.hub.broadcast(&BroadcastMsg{eventName, data})
}

//Emit dispatches an event to s.
func (s *Socket) Emit(eventName string, data interface{}) error {
	return s.send(emitData(eventName, data))
}

//ID returns the unique ID of s
func (s *Socket) ID() string {
	s.l.RLock()
	defer s.l.RUnlock()
	id := s.id
	return id
}

//emitData combines the eventName and data into a payload that is understood
//by the sac-sock protocol. It will return either a string or a []byte
func emitData(eventName string, data interface{}) interface{} {
	buf := bytes.NewBuffer(nil)
	buf.WriteString(eventName)
	buf.WriteByte(startOfHeaderByte)

	switch d := data.(type) {
	case string:
		buf.WriteString(typeStr)
		buf.WriteByte(startOfDataByte)
		buf.WriteString(d)
		return buf.String()

	case []byte:
		buf.WriteString(typeBin)
		buf.WriteByte(startOfDataByte)
		buf.Write(d)
		return buf.Bytes()

	default:
		buf.WriteString(typeJSON)
		buf.WriteByte(startOfDataByte)
		jsonData, err := json.Marshal(d)
		if err != nil {
			log.Err.Println(err)
		} else {
			buf.Write(jsonData)
		}
		return buf.String()
	}
}

//Close closes the Socket connection and removes the Socket
//from any rooms that it was a member of
func (s *Socket) Close() {
	s.l.Lock()
	isAlreadyClosed := s.closed
	s.closed = true
	s.l.Unlock()

	if isAlreadyClosed { //can't reclose the socket
		return
	}

	defer log.Info.Println(s.ID(), "disconnected")

	err := s.ws.Close()
	if err != nil {
		log.Err.Println(err)
	}

	rooms := s.GetRooms()

	for _, room := range rooms {
		s.Leave(room)
	}

	s.serv.l.RLock()
	event := s.serv.onDisconnectFunc
	s.serv.l.RUnlock()

	if event != nil {
		event(s)
	}

	s.serv.hub.removeSocket(s)
}
<commit_msg>documentation == helpful<commit_after>package ss

import (
	"bytes"
	"encoding/json"
	"golang.org/x/net/websocket"
	"github.com/raz-varren/sacrificial-socket/log"
	"github.com/raz-varren/sacrificial-socket/tools"
	"sync"
)

//Socket represents a websocket connection
type Socket struct {
	l      *sync.RWMutex
	id     string
	ws     *websocket.Conn
	closed bool
	serv   *SocketServer
	roomsl *sync.RWMutex
	rooms  map[string]bool
}

const (
	idLen int = 32

	typeJSON string = "J"
	typeBin         = "B"
	typeStr         = "S"
)

var (
	idChars = []string{
		"0", "1", "2", "3", "4",
		"5", "6", "7", "8", "9",
		"A", "B", "C", "D", "E",
		"F", "G", "H", "I", "J",
		"K", "L", "M", "N", "O",
		"P", "Q", "R", "S", "T",
		"U", "V", "W", "X", "Y",
		"Z", "a", "b", "c", "d",
		"e", "f", "g", "h", "i",
		"j", "k", "l", "m", "n",
		"o", "p", "q", "r", "s",
		"t", "u", "v", "w", "x",
		"y", "z", "=", "_", "-",
		"#", ".",
	}

	idCharLen int = len(idChars) - 1
)

func newSocket(serv *SocketServer, ws *websocket.Conn) *Socket {
	buf := bytes.NewBuffer(nil)
	for i := 0; i < idLen; i++ {
		buf.WriteString(idChars[tools.RandomInt(0, idCharLen)])
	}
	s := &Socket{
		l:      &sync.RWMutex{},
		id:     buf.String(),
		ws:     ws,
		closed: false,
		serv:   serv,
		roomsl: &sync.RWMutex{},
		rooms:  make(map[string]bool),
	}
	serv.hub.addSocket(s)
	return s
}

func (s *Socket) receive(v interface{}) error {
	return websocket.Message.Receive(s.ws, v)
}

func (s *Socket) send(data interface{}) error {
	return websocket.Message.Send(s.ws, data)
}

//InRoom returns true if s is currently a member of roomName
func (s *Socket) InRoom(roomName string) bool {
	s.roomsl.RLock()
	defer s.roomsl.RUnlock()
	inRoom := s.rooms[roomName]
	return inRoom
}

//GetRooms returns a list of rooms that s is a member of
func (s *Socket) GetRooms() []string {
	s.roomsl.RLock()
	defer s.roomsl.RUnlock()

	var roomList []string
	for room, _ := range s.rooms {
		roomList = append(roomList, room)
	}
	return roomList
}

//Join adds s to the specified room. If the room does
//not exist, it will be created
func (s *Socket) Join(roomName string) {
	s.roomsl.Lock()
	defer s.roomsl.Unlock()
	s.serv.hub.joinRoom(&joinRequest{roomName, s})
	s.rooms[roomName] = true
}

//Leave removes s from the specified room. If s
//is not a member of the room, nothing will happen. If the room is
//empty upon removal of s, the room will be closed
func (s *Socket) Leave(roomName string) {
	s.roomsl.Lock()
	defer s.roomsl.Unlock()
	s.serv.hub.leaveRoom(&leaveRequest{roomName, s})
	delete(s.rooms, roomName)
}

//Roomcast dispatches an event to all Sockets in the specified room.
func (s *Socket) Roomcast(roomName, eventName string, data interface{}) {
	s.serv.hub.roomcast(&RoomMsg{roomName, eventName, data})
}

//Broadcast dispatches an event to all Sockets on the SocketServer.
func (s *Socket) Broadcast(eventName string, data interface{}) {
	s.serv.hub.broadcast(&BroadcastMsg{eventName, data})
}

//Emit dispatches an event to s.
func (s *Socket) Emit(eventName string, data interface{}) error {
	return s.send(emitData(eventName, data))
}

//ID returns the unique ID of s
func (s *Socket) ID() string {
	s.l.RLock()
	defer s.l.RUnlock()
	id := s.id
	return id
}

//emitData combines the eventName and data into a payload that is understood
//by the sac-sock protocol. It will return either a string or a []byte
func emitData(eventName string, data interface{}) interface{} {
	buf := bytes.NewBuffer(nil)
	buf.WriteString(eventName)
	buf.WriteByte(startOfHeaderByte)

	switch d := data.(type) {
	case string:
		buf.WriteString(typeStr)
		buf.WriteByte(startOfDataByte)
		buf.WriteString(d)
		return buf.String()

	case []byte:
		buf.WriteString(typeBin)
		buf.WriteByte(startOfDataByte)
		buf.Write(d)
		return buf.Bytes()

	default:
		buf.WriteString(typeJSON)
		buf.WriteByte(startOfDataByte)
		jsonData, err := json.Marshal(d)
		if err != nil {
			log.Err.Println(err)
		} else {
			buf.Write(jsonData)
		}
		return buf.String()
	}
}

//Close closes the Socket connection and removes the Socket
//from any rooms that it was a member of
func (s *Socket) Close() {
	s.l.Lock()
	isAlreadyClosed := s.closed
	s.closed = true
	s.l.Unlock()

	if isAlreadyClosed { //can't reclose the socket
		return
	}

	defer log.Info.Println(s.ID(), "disconnected")

	err := s.ws.Close()
	if err != nil {
		log.Err.Println(err)
	}

	rooms := s.GetRooms()

	for _, room := range rooms {
		s.Leave(room)
	}

	s.serv.l.RLock()
	event := s.serv.onDisconnectFunc
	s.serv.l.RUnlock()

	if event != nil {
		event(s)
	}

	s.serv.hub.removeSocket(s)
}
<|endoftext|>


<commit_before>package statistics

import "testing"

func TestMean(t *testing.T) {

}<commit_msg>Tests for mean function<commit_after>package statistics

import "testing"
import "strconv"

func TestMean(t *testing.T) {
	list := []float64{1, 2, 3}
	if mean, _ := Mean(list); mean != 2 {
		t.Error("The mean must be 2 no: "+
			strconv.FormatFloat(mean, 'f', 2, 64),
		)
	}

	if mean, err := Mean([]float64{}); mean != 0.0 && err != nil {
		t.Error(err)
	}
}<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"time"

	"github.com/koron/go-arch"
)

var (
	errUnknownSource     = errors.New("unknown source")
	errSourceNotFound    = errors.New("source not found")
	errSourceNotModified = errors.New("source not modified")
)

type sourceType int

const (
	releaseSource sourceType = iota
	developSource
	canarySource
)

func toSourceType(s string) (sourceType, error) {
	switch s {
	case "release":
		return releaseSource, nil
	case "develop":
		return developSource, nil
	case "canary":
		return canarySource, nil
	}
	return 0, errUnknownSource
}

type source interface {
	download(outdir string, pivot time.Time) (path string, err error)
}

type directSource struct {
	url string
}

var _ source = (*directSource)(nil)

func (ds *directSource) download(d string, p time.Time) (string, error) {
	out, err := ds.outPath(d)
	if err != nil {
		return "", err
	}

	req, err := http.NewRequest("GET", ds.url, nil)
	if err != nil {
		return "", err
	}
	if !p.IsZero() {
		t := p.UTC().Format(http.TimeFormat)
		req.Header.Set("If-Modified-Since", t)
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	switch resp.StatusCode {
	case http.StatusOK:
		f, err := os.Create(out)
		if err != nil {
			return "", err
		}
		defer f.Close()
		if _, err := io.Copy(f, resp.Body); err != nil {
			return "", err
		}

	case http.StatusNotModified:
		return "", errSourceNotModified

	default:
		return "", fmt.Errorf("unexpected response: %s", resp.Status)
	}
	return out, nil
}

func (ds *directSource) outPath(d string) (string, error) {
	u, err := url.Parse(ds.url)
	if err != nil {
		return "", err
	}
	return filepath.Join(d, filepath.Base(u.Path)), nil
}

type githubSource struct {
	user        string
	project     string
	namePattern *regexp.Regexp
}

var _ source = (*githubSource)(nil)

func (gs *githubSource) download(d string, a time.Time) (string, error) {
	// TODO:
	return "", errors.New("not implmented yet")
}

var sources = map[sourceType]map[arch.CPU]source{
	releaseSource: {
		arch.X86: &githubSource{
			user:        "koron",
			project:     "vim-kaoriya",
			namePattern: regexp.MustCompile(`-win32-.*\.zip$`),
		},
		arch.AMD64: &githubSource{
			user:        "koron",
			project:     "vim-kaoriya",
			namePattern: regexp.MustCompile(`-win64-.*\.zip$`),
		},
	},
	developSource: {
		arch.X86: &directSource{
			url: "http://files.kaoriya.net/vim/vim74-kaoriya-win32.zip",
		},
		arch.AMD64: &directSource{
			url: "http://files.kaoriya.net/vim/vim74-kaoriya-win64.zip",
		},
	},
	canarySource: {
		arch.X86: &directSource{
			url: "http://files.kaoriya.net/vim/vim74-kaoriya-win32-test.zip",
		},
		arch.AMD64: &directSource{
			url: "http://files.kaoriya.net/vim/vim74-kaoriya-win64-test.zip",
		},
	},
}

func determineSource(st sourceType, cpu arch.CPU) (source, error) {
	m, ok := sources[st]
	if !ok {
		return nil, errSourceNotFound
	}
	s, ok := m[cpu]
	if !ok {
		return nil, errSourceNotFound
	}
	return s, nil
}
<commit_msg>github source<commit_after>package main

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"time"

	"github.com/koron/go-arch"
	"github.com/koron/go-github"
)

var (
	errUnknownSource     = errors.New("unknown source")
	errSourceNotFound    = errors.New("source not found")
	errSourceNotModified = errors.New("source not modified")

	errGithubNoRelease       = errors.New("absence of github release")
	errGithubNoAssets        = errors.New("no matched assets in github release")
	errGithubIncompleteAsset = errors.New("incomplete github asset")
)

type sourceType int

const (
	releaseSource sourceType = iota
	developSource
	canarySource
)

func toSourceType(s string) (sourceType, error) {
	switch s {
	case "release":
		return releaseSource, nil
	case "develop":
		return developSource, nil
	case "canary":
		return canarySource, nil
	}
	return 0, errUnknownSource
}

type source interface {
	// download downloads source file to outdir, return its path name.
	// if pivot is not zero, this checks changes of source from pivot.
	download(outdir string, pivot time.Time) (path string, err error)
}

type directSource struct {
	url string
}

var _ source = (*directSource)(nil)

func (ds *directSource) download(d string, p time.Time) (string, error) {
	return download(d, ds.url, p)
}

type githubSource struct {
	user    string
	project string
	namePat *regexp.Regexp
}

var _ source = (*githubSource)(nil)

func (gs *githubSource) download(d string, p time.Time) (string, error) {
	a, err := gs.fetchAsset()
	if err != nil {
		return "", err
	}
	if !p.IsZero() && p.After(a.UpdatedAt) {
		return "", errSourceNotModified
	}
	return download(a.DownloadURL, d, p)
}

func (gs *githubSource) fetchAsset() (*github.Asset, error) {
	r, err := github.Latest(gs.user, gs.project)
	if err != nil {
		return nil, err
	}
	if r.Draft || r.PreRelease {
		return nil, errGithubNoRelease
	}
	var t *github.Asset
	for _, a := range r.Assets {
		if gs.namePat.MatchString(a.Name) {
			t = &a
			break
		}
	}
	if t == nil {
		return nil, errGithubNoAssets
	}
	if t.State != "uploaded" {
		return nil, errGithubIncompleteAsset
	}
	return t, nil
}

var sources = map[sourceType]map[arch.CPU]source{
	releaseSource: {
		arch.X86: &githubSource{
			user:    "koron",
			project: "vim-kaoriya",
			namePat: regexp.MustCompile(`-win32-.*\.zip$`),
		},
		arch.AMD64: &githubSource{
			user:    "koron",
			project: "vim-kaoriya",
			namePat: regexp.MustCompile(`-win64-.*\.zip$`),
		},
	},
	developSource: {
		arch.X86: &directSource{
			url: "http://files.kaoriya.net/vim/vim74-kaoriya-win32.zip",
		},
		arch.AMD64: &directSource{
			url: "http://files.kaoriya.net/vim/vim74-kaoriya-win64.zip",
		},
	},
	canarySource: {
		arch.X86: &directSource{
			url: "http://files.kaoriya.net/vim/vim74-kaoriya-win32-test.zip",
		},
		arch.AMD64: &directSource{
			url: "http://files.kaoriya.net/vim/vim74-kaoriya-win64-test.zip",
		},
	},
}

func determineSource(st sourceType, cpu arch.CPU) (source, error) {
	m, ok := sources[st]
	if !ok {
		return nil, errSourceNotFound
	}
	s, ok := m[cpu]
	if !ok {
		return nil, errSourceNotFound
	}
	return s, nil
}

func downloadFilepath(inURL, outdir string) (string, error) {
	u, err := url.Parse(inURL)
	if err != nil {
		return "", err
	}
	return filepath.Join(outdir, filepath.Base(u.Path)), nil
}

func downloadAsFile(inURL, outPath string, pivot time.Time) error {
	req, err := http.NewRequest("GET", inURL, nil)
	if err != nil {
		return err
	}
	if !pivot.IsZero() {
		t := pivot.UTC().Format(http.TimeFormat)
		req.Header.Set("If-Modified-Since", t)
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	switch resp.StatusCode {
	case http.StatusOK:
		f, err := os.Create(outPath)
		if err != nil {
			return err
		}
		defer f.Close()
		if _, err := io.Copy(f, resp.Body); err != nil {
			return err
		}
	case http.StatusNotModified:
		return errSourceNotModified
	default:
		return fmt.Errorf("unexpected response: %s", resp.Status)
	}
	return nil
}

// download downloads URL and saves as a file to outdir, return its path name.
// if pivot is not zero, this checks changes of source after pivot.
func download(inURL, outdir string, pivot time.Time) (string, error) {
	path, err := downloadFilepath(inURL, outdir)
	if err != nil {
		return "", err
	}
	if err := downloadAsFile(inURL, path, pivot); err != nil {
		return "", err
	}
	return path, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Brian J. Downs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package spinner is a simple library to add a spinner to your application.
package spinner

import (
	"fmt"
	//"github.com/fatih/color"
	"strconv"
	"time"
)

// CharSets is a map of slices containing the available character sets
var CharSets = map[int][]string{
	1:  []string{"←", "↖", "↑", "↗", "→", "↘", "↓", "↙"},
	2:  []string{"▁", "▃", "▄", "▅", "▆", "▇", "█", "▇", "▆", "▅", "▄", "▃"},
	3:  []string{"▖", "▘", "▝", "▗"},
	4:  []string{"┤", "┘", "┴", "└", "├", "┌", "┬", "┐"},
	5:  []string{"◢", "◣", "◤", "◥"},
	6:  []string{"◰", "◳", "◲", "◱"},
	7:  []string{"◴", "◷", "◶", "◵"},
	8:  []string{"◐", "◓", "◑", "◒"},
	9:  []string{".", "o", "O", "@", "*"},
	10: []string{"|", "/", "-", "\\"},
	11: []string{"◡◡", "⊙⊙", "◠◠"},
	12: []string{"⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"},
	13: []string{">))'>", " >))'>", "  >))'>", "   >))'>", "    >))'>", "   <'((<", "  <'((<", " <'((<"},
	14: []string{"⠁", "⠂", "⠄", "⡀", "⢀", "⠠", "⠐", "⠈"},
	15: []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"},
	16: []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"},
	17: []string{"▉", "▊", "▋", "▌", "▍", "▎", "▏", "▎", "▍", "▌", "▋", "▊", "▉"},
	18: []string{"■", "□", "▪", "▫"},
	19: []string{"←", "↑", "→", "↓"},
	20: []string{"╫", "╪"},
	21: []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"},
	22: []string{"⇐", "⇖", "⇑", "⇗", "⇒", "⇘", "⇓", "⇙"},
	23: []string{"⠁", "⠁", "⠉", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠤", "⠄", "⠄", "⠤", "⠠", "⠠", "⠤", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋", "⠉", "⠈", "⠈"},
	24: []string{"⠈", "⠉", "⠋", "⠓", "⠒", "⠐", "⠐", "⠒", "⠖", "⠦", "⠤", "⠠", "⠠", "⠤", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋", "⠉", "⠈"},
	25: []string{"⠁", "⠉", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠤", "⠄", "⠄", "⠤", "⠴", "⠲", "⠒", "⠂", "⠂", "⠒", "⠚", "⠙", "⠉", "⠁"},
	26: []string{"⠋", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋"},
}

// StopChan is a bool typed channel used to stop the spinner
var StopChan = make(chan bool, 1)

// Spinner struct to hold the provided options
type Spinner struct {
	Chars  []string
	Delay  time.Duration
	Offset int
	Fmt    string
}

// New provides a pointer to an instance of Spinner with the supplied options
func New(c []string, t time.Duration) *Spinner {
	return &Spinner{
		Chars:  c,
		Delay:  t,
		Offset: len(c) - 1,
	}
}

// Start will start the spinner
func (s *Spinner) Start() {
	count := 0
	go func() {
		for {
			select {
			case <-StopChan:
				return
			default:
				fmt.Printf("\r%s ", s.Chars[count])
				time.Sleep(s.Delay)
				if count != s.Offset {
					count++
				} else {
					count = 0
				}
			}
		}
	}()
}

// Stop stops the spinner
func (s *Spinner) Stop() { StopChan <- true }

// Restart will stop and start the spinner
func (s *Spinner) Restart() {
	s.Stop()
	s.Start()
}

// UpdateSpeed is a convenience function to not have to make you
//create a new instance of the Spinner
func (s *Spinner) UpdateSpeed(delay time.Duration) { s.Delay = delay }

// UpdateCharSet will change the previously select character set to
// the provided one
func (s *Spinner) UpdateCharSet(chars []string) { s.Chars = chars }

// GenerateNumberSequence will generate a slice of integers at the
// provided length and convert them each to a string
func GenerateNumberSequence(length int) []string {
	numSeq := make([]string, 0)
	for i := 0; i < length; i++ {
		numSeq = append(numSeq, strconv.Itoa(i))
	}
	return numSeq
}
<commit_msg>doc update<commit_after>// Copyright 2014 Brian J. Downs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package spinner is a simple pacakge to add a spinner to your application.
package spinner

import (
	"fmt"
	//"github.com/fatih/color"
	"strconv"
	"time"
)

// CharSets is a map of slices containing the available character sets
var CharSets = map[int][]string{
	1:  []string{"←", "↖", "↑", "↗", "→", "↘", "↓", "↙"},
	2:  []string{"▁", "▃", "▄", "▅", "▆", "▇", "█", "▇", "▆", "▅", "▄", "▃"},
	3:  []string{"▖", "▘", "▝", "▗"},
	4:  []string{"┤", "┘", "┴", "└", "├", "┌", "┬", "┐"},
	5:  []string{"◢", "◣", "◤", "◥"},
	6:  []string{"◰", "◳", "◲", "◱"},
	7:  []string{"◴", "◷", "◶", "◵"},
	8:  []string{"◐", "◓", "◑", "◒"},
	9:  []string{".", "o", "O", "@", "*"},
	10: []string{"|", "/", "-", "\\"},
	11: []string{"◡◡", "⊙⊙", "◠◠"},
	12: []string{"⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"},
	13: []string{">))'>", " >))'>", "  >))'>", "   >))'>", "    >))'>", "   <'((<", "  <'((<", " <'((<"},
	14: []string{"⠁", "⠂", "⠄", "⡀", "⢀", "⠠", "⠐", "⠈"},
	15: []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"},
	16: []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"},
	17: []string{"▉", "▊", "▋", "▌", "▍", "▎", "▏", "▎", "▍", "▌", "▋", "▊", "▉"},
	18: []string{"■", "□", "▪", "▫"},
	19: []string{"←", "↑", "→", "↓"},
	20: []string{"╫", "╪"},
	21: []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"},
	22: []string{"⇐", "⇖", "⇑", "⇗", "⇒", "⇘", "⇓", "⇙"},
	23: []string{"⠁", "⠁", "⠉", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠤", "⠄", "⠄", "⠤", "⠠", "⠠", "⠤", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋", "⠉", "⠈", "⠈"},
	24: []string{"⠈", "⠉", "⠋", "⠓", "⠒", "⠐", "⠐", "⠒", "⠖", "⠦", "⠤", "⠠", "⠠", "⠤", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋", "⠉", "⠈"},
	25: []string{"⠁", "⠉", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠤", "⠄", "⠄", "⠤", "⠴", "⠲", "⠒", "⠂", "⠂", "⠒", "⠚", "⠙", "⠉", "⠁"},
	26: []string{"⠋", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋"},
}

// StopChan is a bool typed channel used to stop the spinner
var StopChan = make(chan bool, 1)

// Spinner struct to hold the provided options
type Spinner struct {
	Chars  []string
	Delay  time.Duration
	Offset int
	Fmt    string
}

// New provides a pointer to an instance of Spinner with the supplied options
func New(c []string, t time.Duration, d string) *Spinner {
	return &Spinner{
		Chars:  c,
		Delay:  t,
		Offset: len(c) - 1,
	}
}

// Start will start the spinner
func (s *Spinner) Start() {
	count := 0
	go func() {
		for {
			select {
			case <-StopChan:
				return
			default:
				fmt.Printf("\r%s ", s.Chars[count])
				time.Sleep(s.Delay)
				if count != s.Offset {
					count++
				} else {
					count = 0
				}
			}
		}
	}()
}

// Stop stops the spinner
func (s *Spinner) Stop() { StopChan <- true }

// Restart will stop and start the spinner
func (s *Spinner) Restart() {
	s.Stop()
	s.Start()
}

// Reverse will
func (s *Spinner) Reverse() {}

// UpdateSpeed is a convenience function to not have to make you
//create a new instance of the Spinner
func (s *Spinner) UpdateSpeed(delay time.Duration) { s.Delay = delay }

// UpdateCharSet will change the previously select character set to
// the provided one
func (s *Spinner) UpdateCharSet(chars []string) { s.Chars = chars }

// GenerateNumberSequence will generate a slice of integers at the
// provided length and convert them each to a string
func GenerateNumberSequence(length int) []string {
	numSeq := make([]string, 0)
	for i := 0; i < length; i++ {
		numSeq = append(numSeq, strconv.Itoa(i))
	}
	return numSeq
}
<|endoftext|>


<commit_before>package metadata

import (
	"encoding/json"
	"fmt"
)

func NewCollection(name string) *Collection {
	return &Collection{
		Name:    name,
		Indexes: make(map[string]*CollectionIndex),
	}
}

type Collection struct {
	ID   int64  `json:"_id,omitempty"`
	Name string `json:"name"`

	// NOTE: we reserve the "_" namespace for fields for our own data (created, etc.)
	// All the columns in this table
	Fields map[string]*CollectionField `json:"fields"`

	// map of name -> index
	Indexes map[string]*CollectionIndex `json:"indexes,omitempty"`
	// Link directly to primary index (for convenience)
	PrimaryIndex *CollectionIndex `json:"-"`

	ProvisionState ProvisionState `json:"provision_state"`
}

func (c *Collection) UnmarshalJSON(data []byte) error {
	type Alias Collection
	aux := &struct {
		*Alias
	}{
		Alias: (*Alias)(c),
	}
	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}

	for _, index := range c.Indexes {
		if index.Primary {
			if c.PrimaryIndex == nil {
				c.PrimaryIndex = index
			} else {
				return fmt.Errorf("Collections can only have one primary index")
			}
		}
	}
	if c.PrimaryIndex == nil {
		return fmt.Errorf("Collection %s missing primary index", c.Name)
	}

	return nil
}

func (c *Collection) GetField(nameParts []string) *CollectionField {
	field := c.Fields[nameParts[0]]

	for _, part := range nameParts[1:] {
		field = field.SubFields[part]
	}

	return field
}

func (c *Collection) Equal(o *Collection) bool {
	if c.Name != o.Name {
		return false
	}

	return true
}

func (c *Collection) ListIndexes() []string {
	indexes := make([]string, 0, len(c.Indexes))
	for name, _ := range c.Indexes {
		indexes = append(indexes, name)
	}
	return indexes
}

func (c *Collection) ValidateRecord(record map[string]interface{}) *ValidationResult {
	result := &ValidationResult{Fields: make(map[string]*ValidationResult)}
	// TODO: We need to check that we where given no more than the Fields we know about
	for fieldName, field := range c.Fields {
		if v, ok := record[fieldName]; ok {
			record[fieldName], result.Fields[fieldName] = field.Normalize(v)
		} else {
			if field.NotNull && field.Default == nil {
				result.Fields[fieldName] = &ValidationResult{
					Error: fmt.Sprintf("Missing required field %s %v", fieldName, field.Default),
				}
			}
			// TODO: include an empty result? Not sure if an empty one is any good (also-- check for subfields?)
		}
	}
	return result
}

// TODO: underlying datasources should know how to do this-- us doing it shouldn't
// be necessary
// For updates we want to ensure that all fields we where given are (1) valid and (2) are ones we know about
func (c *Collection) ValidateRecordUpdate(record map[string]interface{}) *ValidationResult {
	result := &ValidationResult{Fields: make(map[string]*ValidationResult)}
	for fieldName, field := range c.Fields {
		if v, ok := record[fieldName]; ok {
			record[fieldName], result.Fields[fieldName] = field.Normalize(v)
		}
	}
	return result
}
<commit_msg>Fix for validation of new records with a "serial" type<commit_after>package metadata

import (
	"encoding/json"
	"fmt"

	"github.com/jacksontj/dataman/src/datamantype"
)

func NewCollection(name string) *Collection {
	return &Collection{
		Name:    name,
		Indexes: make(map[string]*CollectionIndex),
	}
}

type Collection struct {
	ID   int64  `json:"_id,omitempty"`
	Name string `json:"name"`

	// NOTE: we reserve the "_" namespace for fields for our own data (created, etc.)
	// All the columns in this table
	Fields map[string]*CollectionField `json:"fields"`

	// map of name -> index
	Indexes map[string]*CollectionIndex `json:"indexes,omitempty"`
	// Link directly to primary index (for convenience)
	PrimaryIndex *CollectionIndex `json:"-"`

	ProvisionState ProvisionState `json:"provision_state"`
}

func (c *Collection) UnmarshalJSON(data []byte) error {
	type Alias Collection
	aux := &struct {
		*Alias
	}{
		Alias: (*Alias)(c),
	}
	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}

	for _, index := range c.Indexes {
		if index.Primary {
			if c.PrimaryIndex == nil {
				c.PrimaryIndex = index
			} else {
				return fmt.Errorf("Collections can only have one primary index")
			}
		}
	}
	if c.PrimaryIndex == nil {
		return fmt.Errorf("Collection %s missing primary index", c.Name)
	}

	return nil
}

func (c *Collection) GetField(nameParts []string) *CollectionField {
	field := c.Fields[nameParts[0]]

	for _, part := range nameParts[1:] {
		field = field.SubFields[part]
	}

	return field
}

func (c *Collection) Equal(o *Collection) bool {
	if c.Name != o.Name {
		return false
	}

	return true
}

func (c *Collection) ListIndexes() []string {
	indexes := make([]string, 0, len(c.Indexes))
	for name, _ := range c.Indexes {
		indexes = append(indexes, name)
	}
	return indexes
}

func (c *Collection) ValidateRecord(record map[string]interface{}) *ValidationResult {
	result := &ValidationResult{Fields: make(map[string]*ValidationResult)}
	// TODO: We need to check that we where given no more than the Fields we know about
	for fieldName, field := range c.Fields {
		switch field.FieldType.DatamanType {
		case datamantype.Serial:
			// TODO: check the serial type
		default:
			if v, ok := record[fieldName]; ok {
				record[fieldName], result.Fields[fieldName] = field.Normalize(v)
			} else {
				if field.NotNull && field.Default == nil {
					result.Fields[fieldName] = &ValidationResult{
						Error: fmt.Sprintf("Missing required field %s %v", fieldName, field.Default),
					}
				}
				// TODO: include an empty result? Not sure if an empty one is any good (also-- check for subfields?)
			}
		}
	}
	return result
}

// TODO: underlying datasources should know how to do this-- us doing it shouldn't
// be necessary
// For updates we want to ensure that all fields we where given are (1) valid and (2) are ones we know about
func (c *Collection) ValidateRecordUpdate(record map[string]interface{}) *ValidationResult {
	result := &ValidationResult{Fields: make(map[string]*ValidationResult)}
	for fieldName, field := range c.Fields {
		if v, ok := record[fieldName]; ok {
			record[fieldName], result.Fields[fieldName] = field.Normalize(v)
		}
	}
	return result
}
<|endoftext|>