<commit_before>package hashring

import (
	"crypto/md5"
	"fmt"
	"math"
	"sort"
)

type HashKey uint32
type HashKeyOrder []HashKey

func (h HashKeyOrder) Len() int           { return len(h) }
func (h HashKeyOrder) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
func (h HashKeyOrder) Less(i, j int) bool { return h[i] < h[j] }

type HashRing struct {
	ring       map[HashKey]string
	sortedKeys []HashKey
	nodes      []string
	weights    map[string]int
}

func New(nodes []string) *HashRing {
	hashRing := &HashRing{
		ring:       make(map[HashKey]string),
		sortedKeys: make([]HashKey, 0),
		nodes:      nodes,
		weights:    make(map[string]int),
	}
	hashRing.generateCircle()
	return hashRing
}

func NewWithWeights(weights map[string]int) *HashRing {
	nodes := make([]string, 0, len(weights))
	for node, _ := range weights {
		nodes = append(nodes, node)
	}
	hashRing := &HashRing{
		ring:       make(map[HashKey]string),
		sortedKeys: make([]HashKey, 0),
		nodes:      nodes,
		weights:    weights,
	}
	hashRing.generateCircle()
	return hashRing
}

func (h *HashRing) generateCircle() {
	totalWeight := 0
	for _, node := range h.nodes {
		if weight, ok := h.weights[node]; ok {
			totalWeight += weight
		} else {
			totalWeight += 1
		}
	}

	for _, node := range h.nodes {
		weight := 1

		if _, ok := h.weights[node]; ok {
			weight = h.weights[node]
		}

		factor := math.Floor(float64(40*len(h.nodes)*weight) / float64(totalWeight))

		for j := 0; j < int(factor); j++ {
			nodeKey := fmt.Sprintf("%s-%d", node, j)
			bKey := hashDigest(nodeKey)

			for i := 0; i < 3; i++ {
				key := hashVal(bKey[i*4 : i*4+4])
				h.ring[key] = node
				h.sortedKeys = append(h.sortedKeys, key)
			}
		}
	}

	sort.Sort(HashKeyOrder(h.sortedKeys))
}

func (h *HashRing) GetNode(stringKey string) (node string, ok bool) {
	pos, ok := h.GetNodePos(stringKey)
	if !ok {
		return "", false
	}
	return h.ring[h.sortedKeys[pos]], true
}

func (h *HashRing) GetNodePos(stringKey string) (pos int, ok bool) {
	if len(h.ring) == 0 {
		return 0, false
	}

	key := h.GenKey(stringKey)

	nodes := h.sortedKeys
	pos = sort.Search(len(nodes), func(i int) bool { return nodes[i] > key })

	if pos == len(nodes) {
		// Wrap the search, should return first node
		return 0, true
	} else {
		return pos, true
	}
}

func (h *HashRing) GenKey(key string) HashKey {
	bKey := hashDigest(key)
	return hashVal(bKey[0:4])
}

func (h *HashRing) GetNodes(stringKey string, size int) (nodes []string, ok bool) {
	pos, ok := h.GetNodePos(stringKey)
	if !ok {
		return []string{}, false
	}

	if size > len(h.nodes) {
		return []string{}, false
	}

	returnedValues := make(map[string]bool, size)
	//mergedSortedKeys := append(h.sortedKeys[pos:], h.sortedKeys[:pos]...)
	resultSlice := make([]string, 0, size)

	for i := pos; i < pos+len(h.sortedKeys); i++ {
		key := h.sortedKeys[i%len(h.sortedKeys)]
		val := h.ring[key]
		if !returnedValues[val] {
			returnedValues[val] = true
			resultSlice = append(resultSlice, val)
		}
		if len(returnedValues) == size {
			break
		}
	}

	return resultSlice, len(resultSlice) == size
}

func (h *HashRing) AddNode(node string) *HashRing {
	return h.AddWeightedNode(node, 1)
}

func (h *HashRing) AddWeightedNode(node string, weight int) *HashRing {
	if weight <= 0 {
		return h
	}

	for _, eNode := range h.nodes {
		if eNode == node {
			return h
		}
	}

	nodes := make([]string, len(h.nodes), len(h.nodes)+1)
	copy(nodes, h.nodes)
	nodes = append(nodes, node)

	weights := make(map[string]int)
	for eNode, eWeight := range h.weights {
		weights[eNode] = eWeight
	}
	weights[node] = weight

	hashRing := &HashRing{
		ring:       make(map[HashKey]string),
		sortedKeys: make([]HashKey, 0),
		nodes:      nodes,
		weights:    weights,
	}
	hashRing.generateCircle()
	return hashRing
}

func (h *HashRing) UpdateWeightedNode(node string, weight int) *HashRing {
	if weight <= 0 {
		return h
	}

	/* node is not need to update for node is not existed or weight is not changed */
	if oldWeight, ok := h.weights[node]; (!ok) || (ok && oldWeight == weight) {
		return h
	}

	nodes := make([]string, len(h.nodes), len(h.nodes))
	copy(nodes, h.nodes)

	weights := make(map[string]int)
	for eNode, eWeight := range h.weights {
		weights[eNode] = eWeight
	}
	weights[node] = weight

	hashRing := &HashRing{
		ring:       make(map[HashKey]string),
		sortedKeys: make([]HashKey, 0),
		nodes:      nodes,
		weights:    weights,
	}
	hashRing.generateCircle()
	return hashRing
}
func (h *HashRing) RemoveNode(node string) *HashRing {
	nodes := make([]string, 0)
	for _, eNode := range h.nodes {
		if eNode != node {
			nodes = append(nodes, eNode)
		}
	}

	/* if node isn't exist in hashring, don't refresh hashring */
	if len(nodes) == len(h.nodes) {
		return h
	}

	weights := make(map[string]int)
	for eNode, eWeight := range h.weights {
		if eNode != node {
			weights[eNode] = eWeight
		}
	}

	hashRing := &HashRing{
		ring:       make(map[HashKey]string),
		sortedKeys: make([]HashKey, 0),
		nodes:      nodes,
		weights:    weights,
	}
	hashRing.generateCircle()
	return hashRing
}

func hashVal(bKey []byte) HashKey {
	return ((HashKey(bKey[3]) << 24) |
		(HashKey(bKey[2]) << 16) |
		(HashKey(bKey[1]) << 8) |
		(HashKey(bKey[0])))
}

func hashDigest(key string) []byte {
	m := md5.New()
	m.Write([]byte(key))
	return m.Sum(nil)
}
<commit_msg>add update hashring with weights(when weights isn't changed,keep the old hashring)<commit_after>package hashring

import (
	"crypto/md5"
	"fmt"
	"math"
	"sort"
)

type HashKey uint32
type HashKeyOrder []HashKey

func (h HashKeyOrder) Len() int           { return len(h) }
func (h HashKeyOrder) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
func (h HashKeyOrder) Less(i, j int) bool { return h[i] < h[j] }

type HashRing struct {
	ring       map[HashKey]string
	sortedKeys []HashKey
	nodes      []string
	weights    map[string]int
}

func New(nodes []string) *HashRing {
	hashRing := &HashRing{
		ring:       make(map[HashKey]string),
		sortedKeys: make([]HashKey, 0),
		nodes:      nodes,
		weights:    make(map[string]int),
	}
	hashRing.generateCircle()
	return hashRing
}

func NewWithWeights(weights map[string]int) *HashRing {
	nodes := make([]string, 0, len(weights))
	for node, _ := range weights {
		nodes = append(nodes, node)
	}
	hashRing := &HashRing{
		ring:       make(map[HashKey]string),
		sortedKeys: make([]HashKey, 0),
		nodes:      nodes,
		weights:    weights,
	}
	hashRing.generateCircle()
	return hashRing
}

func (h *HashRing) UpdateWithWeights(weights map[string]int) {
	nodesChgFlg := false
	if len(weights) != len(h.weights) {
		nodesChgFlg = true
	} else {
		for node, newWeight := range weights {
			oldWeight, ok := h.weights[node]
			if !ok || oldWeight != newWeight {
				nodesChgFlg = true
				break
			}
		}
	}

	if nodesChgFlg {
		newhring := NewWithWeights(weights)
		h.weights = newhring.weights
		h.nodes = newhring.nodes
		h.ring = newhring.ring
		h.sortedKeys = newhring.sortedKeys
	}
}

func (h *HashRing) generateCircle() {
	totalWeight := 0
	for _, node := range h.nodes {
		if weight, ok := h.weights[node]; ok {
			totalWeight += weight
		} else {
			totalWeight += 1
		}
	}

	for _, node := range h.nodes {
		weight := 1

		if _, ok := h.weights[node]; ok {
			weight = h.weights[node]
		}

		factor := math.Floor(float64(40*len(h.nodes)*weight) / float64(totalWeight))

		for j := 0; j < int(factor); j++ {
			nodeKey := fmt.Sprintf("%s-%d", node, j)
			bKey := hashDigest(nodeKey)

			for i := 0; i < 3; i++ {
				key := hashVal(bKey[i*4 : i*4+4])
				h.ring[key] = node
				h.sortedKeys = append(h.sortedKeys, key)
			}
		}
	}

	sort.Sort(HashKeyOrder(h.sortedKeys))
}

func (h *HashRing) GetNode(stringKey string) (node string, ok bool) {
	pos, ok := h.GetNodePos(stringKey)
	if !ok {
		return "", false
	}
	return h.ring[h.sortedKeys[pos]], true
}

func (h *HashRing) GetNodePos(stringKey string) (pos int, ok bool) {
	if len(h.ring) == 0 {
		return 0, false
	}

	key := h.GenKey(stringKey)

	nodes := h.sortedKeys
	pos = sort.Search(len(nodes), func(i int) bool { return nodes[i] > key })

	if pos == len(nodes) {
		// Wrap the search, should return first node
		return 0, true
	} else {
		return pos, true
	}
}

func (h *HashRing) GenKey(key string) HashKey {
	bKey := hashDigest(key)
	return hashVal(bKey[0:4])
}

func (h *HashRing) GetNodes(stringKey string, size int) (nodes []string, ok bool) {
	pos, ok := h.GetNodePos(stringKey)
	if !ok {
		return []string{}, false
	}

	if size > len(h.nodes) {
		return []string{}, false
	}

	returnedValues := make(map[string]bool, size)
	//mergedSortedKeys := append(h.sortedKeys[pos:], h.sortedKeys[:pos]...)
	resultSlice := make([]string, 0, size)

	for i := pos; i < pos+len(h.sortedKeys); i++ {
		key := h.sortedKeys[i%len(h.sortedKeys)]
		val := h.ring[key]
		if !returnedValues[val] {
			returnedValues[val] = true
			resultSlice = append(resultSlice, val)
		}
		if len(returnedValues) == size {
			break
		}
	}

	return resultSlice, len(resultSlice) == size
}

func (h *HashRing) AddNode(node string) *HashRing {
	return h.AddWeightedNode(node, 1)
}

func (h *HashRing) AddWeightedNode(node string, weight int) *HashRing {
	if weight <= 0 {
		return h
	}

	for _, eNode := range h.nodes {
		if eNode == node {
			return h
		}
	}

	nodes := make([]string, len(h.nodes), len(h.nodes)+1)
	copy(nodes, h.nodes)
	nodes = append(nodes, node)

	weights := make(map[string]int)
	for eNode, eWeight := range h.weights {
		weights[eNode] = eWeight
	}
	weights[node] = weight

	hashRing := &HashRing{
		ring:       make(map[HashKey]string),
		sortedKeys: make([]HashKey, 0),
		nodes:      nodes,
		weights:    weights,
	}
	hashRing.generateCircle()
	return hashRing
}

func (h *HashRing) UpdateWeightedNode(node string, weight int) *HashRing {
	if weight <= 0 {
		return h
	}

	/* node is not need to update for node is not existed or weight is not changed */
	if oldWeight, ok := h.weights[node]; (!ok) || (ok && oldWeight == weight) {
		return h
	}

	nodes := make([]string, len(h.nodes), len(h.nodes))
	copy(nodes, h.nodes)

	weights := make(map[string]int)
	for eNode, eWeight := range h.weights {
		weights[eNode] = eWeight
	}
	weights[node] = weight

	hashRing := &HashRing{
		ring:       make(map[HashKey]string),
		sortedKeys: make([]HashKey, 0),
		nodes:      nodes,
		weights:    weights,
	}
	hashRing.generateCircle()
	return hashRing
}
func (h *HashRing) RemoveNode(node string) *HashRing {
	nodes := make([]string, 0)
	for _, eNode := range h.nodes {
		if eNode != node {
			nodes = append(nodes, eNode)
		}
	}

	/* if node isn't exist in hashring, don't refresh hashring */
	if len(nodes) == len(h.nodes) {
		return h
	}

	weights := make(map[string]int)
	for eNode, eWeight := range h.weights {
		if eNode != node {
			weights[eNode] = eWeight
		}
	}

	hashRing := &HashRing{
		ring:       make(map[HashKey]string),
		sortedKeys: make([]HashKey, 0),
		nodes:      nodes,
		weights:    weights,
	}
	hashRing.generateCircle()
	return hashRing
}

func hashVal(bKey []byte) HashKey {
	return ((HashKey(bKey[3]) << 24) |
		(HashKey(bKey[2]) << 16) |
		(HashKey(bKey[1]) << 8) |
		(HashKey(bKey[0])))
}

func hashDigest(key string) []byte {
	m := md5.New()
	m.Write([]byte(key))
	return m.Sum(nil)
}
<|endoftext|>


<commit_before>package pitchfork

type PfDetail struct {
	Type        string
	DisplayName string
}

func (td *PfDetail) ToString() (out string) {
	out = td.Type + " " + td.DisplayName
	return
}

func DetailList() (details []PfDetail, err error) {
	q := "SELECT " +
		"type, " +
		"display_name " +
		"FROM member_detail_types " +
		"ORDER BY type"
	rows, err := DB.Query(q)

	if err != nil {
		return
	}

	defer rows.Close()

	for rows.Next() {
		var detail PfDetail

		err = rows.Scan(&detail.Type, &detail.DisplayName)
		if err != nil {
			details = nil
			return
		}

		details = append(details, detail)
	}
	return
}

func detail_new(ctx PfCtx, args []string) (err error) {
	type_name := args[0]
	type_descr := args[1]

	/* Validate name */
	type_name, err = Chk_ident("Detail Type Name", type_name)
	if err != nil {
		return
	}

	q := "INSERT INTO member_detail_types " +
		"(type, display_name) " +
		"VALUES ($1, $2)"
	err = DB.Exec(ctx,
		"Added new Member Detail Type: ($1, $2)",
		1, q,
		type_name, type_descr)
	return
}

func detail_list(ctx PfCtx, args []string) (err error) {
	details, err := DetailList()
	if err != nil {
		return
	}

	ctx.OutLn("Detail\tDescription\n")

	for _, detail := range details {
		ctx.OutLn("%s", detail.ToString())
	}

	return
}

func detail_menu(ctx PfCtx, args []string) (err error) {
	menu := NewPfMenu([]PfMEntry{
		{"list", detail_list, 0, -1, nil, PERM_USER, "List details types"},
		{"new", detail_new, 2, 2, []string{"type_name", "description"}, PERM_SYS_ADMIN, "Add a new detail type"},
	})

	err = ctx.Menu(args, menu)
	return
}
<commit_msg>Add DetailType() + DetailCheck() to be able to filter detail types better before passing them to SQL<commit_after>package pitchfork

import (
	"errors"
	"strings"
)

type PfDetail struct {
	Type        string
	DisplayName string
}

func (td *PfDetail) ToString() (out string) {
	out = td.Type + " " + td.DisplayName
	return
}

func DetailType(detail string) (out string) {
	out = detail

	idx := strings.Index(detail, " ")
	if idx != -1 {
		out = detail[:idx]
	}

	return
}

func DetailCheck(detail string) (err error) {
	/* Verify that detail is a valid detail */
	details, err := DetailList()
	if err != nil {
		return
	}

	for _, d := range details {
		if d.Type == detail {
			err = nil
			return
		}
	}

	err = errors.New("Invalid detail type")
	return
}

func DetailList() (details []PfDetail, err error) {
	q := "SELECT " +
		"type, " +
		"display_name " +
		"FROM member_detail_types " +
		"ORDER BY type"
	rows, err := DB.Query(q)

	if err != nil {
		return
	}

	defer rows.Close()

	for rows.Next() {
		var detail PfDetail

		err = rows.Scan(&detail.Type, &detail.DisplayName)
		if err != nil {
			details = nil
			return
		}

		details = append(details, detail)
	}
	return
}

func detail_new(ctx PfCtx, args []string) (err error) {
	type_name := args[0]
	type_descr := args[1]

	/* Validate name */
	type_name, err = Chk_ident("Detail Type Name", type_name)
	if err != nil {
		return
	}

	q := "INSERT INTO member_detail_types " +
		"(type, display_name) " +
		"VALUES ($1, $2)"
	err = DB.Exec(ctx,
		"Added new Member Detail Type: ($1, $2)",
		1, q,
		type_name, type_descr)
	return
}

func detail_list(ctx PfCtx, args []string) (err error) {
	details, err := DetailList()
	if err != nil {
		return
	}

	ctx.OutLn("Detail\tDescription\n")

	for _, detail := range details {
		ctx.OutLn("%s", detail.ToString())
	}

	return
}

func detail_menu(ctx PfCtx, args []string) (err error) {
	menu := NewPfMenu([]PfMEntry{
		{"list", detail_list, 0, -1, nil, PERM_USER, "List details types"},
		{"new", detail_new, 2, 2, []string{"type_name", "description"}, PERM_SYS_ADMIN, "Add a new detail type"},
	})

	err = ctx.Menu(args, menu)
	return
}
<|endoftext|>


<commit_before>package transformer

import (
	"github.com/viant/dsc"
	"time"
)

const (
	//StatusTaskNotRunning  represent terminated task
	StatusTaskNotRunning = iota
	//StatusTaskRunning represents active copy task
	StatusTaskRunning
)

//BaseResponse represents a base response
type BaseResponse struct {
	Status    string
	Error     string
	StartTime time.Time
	EndTime   time.Time
}

//DatasetResource represents a datastore resource
type DatasetResource struct {
	DsConfig  *dsc.Config
	Table     string
	PkColumns []string
	Columns   []string
	SQL       string
}

//AsTableDescription converts data resource as table descriptor
func (r *DatasetResource) AsTableDescription() *dsc.TableDescriptor {
	return &dsc.TableDescriptor{
		Table:     r.Table,
		Columns:   r.Columns,
		PkColumns: r.PkColumns,
	}
}

//TaskInfo represents processed record info
type TaskInfo struct {
	Status             string
	StatusCode         int32
	SkippedRecordCount int
	EmptyRecordCount   int
	RecordCount        int
}

//CopyRequest represents a copy request
type CopyRequest struct {
	BatchSize   int
	InsertMode  bool
	Source      *DatasetResource
	Destination *DatasetResource
	Transformer string
}

//CopyResponse represents a copy response
type CopyResponse struct {
	*BaseResponse
	*TaskInfo
}

//TaskListRequest represents a task list request
type TaskListRequest struct {
	Table string
}

//Task represents a task
type Task struct {
	ID         string
	Status     string
	StatusCode int32
	Table      string
	Request    interface{}
	*BaseResponse
	*TaskInfo
}

func (t *Task) Expired(currentTime time.Time) bool {
	if !t.EndTime.IsZero() {
		return currentTime.Sub(t.EndTime) > time.Hour
	}
	return false
}

//TaskListResponse represents task list response
type TaskListResponse struct {
	Status string
	Tasks  []*Task
}

//KillTaskRequest represents kill task
type KillTaskRequest struct {
	ID string
}

//KillTaskResponse represents kill task response
type KillTaskResponse struct {
	*BaseResponse
	Task *Task
}
<commit_msg>addessed gocyclo  recomendation<commit_after>package transformer

import (
	"github.com/viant/dsc"
	"time"
)

const (
	//StatusTaskNotRunning  represent terminated task
	StatusTaskNotRunning = iota
	//StatusTaskRunning represents active copy task
	StatusTaskRunning
)

//BaseResponse represents a base response
type BaseResponse struct {
	Status    string
	Error     string
	StartTime time.Time
	EndTime   time.Time
}

//DatasetResource represents a datastore resource
type DatasetResource struct {
	DsConfig  *dsc.Config
	Table     string
	PkColumns []string
	Columns   []string
	SQL       string
}

//AsTableDescription converts data resource as table descriptor
func (r *DatasetResource) AsTableDescription() *dsc.TableDescriptor {
	return &dsc.TableDescriptor{
		Table:     r.Table,
		Columns:   r.Columns,
		PkColumns: r.PkColumns,
	}
}

//TaskInfo represents processed record info
type TaskInfo struct {
	Status             string
	StatusCode         int32
	SkippedRecordCount int
	EmptyRecordCount   int
	RecordCount        int
}

//CopyRequest represents a copy request
type CopyRequest struct {
	BatchSize   int
	InsertMode  bool
	Source      *DatasetResource
	Destination *DatasetResource
	Transformer string
}

//CopyResponse represents a copy response
type CopyResponse struct {
	*BaseResponse
	*TaskInfo
}

//TaskListRequest represents a task list request
type TaskListRequest struct {
	Table string
}

//Task represents a task
type Task struct {
	ID         string
	Status     string
	StatusCode int32
	Table      string
	Request    interface{}
	*BaseResponse
	*TaskInfo
}

//Expired returns true if task expired
func (t *Task) Expired(currentTime time.Time) bool {
	if !t.EndTime.IsZero() {
		return currentTime.Sub(t.EndTime) > time.Hour
	}
	return false
}

//TaskListResponse represents task list response
type TaskListResponse struct {
	Status string
	Tasks  []*Task
}

//KillTaskRequest represents kill task
type KillTaskRequest struct {
	ID string
}

//KillTaskResponse represents kill task response
type KillTaskResponse struct {
	*BaseResponse
	Task *Task
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"strings"
	"testing"
)

func testcmd(cmd string) (string, error) {
	switch {
	case cmd == "sudo ls -a1F /mnt/sda1/var/lib/docker/vfs/dir":
		return currenttest.vs.ls(), nil
	case cmd == "docker ps -aq --no-trunc":
		return "", nil
	case strings.HasPrefix(cmd, "docker inspect -f '{{ .Name }},{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' "):
		return "", nil
	case strings.HasPrefix(cmd, "sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):
		deleted := cmd[len("sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):]
		deletions = append(deletions, deleted)
		return "", nil
	default:
		return fmt.Sprintf("test '%s'", cmd), errors.New("unknown command")
	}
}

type volspecs []string
type Test struct {
	title string
	vs    volspecs
	res   []int
}

func (vs volspecs) ls() string {
	if len(vs) == 0 {
		return ""
	}
	res := ""
	for i, spec := range vs {
		if strings.HasSuffix(spec, "/") {
			spec = spec[:len(spec)-1]
			res = res + spec + strings.Repeat(fmt.Sprintf("%d", i), 64-len(spec)) + "/\n"
		}
		if strings.HasSuffix(spec, "@") {
			mp := "." + strings.Replace(spec, ";", "###", -1)
			mp = strings.Replace(mp, "/", ",#,", -1)
			res = res + mp + "\n"
		}
	}
	return res
}

var deletions = []string{}
var tests = []Test{
	Test{"empty vfs", []string{}, []int{0, 0, 0, 0, 0}},
	Test{"two volumes", []string{"fa/", "fb/"}, []int{0, 0, 2, 2, 0}},
	Test{"Invalid (ill-formed) markers must be deleted", []string{"cainv/path/a@"}, []int{0, 0, 0, 0, -1}},
	Test{"Invalid (no readlink) markers must be deleted", []string{"ca;/path/a@", "cb;/path/b@"}, []int{0, 0, 0, 0, -2}},
}
var currenttest Test

// TestContainers test different vfs scenarios
func TestContainers(t *testing.T) {
	cmd = testcmd
	for i, test := range tests {
		currenttest = test
		deletions = []string{}
		main()
		tc := Containers()
		toc := OrphanedContainers()
		tv := Volumes()
		tov := OrphanedVolumes()
		tm := Markers()
		if len(tc) != test.res[0] {
			t.Errorf("Test %d: '%s' expected '%d' containers, got '%d'", i+1, test.title, test.res[0], len(tc))
		}
		if len(toc) != test.res[1] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned containers, got '%d'", i+1, test.title, test.res[1], len(toc))
		}
		if len(tv) != test.res[2] {
			t.Errorf("Test %d: '%s' expected '%d' volumes, got '%d'", i+1, test.title, test.res[2], len(tv))
		}
		if len(tov) != test.res[3] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned volumes, got '%d'", i+1, test.title, test.res[3], len(tov))
		}
		if nbmarkers(tm) != test.res[4] {
			t.Errorf("Test %d: '%s' expected '%d' markers, got '%d'", i+1, test.title, test.res[4], nbmarkers(tm))
		}
		fmt.Println("----------------")
	}
}

func nbmarkers(tm markers) int {
	res := len(tm)
	for _, d := range deletions {
		if strings.HasPrefix(d, ".") {
			res = res - 1
		}
	}
	return res
}
<commit_msg>gcl_test.go: implements readlink command, for non-existent link markers<commit_after>package main

import (
	"errors"
	"fmt"
	"strings"
	"testing"
)

func testcmd(cmd string) (string, error) {
	switch {
	case cmd == "sudo ls -a1F /mnt/sda1/var/lib/docker/vfs/dir":
		return currenttest.vs.ls(), nil
	case cmd == "docker ps -aq --no-trunc":
		return "", nil
	case strings.HasPrefix(cmd, "docker inspect -f '{{ .Name }},{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' "):
		return "", nil
	case strings.HasPrefix(cmd, "sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):
		deleted := cmd[len("sudo rm /mnt/sda1/var/lib/docker/vfs/dir/"):]
		deletions = append(deletions, deleted)
		return "", nil
	case strings.HasPrefix(cmd, "sudo readlink /mnt/sda1/var/lib/docker/vfs/dir/"):
		if strings.Contains(cmd, ",nonexistent") {
			return "", errors.New("non-existent linked folder")
		}
		return "", nil
	default:
		return fmt.Sprintf("test '%s'", cmd), errors.New("unknown command")
	}
}

type volspecs []string
type Test struct {
	title string
	vs    volspecs
	res   []int
}

func (vs volspecs) ls() string {
	if len(vs) == 0 {
		return ""
	}
	res := ""
	for i, spec := range vs {
		if strings.HasSuffix(spec, "/") {
			spec = spec[:len(spec)-1]
			res = res + spec + strings.Repeat(fmt.Sprintf("%d", i), 64-len(spec)) + "/\n"
		}
		if strings.HasSuffix(spec, "@") {
			mp := "." + strings.Replace(spec, ";", "###", -1)
			mp = strings.Replace(mp, "/", ",#,", -1)
			res = res + mp + "\n"
		}
	}
	return res
}

var deletions = []string{}
var tests = []Test{
	Test{"empty vfs", []string{}, []int{0, 0, 0, 0, 0}},
	Test{"two volumes", []string{"fa/", "fb/"}, []int{0, 0, 2, 2, 0}},
	Test{"Invalid (ill-formed) markers must be deleted", []string{"cainv/path/a@"}, []int{0, 0, 0, 0, -1}},
	Test{"Invalid (no readlink) markers must be deleted", []string{"ca;/path/nonexistenta@", "cb;/path/nonexistentb@"}, []int{0, 0, 0, 0, -2}},
}
var currenttest Test

// TestContainers test different vfs scenarios
func TestContainers(t *testing.T) {
	cmd = testcmd
	for i, test := range tests {
		currenttest = test
		deletions = []string{}
		main()
		tc := Containers()
		toc := OrphanedContainers()
		tv := Volumes()
		tov := OrphanedVolumes()
		tm := Markers()
		if len(tc) != test.res[0] {
			t.Errorf("Test %d: '%s' expected '%d' containers, got '%d'", i+1, test.title, test.res[0], len(tc))
		}
		if len(toc) != test.res[1] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned containers, got '%d'", i+1, test.title, test.res[1], len(toc))
		}
		if len(tv) != test.res[2] {
			t.Errorf("Test %d: '%s' expected '%d' volumes, got '%d'", i+1, test.title, test.res[2], len(tv))
		}
		if len(tov) != test.res[3] {
			t.Errorf("Test %d: '%s' expected '%d' orphaned volumes, got '%d'", i+1, test.title, test.res[3], len(tov))
		}
		if nbmarkers(tm) != test.res[4] {
			t.Errorf("Test %d: '%s' expected '%d' markers, got '%d'", i+1, test.title, test.res[4], nbmarkers(tm))
		}
		fmt.Println("----------------")
	}
}

func nbmarkers(tm markers) int {
	res := len(tm)
	for _, d := range deletions {
		if strings.HasPrefix(d, ".") {
			res = res - 1
		}
	}
	return res
}
<|endoftext|>


<commit_before>package lahman

import "fmt"

// struct that list the unique identifying information for a given player's stats
// for embedding into more complete structs: Batter, Pitcher, Fielder
type bio struct {
	ID     string
	Year   float64
	Stint  string
	Team   string
	League string
}

// String prints biographical information about a player.
func (p bio) String() string {
	return fmt.Sprintf("%s,%.0f,%s,%s,%s", p.ID, p.Year, p.Stint, p.Team, p.League)
}

// A Player is a struct that contains all batting and pitching stats for a player.
type Player struct {
	Bat BatStats
	Pit PitchStats
	bio
	master
	p, b bool // p is true if player pitched, b is true if player batted and did *not* pitch
}

// IsPosPitcher returns true if the player was normally a position player, but pitched at least once.
func (p Player) IsPosPitcher() bool {
	return p.b && p.p
}

// String prints a player's information
func (p Player) String() string {
	var name string
	if f, l := p.FirstName(), p.LastName(); f != "" && l != "" {
		name = fmt.Sprintf("%s %s,", f, l)
	}
	s := name + p.bio.String()
	if p.b {
		s += "\nBatting:  " + p.Bat.String()
	}
	if p.p {
		s += "\nPitching: " + p.Pit.String()
	}
	return s
}

// Master lists biographical data fromt he Master database
type master [24]string

//csvRead pulls data from a csv file into a Master object
func (mas master) csvRead(line []string) (csvReader, error) {
	m := master{}

	for i := 0; i < len(line); i++ {
		m[i] = line[i]
	}

	return m, nil
}

func (m master) FirstName() string { return m[13] }
func (m master) LastName() string  { return m[14] }
<commit_msg>tweak<commit_after>package lahman

import "fmt"

// struct that list the unique identifying information for a given player's stats
// for embedding into more complete structs: Batter, Pitcher, Fielder
type bio struct {
	ID     string
	Year   float64
	Stint  string
	Team   string
	League string
}

// String prints biographical information about a player.
func (p bio) String() string {
	return fmt.Sprintf("%s,%.0f,%s,%s,%s", p.ID, p.Year, p.Stint, p.Team, p.League)
}

// A Player is a struct that contains all batting and pitching stats for a player.
type Player struct {
	Bat BatStats
	Pit PitchStats
	bio
	master
	p, b bool // p is true if player pitched, b is true if player batted and did *not* pitch
}

// IsPosPitcher returns true if the player was normally a position player, but pitched at least once.
func (p Player) IsPosPitcher() bool {
	return p.b && p.p
}

// String prints a player's information
func (p Player) String() string {
	var name string
	if f, l := p.FirstName(), p.LastName(); f != "" && l != "" {
		name = fmt.Sprintf("%s %s,", f, l)
	}
	s := name + p.bio.String()
	if p.b {
		s += "\nBatting:  " + p.Bat.String()
	}
	if p.p {
		s += "\nPitching: " + p.Pit.String()
	}
	return s
}

// Master lists biographical data fromt he Master database
type master [24]string

//csvRead pulls data from a csv file into a Master object
func (mas master) csvRead(line []string) (csvReader, error) {
	m := master{}

	for i := 0; i < len(line); i++ {
		m[i] = line[i]
	}

	return m, nil
}

func (p master) FirstName() string { return m[13] }
func (p master) LastName() string  { return m[14] }
<|endoftext|>


<commit_before>// Copyright (c) 2013 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

import(`sort`)

// All moves.
func (p *Position) Moves(ply int) (moves []*Move) {
        for i, piece := range p.pieces {
                if piece != 0 && piece.color() == p.color {
                        moves = append(moves, p.possibleMoves(i, piece)...)
                }
        }
        moves = p.reorderMoves(moves, best[0][ply], killer[ply])
        Log("%d candidates for %s: %v\n", len(moves), C(p.color), moves)
        return
}

func (p *Position) Captures(ply int) (moves []*Move) {
        for i, piece := range p.pieces {
                if piece != 0 && piece.color() == p.color {
                        moves = append(moves, p.possibleCaptures(i, piece)...)
                }
        }
        if bestMove := best[0][ply]; bestMove != nil && bestMove.captured != 0 {
                moves = p.reorderCaptures(moves, bestMove)
        } else {
                sort.Sort(byScore{moves})
        }

        Log("%d capture candidates for %s: %v\n", len(moves), C(p.color), moves)
        return
}

// All moves for the piece in certain square. This might include illegal
// moves that cause check to the king.
func (p *Position) possibleMoves(square int, piece Piece) (moves []*Move) {
        targets := p.targets[square]

        for targets.IsNotEmpty() {
                target := targets.FirstSet()
                //
                // For regular moves each target square represents one possible
                // move. For pawn promotion, however, we have to generate four
                // possible moves, one for each promoted piece.
                //
                if !p.isPawnPromotion(piece, target) {
                        moves = append(moves, NewMove(p, square, target))
                } else {
                        for _,name := range([]int{ QUEEN, ROOK, BISHOP, KNIGHT }) {
                                candidate := NewMove(p, square, target).promote(name)
                                moves = append(moves, candidate)
                        }
                }
                targets.Clear(target)
        }
        return
}

// All capture moves for the piece in certain square. This might include
// illegal moves that cause check to the king.
func (p *Position) possibleCaptures(square int, piece Piece) (moves []*Move) {
        targets := p.targets[square]

        for targets.IsNotEmpty() {
                target := targets.FirstSet()
                capture := p.pieces[target]
                if capture != 0 {
                        if !p.isPawnPromotion(piece, target) {
                                moves = append(moves, NewMove(p, square, target))
                        } else {
                                for _,name := range([]int{ QUEEN, ROOK, BISHOP, KNIGHT }) {
                                        candidate := NewMove(p, square, target).promote(name)
                                        moves = append(moves, candidate)
                                }
                        }
                } else if p.enpassant != 0 && target == p.enpassant {
                        moves = append(moves, NewMove(p, square, target))
                }
                targets.Clear(target)
        }
        return
}

func (p *Position) reorderMoves(moves []*Move, bestMove *Move, goodMove [2]*Move) []*Move {
        var principal, killers, captures, promotions, remaining []*Move

        for _, move := range moves {
                if len(principal) == 0 && bestMove != nil && move.is(bestMove) {
                        principal = append(principal, move)
                } else if (goodMove[0] != nil && move.is(goodMove[0])) || (goodMove[1] != nil && move.is(goodMove[1])) {
                        killers = append(killers, move)
                } else if move.captured != 0 {
                        captures = append(captures, move)
                } else if move.promoted != 0 {
                        promotions = append(promotions, move)
                } else {
                        remaining = append(remaining, move)
                }
        }
        if len(killers) > 1 && killers[0] == goodMove[1] {
                killers[0], killers[1] = killers[1], killers[0]
        }

        sort.Sort(byScore{captures})
        sort.Sort(byScore{remaining})
        return append(append(append(append(append(principal, killers...), captures...), promotions...), remaining...))
}

func (p *Position) reorderCaptures(moves []*Move, bestMove *Move) []*Move {
        var principal, remaining []*Move

        for _, move := range moves {
                if len(principal) == 0 && move.is(bestMove) {
                        principal = append(principal, move)
                } else {
                        remaining = append(remaining, move)
                }
        }
        sort.Sort(byScore{remaining})
        return append(principal, remaining...)
}

// Sorting moves by their relative score based on piece/square for regular moves
// or least valuaeable attacker/most valueable victim for captures.
type byScore struct {
        moves     []*Move
}
func (her byScore) Len() int           { return len(her.moves)}
func (her byScore) Swap(i, j int)      { her.moves[i], her.moves[j] = her.moves[j], her.moves[i] }
func (her byScore) Less(i, j int) bool { return her.moves[i].score > her.moves[j].score }
<commit_msg>Killer moves follow captures and promotions<commit_after>// Copyright (c) 2013 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

import(`sort`)

// All moves.
func (p *Position) Moves(ply int) (moves []*Move) {
        for i, piece := range p.pieces {
                if piece != 0 && piece.color() == p.color {
                        moves = append(moves, p.possibleMoves(i, piece)...)
                }
        }
        moves = p.reorderMoves(moves, best[0][ply], killer[ply])
        Log("%d candidates for %s: %v\n", len(moves), C(p.color), moves)
        return
}

func (p *Position) Captures(ply int) (moves []*Move) {
        for i, piece := range p.pieces {
                if piece != 0 && piece.color() == p.color {
                        moves = append(moves, p.possibleCaptures(i, piece)...)
                }
        }
        if bestMove := best[0][ply]; bestMove != nil && bestMove.captured != 0 {
                moves = p.reorderCaptures(moves, bestMove)
        } else {
                sort.Sort(byScore{moves})
        }

        Log("%d capture candidates for %s: %v\n", len(moves), C(p.color), moves)
        return
}

// All moves for the piece in certain square. This might include illegal
// moves that cause check to the king.
func (p *Position) possibleMoves(square int, piece Piece) (moves []*Move) {
        targets := p.targets[square]

        for targets.IsNotEmpty() {
                target := targets.FirstSet()
                //
                // For regular moves each target square represents one possible
                // move. For pawn promotion, however, we have to generate four
                // possible moves, one for each promoted piece.
                //
                if !p.isPawnPromotion(piece, target) {
                        moves = append(moves, NewMove(p, square, target))
                } else {
                        for _,name := range([]int{ QUEEN, ROOK, BISHOP, KNIGHT }) {
                                candidate := NewMove(p, square, target).promote(name)
                                moves = append(moves, candidate)
                        }
                }
                targets.Clear(target)
        }
        return
}

// All capture moves for the piece in certain square. This might include
// illegal moves that cause check to the king.
func (p *Position) possibleCaptures(square int, piece Piece) (moves []*Move) {
        targets := p.targets[square]

        for targets.IsNotEmpty() {
                target := targets.FirstSet()
                capture := p.pieces[target]
                if capture != 0 {
                        if !p.isPawnPromotion(piece, target) {
                                moves = append(moves, NewMove(p, square, target))
                        } else {
                                for _,name := range([]int{ QUEEN, ROOK, BISHOP, KNIGHT }) {
                                        candidate := NewMove(p, square, target).promote(name)
                                        moves = append(moves, candidate)
                                }
                        }
                } else if p.enpassant != 0 && target == p.enpassant {
                        moves = append(moves, NewMove(p, square, target))
                }
                targets.Clear(target)
        }
        return
}

func (p *Position) reorderMoves(moves []*Move, bestMove *Move, goodMove [2]*Move) []*Move {
        var principal, killers, captures, promotions, remaining []*Move

        for _, move := range moves {
                if len(principal) == 0 && bestMove != nil && move.is(bestMove) {
                        principal = append(principal, move)
                } else if move.captured != 0 {
                        captures = append(captures, move)
                } else if move.promoted != 0 {
                        promotions = append(promotions, move)
                } else if (goodMove[0] != nil && move.is(goodMove[0])) || (goodMove[1] != nil && move.is(goodMove[1])) {
                        killers = append(killers, move)
                } else {
                        remaining = append(remaining, move)
                }
        }
        if len(killers) > 1 && killers[0] == goodMove[1] {
                killers[0], killers[1] = killers[1], killers[0]
        }

        sort.Sort(byScore{captures})
        sort.Sort(byScore{remaining})
        return append(append(append(append(append(principal, captures...), promotions...), killers...), remaining...))
}

func (p *Position) reorderCaptures(moves []*Move, bestMove *Move) []*Move {
        var principal, remaining []*Move

        for _, move := range moves {
                if len(principal) == 0 && move.is(bestMove) {
                        principal = append(principal, move)
                } else {
                        remaining = append(remaining, move)
                }
        }
        sort.Sort(byScore{remaining})
        return append(principal, remaining...)
}

// Sorting moves by their relative score based on piece/square for regular moves
// or least valuaeable attacker/most valueable victim for captures.
type byScore struct {
        moves     []*Move
}
func (her byScore) Len() int           { return len(her.moves)}
func (her byScore) Swap(i, j int)      { her.moves[i], her.moves[j] = her.moves[j], her.moves[i] }
func (her byScore) Less(i, j int) bool { return her.moves[i].score > her.moves[j].score }
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/raintank/raintank-metric/dur"
	"github.com/raintank/raintank-metric/metricdef"
	"github.com/raintank/raintank-metric/schema"
	"log"
	"os"
	"time"
)

func perror(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

var esAddr = flag.String("es-addr", "localhost:9200", "elasticsearch address")
var esIndex = flag.String("es-index", "metric", "elasticsearch index to query")
var format = flag.String("format", "list", "format: list|vegeta-graphite|vegeta-mt")
var maxAge = flag.Int("max-age", 23400, "max age (last update diff with now) of metricdefs. defaults to 6.5hr. use 0 to disable")
var from = flag.String("from", "30min", "from. eg '30min', '5h', '14d', etc")
var silent = flag.Bool("silent", false, "silent mode (don't print number of metrics loaded to stderr)")
var fromS uint32
var total int

func showList(ds []*schema.MetricDefinition) {
	for _, d := range ds {
		if *maxAge != 0 && d.LastUpdate > time.Now().Unix()-int64(*maxAge) {
			total += 1
			fmt.Println(d.OrgId, d.Name)
		}
	}
}
func showVegetaGraphite(ds []*schema.MetricDefinition) {
	for _, d := range ds {
		if *maxAge != 0 && d.LastUpdate > time.Now().Unix()-int64(*maxAge) {
			total += 1
			fmt.Printf("GET http://localhost:8888/render?target=%s&from=-%s\nX-Org-Id: %d\n", d.Name, *from, d.OrgId)
		}
	}
}
func showVegetaMT(ds []*schema.MetricDefinition) {
	from := time.Now().Add(-time.Duration(fromS) * time.Second)
	for _, d := range ds {
		if *maxAge != 0 && d.LastUpdate > time.Now().Unix()-int64(*maxAge) {
			total += 1
			fmt.Printf("GET http://localhost:18763/get?target=%s&from=%d\n", d.Id, from.Unix())
		}
	}
}

func main() {
	flag.Parse()
	var show func(ds []*schema.MetricDefinition)
	switch *format {
	case "list":
		show = showList
	case "vegeta-graphite":
		show = showVegetaGraphite
	case "vegeta-mt":
		show = showVegetaMT
	default:
		log.Fatal("invalid format")
	}
	var err error
	fromS, err = dur.ParseUNsec(*from)
	perror(err)
	defs, err := metricdef.NewDefsEs(*esAddr, "", "", *esIndex)
	perror(err)
	met, scroll_id, err := defs.GetMetrics("")
	perror(err)
	show(met)
	for scroll_id != "" {
		met, scroll_id, err = defs.GetMetrics(scroll_id)
		perror(err)
		show(met)
	}
	if !*silent {
		fmt.Fprintf(os.Stderr, "listed %d metrics\n", total)
	}
}
<commit_msg>fix vegeta-graphite format<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/raintank/raintank-metric/dur"
	"github.com/raintank/raintank-metric/metricdef"
	"github.com/raintank/raintank-metric/schema"
	"log"
	"os"
	"time"
)

func perror(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

var esAddr = flag.String("es-addr", "localhost:9200", "elasticsearch address")
var esIndex = flag.String("es-index", "metric", "elasticsearch index to query")
var format = flag.String("format", "list", "format: list|vegeta-graphite|vegeta-mt")
var maxAge = flag.Int("max-age", 23400, "max age (last update diff with now) of metricdefs. defaults to 6.5hr. use 0 to disable")
var from = flag.String("from", "30min", "from. eg '30min', '5h', '14d', etc")
var silent = flag.Bool("silent", false, "silent mode (don't print number of metrics loaded to stderr)")
var fromS uint32
var total int

func showList(ds []*schema.MetricDefinition) {
	for _, d := range ds {
		if *maxAge != 0 && d.LastUpdate > time.Now().Unix()-int64(*maxAge) {
			total += 1
			fmt.Println(d.OrgId, d.Name)
		}
	}
}
func showVegetaGraphite(ds []*schema.MetricDefinition) {
	for _, d := range ds {
		if *maxAge != 0 && d.LastUpdate > time.Now().Unix()-int64(*maxAge) {
			total += 1
			fmt.Printf("GET http://localhost:8888/render?target=%s&from=-%s\nX-Org-Id: %d\n\n", d.Name, *from, d.OrgId)
		}
	}
}
func showVegetaMT(ds []*schema.MetricDefinition) {
	from := time.Now().Add(-time.Duration(fromS) * time.Second)
	for _, d := range ds {
		if *maxAge != 0 && d.LastUpdate > time.Now().Unix()-int64(*maxAge) {
			total += 1
			fmt.Printf("GET http://localhost:18763/get?target=%s&from=%d\n", d.Id, from.Unix())
		}
	}
}

func main() {
	flag.Parse()
	var show func(ds []*schema.MetricDefinition)
	switch *format {
	case "list":
		show = showList
	case "vegeta-graphite":
		show = showVegetaGraphite
	case "vegeta-mt":
		show = showVegetaMT
	default:
		log.Fatal("invalid format")
	}
	var err error
	fromS, err = dur.ParseUNsec(*from)
	perror(err)
	defs, err := metricdef.NewDefsEs(*esAddr, "", "", *esIndex)
	perror(err)
	met, scroll_id, err := defs.GetMetrics("")
	perror(err)
	show(met)
	for scroll_id != "" {
		met, scroll_id, err = defs.GetMetrics(scroll_id)
		perror(err)
		show(met)
	}
	if !*silent {
		fmt.Fprintf(os.Stderr, "listed %d metrics\n", total)
	}
}
<|endoftext|>


<commit_before>package install

import (
	"bytes"
	"fmt"
	"sort"

	"github.com/hashicorp/go-version"
)

// Memoizes cluster info for install operations
var Clusters = map[string]*SyncedCluster{}

var installCmds = map[string]string{
	"cassandra": `
echo "deb http://www.apache.org/dist/cassandra/debian 311x main" | \
	sudo tee -a /etc/apt/sources.list.d/cassandra.sources.list;
curl https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -;
sudo apt-get update;
sudo apt-get install -y cassandra;
sudo service cassandra stop;
`,

	"docker": `
sudo apt-get update;
sudo apt-get install  -y \
    apt-transport-https \
    ca-certificates \
    curl \
    software-properties-common;
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -;
sudo add-apt-repository \
   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
   $(lsb_release -cs) \
   stable";

sudo apt-get update;
sudo apt-get install  -y docker-ce;
`,

	"gcc": `
sudo apt-get update;
sudo apt-get install -y gcc;
`,

	// graphviz and rlwrap are useful for pprof
	"go": `
sudo apt-get update;
sudo apt-get install -y graphviz rlwrap;

curl https://dl.google.com/go/go1.11.linux-amd64.tar.gz | sudo tar -C /usr/local -xz;
echo 'export PATH=$PATH:/usr/local/go/bin' | sudo tee /etc/profile.d/go.sh > /dev/null;
sudo chmod +x /etc/profile.d/go.sh;
`,

	"haproxy": `
sudo apt-get update;
sudo apt-get install -y haproxy;
`,

	"ntp": `
sudo apt-get update;
sudo apt-get install -y \
  ntp \
  ntpdate;
`,

	"tools": `
sudo apt-get update;
sudo apt-get install -y \
  fio \
  iftop \
  iotop \
  sysstat \
  linux-tools-common \
  linux-tools-4.10.0-35-generic \
  linux-cloud-tools-4.10.0-35-generic;
`,

	"zfs": `
sudo apt-get update;
sudo apt-get install -y \
  zfsutils-linux;
`,
}

func SortedCmds() []string {
	cmds := make([]string, 0, len(installCmds))
	for cmd := range installCmds {
		cmds = append(cmds, cmd)
	}
	sort.Strings(cmds)
	return cmds
}

func Install(c *SyncedCluster, args []string) error {
	do := func(title, cmd string) error {
		var buf bytes.Buffer
		err := c.Run(&buf, &buf, c.Nodes, "installing "+title, cmd)
		if err != nil {
			fmt.Print(buf.String())
		}
		return err
	}

	for _, arg := range args {
		cmd, ok := installCmds[arg]
		if !ok {
			return fmt.Errorf("unknown tool %q", arg)
		}

		if err := do(arg, cmd); err != nil {
			return err
		}
	}
	return nil
}

func VersionSatifies(v *version.Version, constraintString string) bool {
	constraints, err := version.NewConstraint(constraintString)
	if err != nil {
		panic(err)
	}
	return constraints.Check(v)
}
<commit_msg>Add command to install Confluent 5.0.0 (#208)<commit_after>package install

import (
	"bytes"
	"fmt"
	"sort"

	"github.com/hashicorp/go-version"
)

// Memoizes cluster info for install operations
var Clusters = map[string]*SyncedCluster{}

var installCmds = map[string]string{
	"cassandra": `
echo "deb http://www.apache.org/dist/cassandra/debian 311x main" | \
	sudo tee -a /etc/apt/sources.list.d/cassandra.sources.list;
curl https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -;
sudo apt-get update;
sudo apt-get install -y cassandra;
sudo service cassandra stop;
`,

	"confluent": `
sudo apt-get update;
sudo apt-get install -y default-jdk-headless;
curl https://packages.confluent.io/archive/5.0/confluent-oss-5.0.0-2.11.tar.gz | sudo tar -C /usr/local -xz;
sudo ln -s /usr/local/confluent-5.0.0 /usr/local/confluent;
`,

	"docker": `
sudo apt-get update;
sudo apt-get install  -y \
    apt-transport-https \
    ca-certificates \
    curl \
    software-properties-common;
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -;
sudo add-apt-repository \
   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
   $(lsb_release -cs) \
   stable";

sudo apt-get update;
sudo apt-get install  -y docker-ce;
`,

	"gcc": `
sudo apt-get update;
sudo apt-get install -y gcc;
`,

	// graphviz and rlwrap are useful for pprof
	"go": `
sudo apt-get update;
sudo apt-get install -y graphviz rlwrap;

curl https://dl.google.com/go/go1.11.linux-amd64.tar.gz | sudo tar -C /usr/local -xz;
echo 'export PATH=$PATH:/usr/local/go/bin' | sudo tee /etc/profile.d/go.sh > /dev/null;
sudo chmod +x /etc/profile.d/go.sh;
`,

	"haproxy": `
sudo apt-get update;
sudo apt-get install -y haproxy;
`,

	"ntp": `
sudo apt-get update;
sudo apt-get install -y \
  ntp \
  ntpdate;
`,

	"tools": `
sudo apt-get update;
sudo apt-get install -y \
  fio \
  iftop \
  iotop \
  sysstat \
  linux-tools-common \
  linux-tools-4.10.0-35-generic \
  linux-cloud-tools-4.10.0-35-generic;
`,

	"zfs": `
sudo apt-get update;
sudo apt-get install -y \
  zfsutils-linux;
`,
}

func SortedCmds() []string {
	cmds := make([]string, 0, len(installCmds))
	for cmd := range installCmds {
		cmds = append(cmds, cmd)
	}
	sort.Strings(cmds)
	return cmds
}

func Install(c *SyncedCluster, args []string) error {
	do := func(title, cmd string) error {
		var buf bytes.Buffer
		err := c.Run(&buf, &buf, c.Nodes, "installing "+title, cmd)
		if err != nil {
			fmt.Print(buf.String())
		}
		return err
	}

	for _, arg := range args {
		cmd, ok := installCmds[arg]
		if !ok {
			return fmt.Errorf("unknown tool %q", arg)
		}

		if err := do(arg, cmd); err != nil {
			return err
		}
	}
	return nil
}

func VersionSatifies(v *version.Version, constraintString string) bool {
	constraints, err := version.NewConstraint(constraintString)
	if err != nil {
		panic(err)
	}
	return constraints.Check(v)
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"testing"
	"time"

	integration_util "github.com/grafeas/kritis/pkg/kritis/integration_util"
	"github.com/grafeas/kritis/pkg/kritis/testutil"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/wait"
)

const (
	testAttesationAuthority = `apiVersion: kritis.grafeas.io/v1beta1
kind: AttestationAuthority
metadata:
  name: test-attestor
spec:
  noteReference: v1alpha1/projects/krits-int-test
  privateKeySecretName: %s
  publicKeyData: %s`
)

// Secret name for test-attestor
var aaSecret = "test-attestor"

// CRDs is a map of CRD type to names of the expected CRDs to create.
var crdNames = map[string]string{
	"imagesecuritypolicies.kritis.grafeas.io": "my-isp",
}

func createAttestationAuthority(t *testing.T, ns string) {
	t.Helper()
	// Generate a key value pair
	pubKey, privKey := testutil.CreateKeyPair(t, aaSecret)
	// get the base encoded value for public key.
	pubKeyEnc := base64.StdEncoding.EncodeToString([]byte(pubKey))

	// create a tmp dir for keys
	d, err := ioutil.TempDir("", "_keys")
	if err != nil {
		t.Fatalf("unexpected error %s", err)
	}
	defer os.RemoveAll(d)

	// create file with contents in dir
	pubFile := createFileWithContents(t, d, pubKey)

	privFile := createFileWithContents(t, d, privKey)

	// Finally create a kubernetes secret.
	cmd := exec.Command("kubectl", "create", "secret", "generic", aaSecret,
		fmt.Sprintf("--from-file=public=%s", pubFile),
		fmt.Sprintf("--from-file=private=%s", privFile),
		"-n", ns)
	if _, err := integration_util.RunCmdOut(cmd); err != nil {
		t.Fatalf("unexpected error %s", err)
	}

	// Create the Attestation authority
	createAA(t, ns, pubKeyEnc)
}

func waitForCRDExamples(t *testing.T, ns *v1.Namespace) {
	t.Helper()
	t.Logf("Waiting for CRD examples ...")
	for crd, name := range crdNames {
		err := wait.PollImmediate(500*time.Millisecond, 2*time.Minute, func() (bool, error) {
			crdCmd := exec.Command("kubectl", "get", crd, name, "-n", ns.Name)
			err := integration_util.RunCmd(crdCmd)
			return (err == nil), err
		})
		if err != nil {
			t.Fatalf("timeout waiting for crds: %v", err)
		}
	}
}

func createFileWithContents(t *testing.T, d string, c string) string {
	t.Helper()
	file, err := ioutil.TempFile(d, "gpg")
	if err != nil {
		t.Fatalf("testing error: %v", err)
	}
	if _, err := file.WriteString(c); err != nil {
		t.Fatalf("testing error: %v", err)
	}
	return file.Name()
}

func createAA(t *testing.T, ns string, pubkey string) {
	t.Helper()
	cmd := exec.Command("kubectl", "apply", "-n", ns, "-f", "-")
	cmd.Stdin = bytes.NewReader([]byte(fmt.Sprintf(testAttesationAuthority, aaSecret, pubkey)))
	if _, err := integration_util.RunCmdOut(cmd); err != nil {
		t.Fatalf("testing error: %v", err)
	}
}
<commit_msg>Added KritisConfig to the collection of CRDs to be loaded in integration tests<commit_after>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"testing"
	"time"

	integration_util "github.com/grafeas/kritis/pkg/kritis/integration_util"
	"github.com/grafeas/kritis/pkg/kritis/testutil"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/wait"
)

const (
	testAttesationAuthority = `apiVersion: kritis.grafeas.io/v1beta1
kind: AttestationAuthority
metadata:
  name: test-attestor
spec:
  noteReference: v1alpha1/projects/krits-int-test
  privateKeySecretName: %s
  publicKeyData: %s`
)

// Secret name for test-attestor
var aaSecret = "test-attestor"

// CRDs is a map of CRD type to names of the expected CRDs to create.
var crdNames = map[string]string{
	"imagesecuritypolicies.kritis.grafeas.io": "my-isp",
	"kritisconfigs.kritis.grafeas.io": "my-kritis-config",
}

func createAttestationAuthority(t *testing.T, ns string) {
	t.Helper()
	// Generate a key value pair
	pubKey, privKey := testutil.CreateKeyPair(t, aaSecret)
	// get the base encoded value for public key.
	pubKeyEnc := base64.StdEncoding.EncodeToString([]byte(pubKey))

	// create a tmp dir for keys
	d, err := ioutil.TempDir("", "_keys")
	if err != nil {
		t.Fatalf("unexpected error %s", err)
	}
	defer os.RemoveAll(d)

	// create file with contents in dir
	pubFile := createFileWithContents(t, d, pubKey)

	privFile := createFileWithContents(t, d, privKey)

	// Finally create a kubernetes secret.
	cmd := exec.Command("kubectl", "create", "secret", "generic", aaSecret,
		fmt.Sprintf("--from-file=public=%s", pubFile),
		fmt.Sprintf("--from-file=private=%s", privFile),
		"-n", ns)
	if _, err := integration_util.RunCmdOut(cmd); err != nil {
		t.Fatalf("unexpected error %s", err)
	}

	// Create the Attestation authority
	createAA(t, ns, pubKeyEnc)
}

func waitForCRDExamples(t *testing.T, ns *v1.Namespace) {
	t.Helper()
	t.Logf("Waiting for CRD examples ...")
	for crd, name := range crdNames {
		err := wait.PollImmediate(500*time.Millisecond, 2*time.Minute, func() (bool, error) {
			crdCmd := exec.Command("kubectl", "get", crd, name, "-n", ns.Name)
			err := integration_util.RunCmd(crdCmd)
			return (err == nil), err
		})
		if err != nil {
			t.Fatalf("timeout waiting for crds: %v", err)
		}
	}
}

func createFileWithContents(t *testing.T, d string, c string) string {
	t.Helper()
	file, err := ioutil.TempFile(d, "gpg")
	if err != nil {
		t.Fatalf("testing error: %v", err)
	}
	if _, err := file.WriteString(c); err != nil {
		t.Fatalf("testing error: %v", err)
	}
	return file.Name()
}

func createAA(t *testing.T, ns string, pubkey string) {
	t.Helper()
	cmd := exec.Command("kubectl", "apply", "-n", ns, "-f", "-")
	cmd.Stdin = bytes.NewReader([]byte(fmt.Sprintf(testAttesationAuthority, aaSecret, pubkey)))
	if _, err := integration_util.RunCmdOut(cmd); err != nil {
		t.Fatalf("testing error: %v", err)
	}
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"context"
	"fmt"
	"path"
	"time"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	"github.com/pachyderm/pachyderm/src/server/pkg/hashtree"
	workerpkg "github.com/pachyderm/pachyderm/src/server/pkg/worker"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/clientv3/mirror"
	"go.pedge.io/lion/proto"
	"google.golang.org/grpc"
)

const (
	workerEtcdPrefix = "workers"
)

// An input/output pair for a single datum. When a worker has finished
// processing 'data', it writes the resulting hashtree to 'resp' (each job has
// its own response channel)
type datumAndResp struct {
	jobID  string // This is passed to workers, so they can annotate their logs
	datum  []*pfs.FileInfo
	respCh chan hashtree.HashTree
	errCh  chan struct{}
	retCh  chan *datumAndResp
}

// WorkerPool represents a pool of workers that can be used to process datums.
type WorkerPool interface {
	DataCh() chan *datumAndResp
}

type worker struct {
	ctx          context.Context
	addr         string
	workerClient workerpkg.WorkerClient
	pachClient   *client.APIClient
}

func (w *worker) run(dataCh chan *datumAndResp) {
	for {
		dr, ok := <-dataCh
		if !ok {
			return
		}
		resp, err := w.workerClient.Process(w.ctx, &workerpkg.ProcessRequest{
			JobID: dr.jobID,
			Data:  dr.datum,
		})
		if err != nil {
			if isContextCancelledErr(err) {
				return
			}

			dr.retCh <- dr
			protolion.Errorf("worker %s failed to process datum %v with error %s", w.addr, dr.datum, err)
			continue
		}
		if resp.Tag != nil {
			var buffer bytes.Buffer
			if err := w.pachClient.GetTag(resp.Tag.Name, &buffer); err != nil {
				protolion.Errorf("failed to retrieve hashtree after worker %s has ostensibly processed the datum %v", w.addr, dr.datum)
				dr.retCh <- dr
				continue
			}
			tree, err := hashtree.Deserialize(buffer.Bytes())
			if err != nil {
				protolion.Errorf("failed to serialize hashtree after worker %s has ostensibly processed the datum %v; this is likely a bug", w.addr, dr.datum)
				dr.retCh <- dr
				continue
			}
			dr.respCh <- tree
		} else if resp.Failed {
			close(dr.errCh)
		} else {
			protolion.Errorf("unrecognized response from worker %s when processing datum %v; this is likely a bug", w.addr, dr.datum)
			dr.retCh <- dr
			continue
		}
	}
}

type workerPool struct {
	// Worker pool recieves work via this channel
	dataCh chan *datumAndResp

	// Parent of all worker contexts (see workersMap)
	ctx context.Context

	// The prefix in etcd where new workers can be discovered
	workerDir string

	// Map of worker address to cancel fn -- call fn to kill worker goroutine
	workersMap map[string]context.CancelFunc

	// Used to check for workers added/deleted in etcd
	etcdClient *etcd.Client
}

func (w *workerPool) discoverWorkers(ctx context.Context) {
	b := backoff.NewInfiniteBackOff()
	if err := backoff.RetryNotify(func() error {
		syncer := mirror.NewSyncer(w.etcdClient, w.workerDir, 0)
		respCh, errCh := syncer.SyncBase(ctx)
	getBaseWorkers:
		for {
			select {
			case resp, ok := <-respCh:
				if !ok {
					if err := <-errCh; err != nil {
						return err
					}
					break getBaseWorkers
				}
				for _, kv := range resp.Kvs {
					addr := path.Base(string(kv.Key))
					if err := w.addWorker(addr); err != nil {
						return err
					}
				}
			case err := <-errCh:
				if err != nil {
					return err
				}
			}
		}
		watchCh := syncer.SyncUpdates(ctx)
		protolion.Infof("watching `%s` for workers", w.workerDir)
		for {
			resp, ok := <-watchCh
			if !ok {
				return fmt.Errorf("watcher for prefix %s closed for unknown reasons", w.workerDir)
			}
			if err := resp.Err(); err != nil {
				return err
			}
			for _, event := range resp.Events {
				addr := path.Base(string(event.Kv.Key))
				switch event.Type {
				case etcd.EventTypePut:
					if err := w.addWorker(addr); err != nil {
						return err
					}
				case etcd.EventTypeDelete:
					if err := w.delWorker(addr); err != nil {
						return err
					}
				}
			}
		}
		panic("unreachable")
	}, b, func(err error, d time.Duration) error {
		if err == context.Canceled {
			return err
		}
		protolion.Errorf("error discovering workers: %v; retrying in %v", err, d)
		return nil
	}); err != context.Canceled {
		panic(fmt.Sprintf("the retry loop should not exit with a non-context-cancelled error: %v", err))
	}
}

func (w *workerPool) addWorker(addr string) error {
	if cancel, ok := w.workersMap[addr]; ok {
		cancel()
	}

	conn, err := grpc.Dial(fmt.Sprintf("%s:%d", addr, client.PPSWorkerPort), grpc.WithInsecure(), grpc.WithTimeout(5*time.Second))
	if err != nil {
		return err
	}
	childCtx, cancelFn := context.WithCancel(w.ctx)
	w.workersMap[addr] = cancelFn

	pachClient, err := client.NewInCluster()
	if err != nil {
		return err
	}

	wr := &worker{
		ctx:          childCtx,
		addr:         addr,
		workerClient: workerpkg.NewWorkerClient(conn),
		pachClient:   pachClient,
	}
	protolion.Infof("launching new worker at %v", addr)
	go wr.run(w.dataCh)
	return nil
}

func (w *workerPool) delWorker(addr string) error {
	cancel, ok := w.workersMap[addr]
	if !ok {
		return fmt.Errorf("deleting worker %s which is not in worker pool", addr)
	}
	cancel()
	return nil
}

func (w *workerPool) DataCh() chan *datumAndResp {
	return w.dataCh
}

// workerPool fetches the worker pool associated with 'id', or creates one if
// none exists.
func (a *apiServer) workerPool(ctx context.Context, id string) WorkerPool {
	a.workerPoolsLock.Lock()
	defer a.workerPoolsLock.Unlock()
	workerPool, ok := a.workerPools[id]
	if !ok {
		workerPool = a.newWorkerPool(ctx, id)
		a.workerPools[id] = workerPool
	}
	return workerPool
}

// newWorkerPool generates a new worker pool for the job or pipeline identified
// with 'id'.  Each 'id' used to create a new worker pool must correspond to
// a unique binary (in other words, all workers in the worker pool for 'id'
// will be running the same user binary)
func (a *apiServer) newWorkerPool(ctx context.Context, id string) WorkerPool {
	wp := &workerPool{
		ctx:        ctx,
		dataCh:     make(chan *datumAndResp),
		workerDir:  path.Join(a.etcdPrefix, workerEtcdPrefix, id),
		workersMap: make(map[string]context.CancelFunc),
		etcdClient: a.etcdClient,
	}
	// We need to make sure that the prefix ends with the trailing slash,
	// because
	if wp.workerDir[len(wp.workerDir)-1] != '/' {
		wp.workerDir += "/"
	}

	go wp.discoverWorkers(ctx)
	return wp
}

func (a *apiServer) delWorkerPool(id string) {
	a.workerPoolsLock.Lock()
	defer a.workerPoolsLock.Unlock()
	delete(a.workerPools, id)
}
<commit_msg>Fix a bug where worker goros might exit even if their contexts haven't been canceled<commit_after>package server

import (
	"bytes"
	"context"
	"fmt"
	"path"
	"time"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	"github.com/pachyderm/pachyderm/src/server/pkg/hashtree"
	workerpkg "github.com/pachyderm/pachyderm/src/server/pkg/worker"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/clientv3/mirror"
	"go.pedge.io/lion/proto"
	"google.golang.org/grpc"
)

const (
	workerEtcdPrefix = "workers"
)

// An input/output pair for a single datum. When a worker has finished
// processing 'data', it writes the resulting hashtree to 'resp' (each job has
// its own response channel)
type datumAndResp struct {
	jobID  string // This is passed to workers, so they can annotate their logs
	datum  []*pfs.FileInfo
	respCh chan hashtree.HashTree
	errCh  chan struct{}
	retCh  chan *datumAndResp
}

// WorkerPool represents a pool of workers that can be used to process datums.
type WorkerPool interface {
	DataCh() chan *datumAndResp
}

type worker struct {
	ctx          context.Context
	addr         string
	workerClient workerpkg.WorkerClient
	pachClient   *client.APIClient
}

func (w *worker) run(dataCh chan *datumAndResp) {
	for {
		var dr *datumAndResp
		select {
		case dr = <-dataCh:
		case <-w.ctx.Done():
			return
		}
		resp, err := w.workerClient.Process(w.ctx, &workerpkg.ProcessRequest{
			JobID: dr.jobID,
			Data:  dr.datum,
		})
		if err != nil {
			dr.retCh <- dr
			protolion.Errorf("worker %s failed to process datum %v with error %s", w.addr, dr.datum, err)
			continue
		}
		if resp.Tag != nil {
			var buffer bytes.Buffer
			if err := w.pachClient.GetTag(resp.Tag.Name, &buffer); err != nil {
				protolion.Errorf("failed to retrieve hashtree after worker %s has ostensibly processed the datum %v", w.addr, dr.datum)
				dr.retCh <- dr
				continue
			}
			tree, err := hashtree.Deserialize(buffer.Bytes())
			if err != nil {
				protolion.Errorf("failed to serialize hashtree after worker %s has ostensibly processed the datum %v; this is likely a bug", w.addr, dr.datum)
				dr.retCh <- dr
				continue
			}
			dr.respCh <- tree
		} else if resp.Failed {
			close(dr.errCh)
		} else {
			protolion.Errorf("unrecognized response from worker %s when processing datum %v; this is likely a bug", w.addr, dr.datum)
			dr.retCh <- dr
			continue
		}
	}
}

type workerPool struct {
	// Worker pool recieves work via this channel
	dataCh chan *datumAndResp

	// Parent of all worker contexts (see workersMap)
	ctx context.Context

	// The prefix in etcd where new workers can be discovered
	workerDir string

	// Map of worker address to cancel fn -- call fn to kill worker goroutine
	workersMap map[string]context.CancelFunc

	// Used to check for workers added/deleted in etcd
	etcdClient *etcd.Client
}

func (w *workerPool) discoverWorkers(ctx context.Context) {
	b := backoff.NewInfiniteBackOff()
	if err := backoff.RetryNotify(func() error {
		syncer := mirror.NewSyncer(w.etcdClient, w.workerDir, 0)
		respCh, errCh := syncer.SyncBase(ctx)
	getBaseWorkers:
		for {
			select {
			case resp, ok := <-respCh:
				if !ok {
					if err := <-errCh; err != nil {
						return err
					}
					break getBaseWorkers
				}
				for _, kv := range resp.Kvs {
					addr := path.Base(string(kv.Key))
					if err := w.addWorker(addr); err != nil {
						return err
					}
				}
			case err := <-errCh:
				if err != nil {
					return err
				}
			}
		}
		watchCh := syncer.SyncUpdates(ctx)
		protolion.Infof("watching `%s` for workers", w.workerDir)
		for {
			resp, ok := <-watchCh
			if !ok {
				return fmt.Errorf("watcher for prefix %s closed for unknown reasons", w.workerDir)
			}
			if err := resp.Err(); err != nil {
				return err
			}
			for _, event := range resp.Events {
				addr := path.Base(string(event.Kv.Key))
				switch event.Type {
				case etcd.EventTypePut:
					if err := w.addWorker(addr); err != nil {
						return err
					}
				case etcd.EventTypeDelete:
					if err := w.delWorker(addr); err != nil {
						return err
					}
				}
			}
		}
		panic("unreachable")
	}, b, func(err error, d time.Duration) error {
		if err == context.Canceled {
			return err
		}
		protolion.Errorf("error discovering workers: %v; retrying in %v", err, d)
		return nil
	}); err != context.Canceled {
		panic(fmt.Sprintf("the retry loop should not exit with a non-context-cancelled error: %v", err))
	}
}

func (w *workerPool) addWorker(addr string) error {
	if cancel, ok := w.workersMap[addr]; ok {
		cancel()
	}

	conn, err := grpc.Dial(fmt.Sprintf("%s:%d", addr, client.PPSWorkerPort), grpc.WithInsecure(), grpc.WithTimeout(5*time.Second))
	if err != nil {
		return err
	}
	childCtx, cancelFn := context.WithCancel(w.ctx)
	w.workersMap[addr] = cancelFn

	pachClient, err := client.NewInCluster()
	if err != nil {
		return err
	}

	wr := &worker{
		ctx:          childCtx,
		addr:         addr,
		workerClient: workerpkg.NewWorkerClient(conn),
		pachClient:   pachClient,
	}
	protolion.Infof("launching new worker at %v", addr)
	go wr.run(w.dataCh)
	return nil
}

func (w *workerPool) delWorker(addr string) error {
	cancel, ok := w.workersMap[addr]
	if !ok {
		return fmt.Errorf("deleting worker %s which is not in worker pool", addr)
	}
	cancel()
	return nil
}

func (w *workerPool) DataCh() chan *datumAndResp {
	return w.dataCh
}

// workerPool fetches the worker pool associated with 'id', or creates one if
// none exists.
func (a *apiServer) workerPool(ctx context.Context, id string) WorkerPool {
	a.workerPoolsLock.Lock()
	defer a.workerPoolsLock.Unlock()
	workerPool, ok := a.workerPools[id]
	if !ok {
		workerPool = a.newWorkerPool(ctx, id)
		a.workerPools[id] = workerPool
	}
	return workerPool
}

// newWorkerPool generates a new worker pool for the job or pipeline identified
// with 'id'.  Each 'id' used to create a new worker pool must correspond to
// a unique binary (in other words, all workers in the worker pool for 'id'
// will be running the same user binary)
func (a *apiServer) newWorkerPool(ctx context.Context, id string) WorkerPool {
	wp := &workerPool{
		ctx:        ctx,
		dataCh:     make(chan *datumAndResp),
		workerDir:  path.Join(a.etcdPrefix, workerEtcdPrefix, id),
		workersMap: make(map[string]context.CancelFunc),
		etcdClient: a.etcdClient,
	}
	// We need to make sure that the prefix ends with the trailing slash,
	// because
	if wp.workerDir[len(wp.workerDir)-1] != '/' {
		wp.workerDir += "/"
	}

	go wp.discoverWorkers(ctx)
	return wp
}

func (a *apiServer) delWorkerPool(id string) {
	a.workerPoolsLock.Lock()
	defer a.workerPoolsLock.Unlock()
	delete(a.workerPools, id)
}
<|endoftext|>


<commit_before>package intersect

import "sort"

func mergeIntersect(a, b []uint64, final *[]uint64) {
	ma, mb := len(a), len(b)
	i, j := 0, 0
	for i < ma && j < mb {
		if a[i] == b[j] {
			*final = append(*final, a[i])
			i++
			j++

		} else if a[i] < b[j] {
			for i = i + 1; i < ma && a[i] < b[j]; i++ {
			}

		} else {
			for j = j + 1; j < mb && a[i] > b[j]; j++ {
			}
		}
	}
}

func twoLevelLinear(a, b *DeltaList, final *[]uint64) {
	if len(a.Uids) > len(b.Uids) {
		panic("this is wrong")
	}
	if len(a.Uids) == 0 || len(b.Uids) == 0 {
		return
	}

	var ea uint64
	bsize := int(b.BucketSize)
	var eb uint64 = b.Uids[0]

	bucketIdx := 0
	numb := len(b.Buckets)
	end := bsize
	endBucket := b.Buckets[0]
	var ai, bi int

	for ai < len(a.Uids) && bi < len(b.Uids) {
		ea = a.Uids[ai]

		// find the bucket
		for ; endBucket < ea && bucketIdx < numb; bucketIdx++ {
			bi = bsize * bucketIdx
			endBucket = b.Buckets[bucketIdx]

			end = bsize * (bucketIdx + 1)
			if bucketIdx == numb-1 {
				end = len(b.Uids)
			}
		}
		if ea > endBucket {
			break
		}

		// Iterate within the bucket.
		for ea <= endBucket && ai < len(a.Uids) && bi < end {
			ea = a.Uids[ai]
			eb = b.Uids[bi]
			if eb < ea {
				bi++
			} else if eb > ea {
				ai++
			} else {
				*final = append(*final, ea)
				ai++
				bi++
			}
		}
	}
}

func twoLevelBinary(a, b *DeltaList, final *[]uint64) {
	if len(a.Uids) > len(b.Uids) {
		panic("this is wrong")
	}
	if len(a.Uids) == 0 || len(b.Uids) == 0 {
		return
	}

	var ea, eb uint64
	eb = b.Uids[0]
	var bucketIdx int
	bsize := int(b.BucketSize)
	var ai, bi int
	end := len(b.Uids)
	numb := len(b.Buckets)
	endBucket := b.Buckets[0]

	for ai < len(a.Uids) {
		ea = a.Uids[ai]

		if ea > endBucket {
			bucketIdx = sort.Search(numb, func(i int) bool {
				return b.Buckets[i] >= ea
			})
			if bucketIdx == numb {
				return
			}
			endBucket = b.Buckets[bucketIdx]
			bi = bsize * bucketIdx
			end = bsize * (bucketIdx + 1)
			if bucketIdx == numb-1 {
				end = len(b.Uids)
			}
		}
		if ea > endBucket {
			break
		}

		// LINEAR search here.
		for ea <= endBucket && ai < len(a.Uids) && bi < end {
			ea = a.Uids[ai]
			eb = b.Uids[bi]
			if eb < ea {
				bi++
			} else if eb > ea {
				ai++
			} else {
				*final = append(*final, ea)
				ai++
				bi++
			}
		}
	}
}

func BinIntersect(d, q []uint64, final *[]uint64) {
	ld := len(d)
	lq := len(q)
	if ld == 0 || lq == 0 || d[ld-1] < q[0] || q[lq-1] < d[0] {
		return
	}
	if ld < lq {
		panic("what")
	}

	val := d[0]
	minq := sort.Search(len(q), func(i int) bool {
		return q[i] >= val
	})

	val = d[len(d)-1]
	maxq := sort.Search(len(q), func(i int) bool {
		return q[i] >= val
	})

	binIntersect(d, q[minq:maxq], final)
}

func binIntersect(d, q []uint64, final *[]uint64) {
	if len(d) == 0 || len(q) == 0 {
		return
	}
	midq := len(q) / 2
	qval := q[midq]
	midd := sort.Search(len(d), func(i int) bool {
		return d[i] >= qval
	})

	dd := d[0:midd]
	qq := q[0:midq]
	if len(dd) > len(qq) { // D > Q
		binIntersect(dd, qq, final)
	} else {
		binIntersect(qq, dd, final)
	}

	if midd >= len(d) {
		return
	}
	if d[midd] == qval {
		*final = append(*final, qval)
	} else {
		midd -= 1
	}

	dd = d[midd+1:]
	qq = q[midq+1:]
	if len(dd) > len(qq) { // D > Q
		binIntersect(dd, qq, final)
	} else {
		binIntersect(qq, dd, final)
	}
}

func encodeDelta(d []uint64, bucketSize int) *DeltaList {
	l := new(DeltaList)
	if len(d) == 0 {
		return l
	}
	l.BucketSize = int32(bucketSize)

	var last uint64
	for i, cur := range d {
		if i%bucketSize == bucketSize-1 { // Store the max of the bucket.
			l.Buckets = append(l.Buckets, cur)
		}
		l.Uids = append(l.Uids, cur)
		last = cur
	}
	if len(l.Buckets) == 0 || l.Buckets[len(l.Buckets)-1] != last {
		l.Buckets = append(l.Buckets, last)
	}

	return l
}

func encodeFixed(d []uint64) *FixedList {
	f := new(FixedList)
	f.Uids = d
	return f
}
<commit_msg>A bug fix<commit_after>package intersect

import (
	"fmt"
	"sort"
)

func mergeIntersect(a, b []uint64, final *[]uint64) {
	ma, mb := len(a), len(b)
	i, j := 0, 0
	for i < ma && j < mb {
		if a[i] == b[j] {
			*final = append(*final, a[i])
			i++
			j++

		} else if a[i] < b[j] {
			for i = i + 1; i < ma && a[i] < b[j]; i++ {
			}

		} else {
			for j = j + 1; j < mb && a[i] > b[j]; j++ {
			}
		}
	}
}

func twoLevelLinear(a, b *DeltaList, final *[]uint64) {
	if len(a.Uids) > len(b.Uids) {
		panic("this is wrong")
	}
	if len(a.Uids) == 0 || len(b.Uids) == 0 {
		return
	}

	var ea uint64
	bsize := int(b.BucketSize)
	var eb uint64 = b.Uids[0]

	bucketIdx := 0
	numb := len(b.Buckets)
	end := bsize
	endBucket := b.Buckets[0]
	var ai, bi int

	for ai < len(a.Uids) && bi < len(b.Uids) {
		ea = a.Uids[ai]

		// find the bucket
		for ; endBucket < ea && bucketIdx < numb; bucketIdx++ {
			bi = bsize * bucketIdx
			endBucket = b.Buckets[bucketIdx]

			end = bsize * (bucketIdx + 1)
			if bucketIdx == numb-1 {
				end = len(b.Uids)
			}
		}
		if ea > endBucket {
			break
		}

		// Iterate within the bucket.
		for ea <= endBucket && ai < len(a.Uids) && bi < end {
			ea = a.Uids[ai]
			eb = b.Uids[bi]
			if eb < ea {
				bi++
			} else if eb > ea {
				ai++
			} else {
				*final = append(*final, ea)
				ai++
				bi++
			}
		}
	}
}

func twoLevelBinary(a, b *DeltaList, final *[]uint64) {
	if len(a.Uids) > len(b.Uids) {
		panic("this is wrong")
	}
	if len(a.Uids) == 0 || len(b.Uids) == 0 {
		return
	}

	var ea, eb uint64
	eb = b.Uids[0]
	var bucketIdx int
	bsize := int(b.BucketSize)
	var ai, bi int
	end := len(b.Uids)
	numb := len(b.Buckets)
	endBucket := b.Buckets[0]

	for ai < len(a.Uids) {
		ea = a.Uids[ai]

		if ea > endBucket {
			bucketIdx = sort.Search(numb, func(i int) bool {
				return b.Buckets[i] >= ea
			})
			if bucketIdx == numb {
				return
			}
			endBucket = b.Buckets[bucketIdx]
			bi = bsize * bucketIdx
			end = bsize * (bucketIdx + 1)
			if bucketIdx == numb-1 {
				end = len(b.Uids)
			}
		}
		if ea > endBucket {
			break
		}

		// LINEAR search here.
		for ea <= endBucket && ai < len(a.Uids) && bi < end {
			ea = a.Uids[ai]
			eb = b.Uids[bi]
			if eb < ea {
				bi++
			} else if eb > ea {
				ai++
			} else {
				*final = append(*final, ea)
				ai++
				bi++
			}
		}
	}
}

func BinIntersect(d, q []uint64, final *[]uint64) {
	ld := len(d)
	lq := len(q)
	if ld == 0 || lq == 0 || d[ld-1] < q[0] || q[lq-1] < d[0] {
		return
	}
	if ld < lq {
		panic("what")
	}

	val := d[0]
	minq := sort.Search(len(q), func(i int) bool {
		return q[i] >= val
	})

	val = d[len(d)-1]
	maxq := sort.Search(len(q), func(i int) bool {
		return q[i] > val
	})

	fmt.Println(d[0], d[len(d)-1])
	fmt.Println(minq, maxq)
	fmt.Println(d, q[minq:maxq])
	binIntersect(d, q[minq:maxq], final)
}

func binIntersect(d, q []uint64, final *[]uint64) {
	if len(d) == 0 || len(q) == 0 {
		return
	}
	midq := len(q) / 2
	qval := q[midq]
	midd := sort.Search(len(d), func(i int) bool {
		return d[i] >= qval
	})

	dd := d[0:midd]
	qq := q[0:midq]
	if len(dd) > len(qq) { // D > Q
		binIntersect(dd, qq, final)
	} else {
		binIntersect(qq, dd, final)
	}

	if midd >= len(d) {
		return
	}
	if d[midd] == qval {
		*final = append(*final, qval)
	} else {
		midd -= 1
	}

	dd = d[midd+1:]
	qq = q[midq+1:]
	if len(dd) > len(qq) { // D > Q
		binIntersect(dd, qq, final)
	} else {
		binIntersect(qq, dd, final)
	}
}

func encodeDelta(d []uint64, bucketSize int) *DeltaList {
	l := new(DeltaList)
	if len(d) == 0 {
		return l
	}
	l.BucketSize = int32(bucketSize)

	var last uint64
	for i, cur := range d {
		if i%bucketSize == bucketSize-1 { // Store the max of the bucket.
			l.Buckets = append(l.Buckets, cur)
		}
		l.Uids = append(l.Uids, cur)
		last = cur
	}
	if len(l.Buckets) == 0 || l.Buckets[len(l.Buckets)-1] != last {
		l.Buckets = append(l.Buckets, last)
	}

	return l
}

func encodeFixed(d []uint64) *FixedList {
	f := new(FixedList)
	f.Uids = d
	return f
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"crypto/rand"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"path"
	"time"
	"strings"
)

var queuePath string
var scriptsPath string

type RunStat struct {
	ScriptName string
	HostName   string
	UserName   string
	StartTime  int64
	EndTime    int64
	Duration   float64
	Status     string
}

func main() {
	flag.StringVar(&queuePath, "queue-path", DefaultQueuePath(), "queue path")
	flag.StringVar(&scriptsPath, "scripts-path", DefaultScriptsPath(), "scripts path")
	flag.Parse()
	CreatePaths()
	if flag.NArg() == 2 {
		switch flag.Arg(0) {
		case "run":
			runScript(flag.Arg(1))
		case "send-queue":
			runQueue(flag.Arg(1))
		}
	}
}

func UserHome() (path string) {
	me, err := user.Current()
	DieIfErr(err)
	return me.HomeDir
}

func UserName() (username string) {
	me, err := user.Current()
	DieIfErr(err)
	return me.Username
}

func HostName() (hostname string) {
	hostname, err := os.Hostname()
	DieIfErr(err)
	return hostname
}

func DefaultScriptsPath() (path string) {
	return filepath.Join(UserHome(), ".clog-scripts")
}

func DefaultQueuePath() (path string) {
	return filepath.Join(UserHome(), ".clog-queue")
}

func QueuePath() (path string) {
	return queuePath
}

func ScriptsPath() (path string) {
	return scriptsPath
}

func createPath(path string) {
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			log.Print("creating path " + path)
			if err := os.Mkdir(path, 0700); err != nil {
				panic("error creating directory " + path)
			}
		} else {
			panic("error getting directory information!")
		}
	}
}

func CreatePaths() {
	createPath(ScriptsPath())
	createPath(QueuePath())
}

func DieIfErr(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

// Saves queue metadata for delivery.
func (runStat *RunStat) writeQueueMetadata(queuePath string) {
	fp, err := os.OpenFile(queuePath+".meta.tmp", os.O_CREATE|os.O_WRONLY, 0600)
	DieIfErr(err)
	json, err := json.Marshal(runStat)
	DieIfErr(err)
	fp.Write(json)
	fp.Close()
	err = os.Rename(queuePath+".meta.tmp", queuePath+".meta")
	DieIfErr(err)
}

// Read queue metadata.
func (runStat *RunStat) readQueueMetadata(queuePath string) {
	fp, err := os.Open(queuePath)
	DieIfErr(err)
	b := make([]byte, 64000)
	n, err := fp.Read(b)
	DieIfErr(err)
	err = json.Unmarshal(b[:n], &runStat)
	DieIfErr(err)
}

func runScript(script string) {

	runStat := new(RunStat)
	runStat.ScriptName = script

	scriptpath := filepath.Join(ScriptsPath(), script)
	log.Print("running script ", scriptpath)

	id := GenId()
	log.Print("queue id: ", id)
	queueLogPath := filepath.Join(QueuePath(), id)
	log.Print("queue path: ", queueLogPath)

	runStat.HostName = HostName()
	runStat.UserName = UserName()

	runStat.StartTime = time.Now().Unix()
	startTime := time.Now().UnixNano()

	cmd := exec.Command(scriptpath)

	// Merging stdout and stderr.
	stdout, err := cmd.StdoutPipe()
	DieIfErr(err)
	cmd.Stderr = cmd.Stdout

	scanner := bufio.NewScanner(stdout)

	err = cmd.Start()
	DieIfErr(err)

	queuelog, err := os.OpenFile(queueLogPath+".out", os.O_CREATE|os.O_WRONLY, 0600)
	DieIfErr(err)

	log.Print("command output starts")
	for scanner.Scan() {
		line := scanner.Text() + "\n"
		fmt.Print(line)
		queuelog.WriteString(line)
	}
	log.Print("command output ends")
	queuelog.Close()

	// Checking command exit status.
	runStat.Status = "ok"
	if err := cmd.Wait(); err != nil {
		if _, ok := err.(*exec.ExitError); ok {
			runStat.Status = "fail"
		} else {
			log.Fatal(err)
		}
	}

	endTime := time.Now().UnixNano()
	runStat.EndTime = time.Now().Unix()
	runStat.Duration = float64(endTime - startTime)/1000000000.0

	log.Print("status: ", runStat.Status)
	log.Print("duration: ", fmt.Sprintf("%0.3fs", runStat.Duration))
	log.Print("user: ", runStat.UserName)
	log.Print("hostname: ", runStat.HostName)

	runStat.writeQueueMetadata(queueLogPath)

}

func runQueue(url string) {
	log.Print("target server: ", url)
	queuePath := QueuePath()
	log.Print("queue path: ", queuePath)
	files, err := ioutil.ReadDir(queuePath)
	DieIfErr(err)
	for _, file := range files {
		name := file.Name()
		if len(name) == 41 && strings.HasSuffix(name, ".meta") {
			queueId := name[:36]
			log.Print("dispatching ", queueId)
			runStat := new(RunStat)
			runStat.readQueueMetadata(path.Join(queuePath, queueId) + ".meta")
		}
	}
}

func GenId() (id string) {
	// Poor man's UUID generator
	const validchars = "0123456789abcdef"
	b := make([]byte, 36)
	if _, err := rand.Read(b); err != nil {
		panic(err)
	}
	for i, j := range b {
		b[i] = validchars[j%byte(len(validchars))]
	}
	b[8] = '-'
	b[13] = '-'
	b[18] = '-'
	b[23] = '-'
	b[14] = '4'
	return string(b)
}
<commit_msg>go fmt<commit_after>package main

import (
	"bufio"
	"crypto/rand"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path"
	"path/filepath"
	"strings"
	"time"
)

var queuePath string
var scriptsPath string

type RunStat struct {
	ScriptName string
	HostName   string
	UserName   string
	StartTime  int64
	EndTime    int64
	Duration   float64
	Status     string
}

func main() {
	flag.StringVar(&queuePath, "queue-path", DefaultQueuePath(), "queue path")
	flag.StringVar(&scriptsPath, "scripts-path", DefaultScriptsPath(), "scripts path")
	flag.Parse()
	CreatePaths()
	if flag.NArg() == 2 {
		switch flag.Arg(0) {
		case "run":
			runScript(flag.Arg(1))
		case "send-queue":
			runQueue(flag.Arg(1))
		}
	}
}

func UserHome() (path string) {
	me, err := user.Current()
	DieIfErr(err)
	return me.HomeDir
}

func UserName() (username string) {
	me, err := user.Current()
	DieIfErr(err)
	return me.Username
}

func HostName() (hostname string) {
	hostname, err := os.Hostname()
	DieIfErr(err)
	return hostname
}

func DefaultScriptsPath() (path string) {
	return filepath.Join(UserHome(), ".clog-scripts")
}

func DefaultQueuePath() (path string) {
	return filepath.Join(UserHome(), ".clog-queue")
}

func QueuePath() (path string) {
	return queuePath
}

func ScriptsPath() (path string) {
	return scriptsPath
}

func createPath(path string) {
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			log.Print("creating path " + path)
			if err := os.Mkdir(path, 0700); err != nil {
				panic("error creating directory " + path)
			}
		} else {
			panic("error getting directory information!")
		}
	}
}

func CreatePaths() {
	createPath(ScriptsPath())
	createPath(QueuePath())
}

func DieIfErr(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

// Saves queue metadata for delivery.
func (runStat *RunStat) writeQueueMetadata(queuePath string) {
	fp, err := os.OpenFile(queuePath+".meta.tmp", os.O_CREATE|os.O_WRONLY, 0600)
	DieIfErr(err)
	json, err := json.Marshal(runStat)
	DieIfErr(err)
	fp.Write(json)
	fp.Close()
	err = os.Rename(queuePath+".meta.tmp", queuePath+".meta")
	DieIfErr(err)
}

// Read queue metadata.
func (runStat *RunStat) readQueueMetadata(queuePath string) {
	fp, err := os.Open(queuePath)
	DieIfErr(err)
	b := make([]byte, 64000)
	n, err := fp.Read(b)
	DieIfErr(err)
	err = json.Unmarshal(b[:n], &runStat)
	DieIfErr(err)
}

func runScript(script string) {

	runStat := new(RunStat)
	runStat.ScriptName = script

	scriptpath := filepath.Join(ScriptsPath(), script)
	log.Print("running script ", scriptpath)

	id := GenId()
	log.Print("queue id: ", id)
	queueLogPath := filepath.Join(QueuePath(), id)
	log.Print("queue path: ", queueLogPath)

	runStat.HostName = HostName()
	runStat.UserName = UserName()

	runStat.StartTime = time.Now().Unix()
	startTime := time.Now().UnixNano()

	cmd := exec.Command(scriptpath)

	// Merging stdout and stderr.
	stdout, err := cmd.StdoutPipe()
	DieIfErr(err)
	cmd.Stderr = cmd.Stdout

	scanner := bufio.NewScanner(stdout)

	err = cmd.Start()
	DieIfErr(err)

	queuelog, err := os.OpenFile(queueLogPath+".out", os.O_CREATE|os.O_WRONLY, 0600)
	DieIfErr(err)

	log.Print("command output starts")
	for scanner.Scan() {
		line := scanner.Text() + "\n"
		fmt.Print(line)
		queuelog.WriteString(line)
	}
	log.Print("command output ends")
	queuelog.Close()

	// Checking command exit status.
	runStat.Status = "ok"
	if err := cmd.Wait(); err != nil {
		if _, ok := err.(*exec.ExitError); ok {
			runStat.Status = "fail"
		} else {
			log.Fatal(err)
		}
	}

	endTime := time.Now().UnixNano()
	runStat.EndTime = time.Now().Unix()
	runStat.Duration = float64(endTime-startTime) / 1000000000.0

	log.Print("status: ", runStat.Status)
	log.Print("duration: ", fmt.Sprintf("%0.3fs", runStat.Duration))
	log.Print("user: ", runStat.UserName)
	log.Print("hostname: ", runStat.HostName)

	runStat.writeQueueMetadata(queueLogPath)

}

func runQueue(url string) {
	log.Print("target server: ", url)
	queuePath := QueuePath()
	log.Print("queue path: ", queuePath)
	files, err := ioutil.ReadDir(queuePath)
	DieIfErr(err)
	for _, file := range files {
		name := file.Name()
		if len(name) == 41 && strings.HasSuffix(name, ".meta") {
			queueId := name[:36]
			log.Print("dispatching ", queueId)
			runStat := new(RunStat)
			runStat.readQueueMetadata(path.Join(queuePath, queueId) + ".meta")
		}
	}
}

func GenId() (id string) {
	// Poor man's UUID generator
	const validchars = "0123456789abcdef"
	b := make([]byte, 36)
	if _, err := rand.Read(b); err != nil {
		panic(err)
	}
	for i, j := range b {
		b[i] = validchars[j%byte(len(validchars))]
	}
	b[8] = '-'
	b[13] = '-'
	b[18] = '-'
	b[23] = '-'
	b[14] = '4'
	return string(b)
}
<|endoftext|>


<commit_before>package rc

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"

	"github.com/ncw/rclone/cmd"
	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/fshttp"
	"github.com/ncw/rclone/fs/rc"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

var (
	noOutput  = false
	url       = "http://localhost:5572/"
	jsonInput = ""
	authUser  = ""
	authPass  = ""
)

func init() {
	cmd.Root.AddCommand(commandDefintion)
	commandDefintion.Flags().BoolVarP(&noOutput, "no-output", "", noOutput, "If set don't output the JSON result.")
	commandDefintion.Flags().StringVarP(&url, "url", "", url, "URL to connect to rclone remote control.")
	commandDefintion.Flags().StringVarP(&jsonInput, "json", "", jsonInput, "Input JSON - use instead of key=value args.")
	commandDefintion.Flags().StringVarP(&authUser, "user", "", "", "Username to use to rclone remote control.")
	commandDefintion.Flags().StringVarP(&authPass, "pass", "", "", "Password to use to connect to rclone remote control.")
}

var commandDefintion = &cobra.Command{
	Use:   "rc commands parameter",
	Short: `Run a command against a running rclone.`,
	Long: `

This runs a command against a running rclone.  Use the --url flag to
specify an non default URL to connect on.  This can be either a
":port" which is taken to mean "http://localhost:port" or a
"host:port" which is taken to mean "http://host:port"

A username and password can be passed in with --user and --pass.

Note that --rc-addr, --rc-user, --rc-pass will be read also for --url,
--user, --pass.

Arguments should be passed in as parameter=value.

The result will be returned as a JSON object by default.

The --json parameter can be used to pass in a JSON blob as an input
instead of key=value arguments.  This is the only way of passing in
more complicated values.

Use "rclone rc" to see a list of all possible commands.`,
	Run: func(command *cobra.Command, args []string) {
		cmd.CheckArgs(0, 1E9, command, args)
		cmd.Run(false, false, command, func() error {
			parseFlags()
			if len(args) == 0 {
				return list()
			}
			return run(args)
		})
	},
}

// Parse the flags
func parseFlags() {
	// set alternates from alternate flags
	setAlternateFlag("rc-addr", &url)
	setAlternateFlag("rc-user", &authUser)
	setAlternateFlag("rc-pass", &authPass)
	// If url is just :port then fix it up
	if strings.HasPrefix(url, ":") {
		url = "localhost" + url
	}
	// if url is just host:port add http://
	if !strings.HasPrefix(url, "http:") && !strings.HasPrefix(url, "https:") {
		url = "http://" + url
	}
	// if url doesn't end with / add it
	if !strings.HasSuffix(url, "/") {
		url += "/"
	}
}

// If the user set flagName set the output to its value
func setAlternateFlag(flagName string, output *string) {
	if rcFlag := pflag.Lookup(flagName); rcFlag != nil && rcFlag.Changed {
		*output = rcFlag.Value.String()
	}
}

// do a call from (path, in) to (out, err).
//
// if err is set, out may be a valid error return or it may be nil
func doCall(path string, in rc.Params) (out rc.Params, err error) {
	// Do HTTP request
	client := fshttp.NewClient(fs.Config)
	url += path
	data, err := json.Marshal(in)
	if err != nil {
		return nil, errors.Wrap(err, "failed to encode JSON")
	}

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
	if err != nil {
		return nil, errors.Wrap(err, "failed to make request")
	}

	req.Header.Set("Content-Type", "application/json")
	if authUser != "" || authPass != "" {
		req.SetBasicAuth(authUser, authPass)
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, "connection failed")
	}
	defer fs.CheckClose(resp.Body, &err)

	if resp.StatusCode != http.StatusOK {
		var body []byte
		body, err = ioutil.ReadAll(resp.Body)
		var bodyString string
		if err == nil {
			bodyString = string(body)
		} else {
			bodyString = err.Error()
		}
		bodyString = strings.TrimSpace(bodyString)
		return nil, errors.Errorf("Failed to read rc response: %s: %s", resp.Status, bodyString)
	}

	// Parse output
	out = make(rc.Params)
	err = json.NewDecoder(resp.Body).Decode(&out)
	if err != nil {
		return nil, errors.Wrap(err, "failed to decode JSON")
	}

	// Check we got 200 OK
	if resp.StatusCode != http.StatusOK {
		err = errors.Errorf("operation %q failed: %v", path, out["error"])
	}

	return out, err
}

// Run the remote control command passed in
func run(args []string) (err error) {
	path := strings.Trim(args[0], "/")

	// parse input
	in := make(rc.Params)
	params := args[1:]
	if jsonInput == "" {
		for _, param := range params {
			equals := strings.IndexRune(param, '=')
			if equals < 0 {
				return errors.Errorf("no '=' found in parameter %q", param)
			}
			key, value := param[:equals], param[equals+1:]
			in[key] = value
		}
	} else {
		if len(params) > 0 {
			return errors.New("can't use --json and parameters together")
		}
		err = json.Unmarshal([]byte(jsonInput), &in)
		if err != nil {
			return errors.Wrap(err, "bad --json input")
		}
	}

	// Do the call
	out, callErr := doCall(path, in)

	// Write the JSON blob to stdout if required
	if out != nil && !noOutput {
		err := rc.WriteJSON(os.Stdout, out)
		if err != nil {
			return errors.Wrap(err, "failed to output JSON")
		}
	}

	return callErr
}

// List the available commands to stdout
func list() error {
	list, err := doCall("rc/list", nil)
	if err != nil {
		return errors.Wrap(err, "failed to list")
	}
	commands, ok := list["commands"].([]interface{})
	if !ok {
		return errors.New("bad JSON")
	}
	for _, command := range commands {
		info, ok := command.(map[string]interface{})
		if !ok {
			return errors.New("bad JSON")
		}
		fmt.Printf("### %s: %s\n\n", info["Path"], info["Title"])
		fmt.Printf("%s\n\n", info["Help"])
		if authRequired := info["AuthRequired"]; authRequired != nil {
			if authRequired.(bool) {
				fmt.Printf("Authentication is required for this call.\n\n")
			}
		}
	}
	return nil
}
<commit_msg>rc: add --loopback flag to run commands directly without a server<commit_after>package rc

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"

	"github.com/ncw/rclone/cmd"
	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/fshttp"
	"github.com/ncw/rclone/fs/rc"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

var (
	noOutput  = false
	url       = "http://localhost:5572/"
	jsonInput = ""
	authUser  = ""
	authPass  = ""
	loopback  = false
)

func init() {
	cmd.Root.AddCommand(commandDefintion)
	commandDefintion.Flags().BoolVarP(&noOutput, "no-output", "", noOutput, "If set don't output the JSON result.")
	commandDefintion.Flags().StringVarP(&url, "url", "", url, "URL to connect to rclone remote control.")
	commandDefintion.Flags().StringVarP(&jsonInput, "json", "", jsonInput, "Input JSON - use instead of key=value args.")
	commandDefintion.Flags().StringVarP(&authUser, "user", "", "", "Username to use to rclone remote control.")
	commandDefintion.Flags().StringVarP(&authPass, "pass", "", "", "Password to use to connect to rclone remote control.")
	commandDefintion.Flags().BoolVarP(&loopback, "loopback", "", false, "If set connect to this rclone instance not via HTTP.")
}

var commandDefintion = &cobra.Command{
	Use:   "rc commands parameter",
	Short: `Run a command against a running rclone.`,
	Long: `

This runs a command against a running rclone.  Use the --url flag to
specify an non default URL to connect on.  This can be either a
":port" which is taken to mean "http://localhost:port" or a
"host:port" which is taken to mean "http://host:port"

A username and password can be passed in with --user and --pass.

Note that --rc-addr, --rc-user, --rc-pass will be read also for --url,
--user, --pass.

Arguments should be passed in as parameter=value.

The result will be returned as a JSON object by default.

The --json parameter can be used to pass in a JSON blob as an input
instead of key=value arguments.  This is the only way of passing in
more complicated values.

Use --loopback to connect to the rclone instance running "rclone rc".
This is very useful for testing commands without having to run an
rclone rc server, eg:

    rclone rc --loopback operations/about fs=/

Use "rclone rc" to see a list of all possible commands.`,
	Run: func(command *cobra.Command, args []string) {
		cmd.CheckArgs(0, 1E9, command, args)
		cmd.Run(false, false, command, func() error {
			parseFlags()
			if len(args) == 0 {
				return list()
			}
			return run(args)
		})
	},
}

// Parse the flags
func parseFlags() {
	// set alternates from alternate flags
	setAlternateFlag("rc-addr", &url)
	setAlternateFlag("rc-user", &authUser)
	setAlternateFlag("rc-pass", &authPass)
	// If url is just :port then fix it up
	if strings.HasPrefix(url, ":") {
		url = "localhost" + url
	}
	// if url is just host:port add http://
	if !strings.HasPrefix(url, "http:") && !strings.HasPrefix(url, "https:") {
		url = "http://" + url
	}
	// if url doesn't end with / add it
	if !strings.HasSuffix(url, "/") {
		url += "/"
	}
}

// If the user set flagName set the output to its value
func setAlternateFlag(flagName string, output *string) {
	if rcFlag := pflag.Lookup(flagName); rcFlag != nil && rcFlag.Changed {
		*output = rcFlag.Value.String()
	}
}

// do a call from (path, in) to (out, err).
//
// if err is set, out may be a valid error return or it may be nil
func doCall(path string, in rc.Params) (out rc.Params, err error) {
	// If loopback set, short circuit HTTP request
	if loopback {
		call := rc.Calls.Get(path)
		if call == nil {
			return nil, errors.Errorf("method %q not found", path)
		}
		return call.Fn(in)
	}

	// Do HTTP request
	client := fshttp.NewClient(fs.Config)
	url += path
	data, err := json.Marshal(in)
	if err != nil {
		return nil, errors.Wrap(err, "failed to encode JSON")
	}

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
	if err != nil {
		return nil, errors.Wrap(err, "failed to make request")
	}

	req.Header.Set("Content-Type", "application/json")
	if authUser != "" || authPass != "" {
		req.SetBasicAuth(authUser, authPass)
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, "connection failed")
	}
	defer fs.CheckClose(resp.Body, &err)

	if resp.StatusCode != http.StatusOK {
		var body []byte
		body, err = ioutil.ReadAll(resp.Body)
		var bodyString string
		if err == nil {
			bodyString = string(body)
		} else {
			bodyString = err.Error()
		}
		bodyString = strings.TrimSpace(bodyString)
		return nil, errors.Errorf("Failed to read rc response: %s: %s", resp.Status, bodyString)
	}

	// Parse output
	out = make(rc.Params)
	err = json.NewDecoder(resp.Body).Decode(&out)
	if err != nil {
		return nil, errors.Wrap(err, "failed to decode JSON")
	}

	// Check we got 200 OK
	if resp.StatusCode != http.StatusOK {
		err = errors.Errorf("operation %q failed: %v", path, out["error"])
	}

	return out, err
}

// Run the remote control command passed in
func run(args []string) (err error) {
	path := strings.Trim(args[0], "/")

	// parse input
	in := make(rc.Params)
	params := args[1:]
	if jsonInput == "" {
		for _, param := range params {
			equals := strings.IndexRune(param, '=')
			if equals < 0 {
				return errors.Errorf("no '=' found in parameter %q", param)
			}
			key, value := param[:equals], param[equals+1:]
			in[key] = value
		}
	} else {
		if len(params) > 0 {
			return errors.New("can't use --json and parameters together")
		}
		err = json.Unmarshal([]byte(jsonInput), &in)
		if err != nil {
			return errors.Wrap(err, "bad --json input")
		}
	}

	// Do the call
	out, callErr := doCall(path, in)

	// Write the JSON blob to stdout if required
	if out != nil && !noOutput {
		err := rc.WriteJSON(os.Stdout, out)
		if err != nil {
			return errors.Wrap(err, "failed to output JSON")
		}
	}

	return callErr
}

// List the available commands to stdout
func list() error {
	list, err := doCall("rc/list", nil)
	if err != nil {
		return errors.Wrap(err, "failed to list")
	}
	commands, ok := list["commands"].([]interface{})
	if !ok {
		return errors.New("bad JSON")
	}
	for _, command := range commands {
		info, ok := command.(map[string]interface{})
		if !ok {
			return errors.New("bad JSON")
		}
		fmt.Printf("### %s: %s\n\n", info["Path"], info["Title"])
		fmt.Printf("%s\n\n", info["Help"])
		if authRequired := info["AuthRequired"]; authRequired != nil {
			if authRequired.(bool) {
				fmt.Printf("Authentication is required for this call.\n\n")
			}
		}
	}
	return nil
}
<|endoftext|>