<commit_before>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"errors"
	"fmt"
	"math"
	"sort"
	"time"

	"github.com/tsuru/config"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/db/storage"
	"github.com/tsuru/tsuru/iaas"
	"github.com/tsuru/tsuru/log"
	"github.com/tsuru/tsuru/safe"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type autoScaleEvent struct {
	ID            interface{} `bson:"_id"`
	MetadataValue string
	Action        string // "rebalance" or "add"
	StartTime     time.Time
	EndTime       time.Time `bson:",omitempty"`
	Successful    bool
	Error         string `bson:",omitempty"`
}

func autoScaleCollection() (*storage.Collection, error) {
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	name, err := config.GetString("docker:collection")
	if err != nil {
		return nil, err
	}
	return conn.Collection(fmt.Sprintf("%s_auto_scale", name)), nil
}

var errAutoScaleRunning = errors.New("autoscale already running")

func newAutoScaleEvent(metadataValue, action string) (*autoScaleEvent, error) {
	// Use metadataValue as ID to ensure only one auto scale process runs for
	// each metadataValue. (*autoScaleEvent).update() will generate a new
	// unique ID and remove this initial record.
	evt := autoScaleEvent{
		ID:            metadataValue,
		StartTime:     time.Now().UTC(),
		MetadataValue: metadataValue,
		Action:        action,
	}
	coll, err := autoScaleCollection()
	if err != nil {
		return nil, err
	}
	defer coll.Close()
	err = coll.Insert(evt)
	if mgo.IsDup(err) {
		return nil, errAutoScaleRunning
	}
	return &evt, err
}

func (evt *autoScaleEvent) update(err error) error {
	if err != nil {
		evt.Error = err.Error()
	}
	evt.Successful = err == nil
	evt.EndTime = time.Now().UTC()
	coll, err := autoScaleCollection()
	if err != nil {
		return err
	}
	defer coll.Close()
	defer coll.RemoveId(evt.ID)
	evt.ID = bson.NewObjectId()
	return coll.Insert(evt)
}

func listAutoScaleEvents(skip, limit int) ([]autoScaleEvent, error) {
	coll, err := autoScaleCollection()
	if err != nil {
		return nil, err
	}
	query := coll.Find(nil).Sort("-_id")
	if skip != 0 {
		query = query.Skip(skip)
	}
	if limit != 0 {
		query = query.Limit(limit)
	}
	var list []autoScaleEvent
	err = query.All(&list)
	if err != nil {
		return nil, err
	}
	return list, nil
}

type autoScaleConfig struct {
	provisioner         *dockerProvisioner
	matadataFilter      string
	groupByMetadata     string
	totalMemoryMetadata string
	maxMemoryRatio      float32
	maxContainerCount   int
	waitTimeNewMachine  time.Duration
	runInterval         time.Duration
	done                chan bool
}

type autoScaler interface {
	scale(groupMetadata string, nodes []*cluster.Node) error
}

type memoryScaler struct {
	*autoScaleConfig
}

type countScaler struct {
	*autoScaleConfig
}

type metaWithFrequency struct {
	metadata map[string]string
	freq     int
}

type metaWithFrequencyList []metaWithFrequency

func (l metaWithFrequencyList) Len() int           { return len(l) }
func (l metaWithFrequencyList) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
func (l metaWithFrequencyList) Less(i, j int) bool { return l[i].freq < l[j].freq }

func (a *autoScaleConfig) run() error {
	isMemoryBased := a.totalMemoryMetadata != "" && a.maxMemoryRatio != 0
	if !isMemoryBased && a.maxContainerCount == 0 {
		err := fmt.Errorf("[node autoscale] aborting node auto scale, either memory information or max container count must be informed in config")
		log.Error(err.Error())
		return err
	}
	oneMinute := 1 * time.Minute
	if a.runInterval < oneMinute {
		a.runInterval = oneMinute
	}
	if a.waitTimeNewMachine < oneMinute {
		a.waitTimeNewMachine = oneMinute
	}
	var scaler autoScaler
	if isMemoryBased {
		scaler = &memoryScaler{a}
	} else {
		scaler = &countScaler{a}
	}
	for {
		err := a.runOnce(scaler)
		if err != nil {
			err = fmt.Errorf("[node autoscale] %s", err.Error())
			log.Error(err.Error())
		}
		select {
		case <-a.done:
			return err
		case <-time.After(a.runInterval):
		}
	}
}

func (a *autoScaleConfig) stop() {
	a.done <- true
}

func (a *autoScaleConfig) runOnce(scaler autoScaler) (retErr error) {
	defer func() {
		if r := recover(); r != nil {
			retErr = fmt.Errorf("recovered panic, we can never stop! panic: %v", r)
		}
	}()
	nodes, err := a.provisioner.getCluster().Nodes()
	if err != nil {
		retErr = fmt.Errorf("error getting nodes: %s", err.Error())
		return
	}
	clusterMap := map[string][]*cluster.Node{}
	for i := range nodes {
		node := &nodes[i]
		if a.groupByMetadata == "" {
			clusterMap[""] = append(clusterMap[""], node)
			continue
		}
		groupMetadata := node.Metadata[a.groupByMetadata]
		if groupMetadata == "" {
			log.Debugf("[node autoscale] skipped node %s, no metadata value for %s.", node.Address, a.groupByMetadata)
			continue
		}
		if a.matadataFilter != "" && a.matadataFilter != groupMetadata {
			continue
		}
		clusterMap[groupMetadata] = append(clusterMap[groupMetadata], node)
	}
	for groupMetadata, nodes := range clusterMap {
		err = scaler.scale(groupMetadata, nodes)
		if err != nil {
			retErr = fmt.Errorf("error scaling group %s: %s", groupMetadata, err.Error())
			return
		}
	}
	return
}

func (a *memoryScaler) scale(groupMetadata string, nodes []*cluster.Node) error {
	return nil
}

func (a *countScaler) scale(groupMetadata string, nodes []*cluster.Node) error {
	totalCount, gap, err := a.provisioner.containerGapInNodes(nodes)
	if err != nil {
		return fmt.Errorf("couldn't find containers from nodes: %s", err)
	}
	freeSlots := (len(nodes) * a.maxContainerCount) - totalCount
	shouldRebalance := false
	var rebalanceFilter map[string]string
	if a.groupByMetadata != "" {
		rebalanceFilter = map[string]string{a.groupByMetadata: groupMetadata}
	}
	var event *autoScaleEvent
	var gapAfter int
	if freeSlots < 0 {
		shouldRebalance = true
		event, err = newAutoScaleEvent(groupMetadata, "add")
		if err != nil {
			if err == errAutoScaleRunning {
				log.Debugf("[node autoscale] skipping already running for: %s", groupMetadata)
				return nil
			}
			return fmt.Errorf("unable to create auto scale event: %s", err)
		}
		log.Debugf("[node autoscale] adding a new machine, metadata value: %s, free slots: %d", groupMetadata, freeSlots)
		err := a.addNode(nodes)
		if err != nil {
			event.update(err)
			return err
		}
	} else {
		buf := safe.NewBuffer(nil)
		rebalanceProvisioner, err := a.provisioner.rebalanceContainersByFilter(buf, nil, rebalanceFilter, true)
		if err != nil {
			log.Errorf("unable to run dry rebalance to check if rebalance is needed: %s - log: %s", err, buf.String())
			return nil
		}
		_, gapAfter, err = rebalanceProvisioner.containerGapInNodes(nodes)
		if err != nil {
			return fmt.Errorf("couldn't find containers from rebalanced nodes: %s", err)
		}
		shouldRebalance = math.Abs((float64)(gap-gapAfter)) > 2.0
	}
	if shouldRebalance {
		log.Debugf("[node autoscale] running rebalance, metadata value: %s, gap before: %d, gap after: %d", groupMetadata, gap, gapAfter)
		if event == nil {
			event, err = newAutoScaleEvent(groupMetadata, "rebalance")
			if err != nil {
				if err == errAutoScaleRunning {
					log.Debugf("[node autoscale] skipping already running for: %s", groupMetadata)
					return nil
				}
				return fmt.Errorf("unable to create auto scale event: %s", err)
			}
		}
		buf := safe.NewBuffer(nil)
		_, err := a.provisioner.rebalanceContainersByFilter(buf, nil, rebalanceFilter, false)
		if err != nil {
			log.Errorf("Unable to rebalance containers: %s - log: %s", err.Error(), buf.String())
		}
		event.update(err)
	}
	return nil
}

func (a *autoScaleConfig) addNode(modelNodes []*cluster.Node) error {
	metadata, err := chooseMetadataFromNodes(modelNodes)
	if err != nil {
		return err
	}
	_, hasIaas := metadata["iaas"]
	if !hasIaas {
		return fmt.Errorf("no IaaS information in nodes metadata: %#v", metadata)
	}
	machine, err := iaas.CreateMachineForIaaS(metadata["iaas"], metadata)
	if err != nil {
		return fmt.Errorf("unable to create machine: %s", err.Error())
	}
	newAddr := machine.FormatNodeAddress()
	log.Debugf("[node autoscale] new machine created: %s - Waiting for docker to start...", newAddr)
	_, err = a.provisioner.getCluster().WaitAndRegister(newAddr, metadata, a.waitTimeNewMachine)
	if err != nil {
		machine.Destroy()
		return fmt.Errorf("error registering new node %s: %s", newAddr, err.Error())
	}
	log.Debugf("[node autoscale] new machine created: %s - started!", newAddr)
	return nil
}

func splitMetadata(nodesMetadata []map[string]string) (metaWithFrequencyList, map[string]string, error) {
	common := make(map[string]string)
	exclusive := make([]map[string]string, len(nodesMetadata))
	for i := range nodesMetadata {
		metadata := nodesMetadata[i]
		for k, v := range metadata {
			isExclusive := false
			for j := range nodesMetadata {
				if i == j {
					continue
				}
				otherMetadata := nodesMetadata[j]
				if v != otherMetadata[k] {
					isExclusive = true
					break
				}
			}
			if isExclusive {
				if exclusive[i] == nil {
					exclusive[i] = make(map[string]string)
				}
				exclusive[i][k] = v
			} else {
				common[k] = v
			}
		}
	}
	var group metaWithFrequencyList
	sameMap := make(map[int]bool)
	for i := range exclusive {
		freq := 1
		for j := range exclusive {
			if i == j {
				continue
			}
			diffCount := 0
			for k, v := range exclusive[i] {
				if exclusive[j][k] != v {
					diffCount++
				}
			}
			if diffCount > 0 && diffCount < len(exclusive[i]) {
				return nil, nil, fmt.Errorf("unbalanced metadata for node group: %v vs %v", exclusive[i], exclusive[j])
			}
			if diffCount == 0 {
				sameMap[j] = true
				freq++
			}
		}
		if !sameMap[i] && exclusive[i] != nil {
			group = append(group, metaWithFrequency{metadata: exclusive[i], freq: freq})
		}
	}
	return group, common, nil
}

func chooseMetadataFromNodes(modelNodes []*cluster.Node) (map[string]string, error) {
	metadataList := make([]map[string]string, len(modelNodes))
	for i, n := range modelNodes {
		metadataList[i] = n.CleanMetadata()
	}
	exclusiveList, baseMetadata, err := splitMetadata(metadataList)
	if err != nil {
		return nil, err
	}
	var chosenExclusive map[string]string
	if exclusiveList != nil {
		sort.Sort(exclusiveList)
		chosenExclusive = exclusiveList[0].metadata
	}
	for k, v := range chosenExclusive {
		baseMetadata[k] = v
	}
	return baseMetadata, nil
}

func (p *dockerProvisioner) containerGapInNodes(nodes []*cluster.Node) (int, int, error) {
	maxCount := 0
	minCount := 0
	totalCount := 0
	for _, n := range nodes {
		contCount, err := p.countRunningContainersByHost(urlToHost(n.Address))
		if err != nil {
			return 0, 0, err
		}
		if contCount > maxCount {
			maxCount = contCount
		}
		if minCount == 0 || contCount < minCount {
			minCount = contCount
		}
		totalCount += contCount
	}
	return totalCount, maxCount - minCount, nil
}
<commit_msg>provision/docker: extracting common behavior from auto scale<commit_after>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"errors"
	"fmt"
	"math"
	"sort"
	"time"

	"github.com/tsuru/config"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/db/storage"
	"github.com/tsuru/tsuru/iaas"
	"github.com/tsuru/tsuru/log"
	"github.com/tsuru/tsuru/safe"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type autoScaleEvent struct {
	ID            interface{} `bson:"_id"`
	MetadataValue string
	Action        string // "rebalance" or "add"
	StartTime     time.Time
	EndTime       time.Time `bson:",omitempty"`
	Successful    bool
	Error         string `bson:",omitempty"`
}

func autoScaleCollection() (*storage.Collection, error) {
	conn, err := db.Conn()
	if err != nil {
		return nil, err
	}
	name, err := config.GetString("docker:collection")
	if err != nil {
		return nil, err
	}
	return conn.Collection(fmt.Sprintf("%s_auto_scale", name)), nil
}

var errAutoScaleRunning = errors.New("autoscale already running")

func newAutoScaleEvent(metadataValue, action string) (*autoScaleEvent, error) {
	// Use metadataValue as ID to ensure only one auto scale process runs for
	// each metadataValue. (*autoScaleEvent).update() will generate a new
	// unique ID and remove this initial record.
	evt := autoScaleEvent{
		ID:            metadataValue,
		StartTime:     time.Now().UTC(),
		MetadataValue: metadataValue,
		Action:        action,
	}
	coll, err := autoScaleCollection()
	if err != nil {
		return nil, err
	}
	defer coll.Close()
	err = coll.Insert(evt)
	if mgo.IsDup(err) {
		return nil, errAutoScaleRunning
	}
	return &evt, err
}

func (evt *autoScaleEvent) update(err error) error {
	if err != nil {
		evt.Error = err.Error()
	}
	evt.Successful = err == nil
	evt.EndTime = time.Now().UTC()
	coll, err := autoScaleCollection()
	if err != nil {
		return err
	}
	defer coll.Close()
	defer coll.RemoveId(evt.ID)
	evt.ID = bson.NewObjectId()
	return coll.Insert(evt)
}

func listAutoScaleEvents(skip, limit int) ([]autoScaleEvent, error) {
	coll, err := autoScaleCollection()
	if err != nil {
		return nil, err
	}
	query := coll.Find(nil).Sort("-_id")
	if skip != 0 {
		query = query.Skip(skip)
	}
	if limit != 0 {
		query = query.Limit(limit)
	}
	var list []autoScaleEvent
	err = query.All(&list)
	if err != nil {
		return nil, err
	}
	return list, nil
}

type autoScaleConfig struct {
	provisioner         *dockerProvisioner
	matadataFilter      string
	groupByMetadata     string
	totalMemoryMetadata string
	maxMemoryRatio      float32
	maxContainerCount   int
	waitTimeNewMachine  time.Duration
	runInterval         time.Duration
	done                chan bool
}

type autoScaler interface {
	scale(groupMetadata string, nodes []*cluster.Node) (*autoScaleEvent, error)
}

type memoryScaler struct {
	*autoScaleConfig
}

type countScaler struct {
	*autoScaleConfig
}

type metaWithFrequency struct {
	metadata map[string]string
	freq     int
}

type metaWithFrequencyList []metaWithFrequency

func (l metaWithFrequencyList) Len() int           { return len(l) }
func (l metaWithFrequencyList) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
func (l metaWithFrequencyList) Less(i, j int) bool { return l[i].freq < l[j].freq }

func (a *autoScaleConfig) run() error {
	isMemoryBased := a.totalMemoryMetadata != "" && a.maxMemoryRatio != 0
	if !isMemoryBased && a.maxContainerCount == 0 {
		err := fmt.Errorf("[node autoscale] aborting node auto scale, either memory information or max container count must be informed in config")
		log.Error(err.Error())
		return err
	}
	oneMinute := 1 * time.Minute
	if a.runInterval < oneMinute {
		a.runInterval = oneMinute
	}
	if a.waitTimeNewMachine < oneMinute {
		a.waitTimeNewMachine = oneMinute
	}
	var scaler autoScaler
	if isMemoryBased {
		scaler = &memoryScaler{a}
	} else {
		scaler = &countScaler{a}
	}
	for {
		err := a.runOnce(scaler)
		if err != nil {
			err = fmt.Errorf("[node autoscale] %s", err.Error())
			log.Error(err.Error())
		}
		select {
		case <-a.done:
			return err
		case <-time.After(a.runInterval):
		}
	}
}

func (a *autoScaleConfig) stop() {
	a.done <- true
}

func (a *autoScaleConfig) runOnce(scaler autoScaler) (retErr error) {
	defer func() {
		if r := recover(); r != nil {
			retErr = fmt.Errorf("recovered panic, we can never stop! panic: %v", r)
		}
	}()
	nodes, err := a.provisioner.getCluster().Nodes()
	if err != nil {
		retErr = fmt.Errorf("error getting nodes: %s", err.Error())
		return
	}
	clusterMap := map[string][]*cluster.Node{}
	for i := range nodes {
		node := &nodes[i]
		if a.groupByMetadata == "" {
			clusterMap[""] = append(clusterMap[""], node)
			continue
		}
		groupMetadata := node.Metadata[a.groupByMetadata]
		if groupMetadata == "" {
			log.Debugf("[node autoscale] skipped node %s, no metadata value for %s.", node.Address, a.groupByMetadata)
			continue
		}
		if a.matadataFilter != "" && a.matadataFilter != groupMetadata {
			continue
		}
		clusterMap[groupMetadata] = append(clusterMap[groupMetadata], node)
	}
	for groupMetadata, nodes := range clusterMap {
		var rebalanceFilter map[string]string
		if a.groupByMetadata != "" {
			rebalanceFilter = map[string]string{a.groupByMetadata: groupMetadata}
		}
		event, err := scaler.scale(groupMetadata, nodes)
		if err != nil {
			if err == errAutoScaleRunning {
				log.Debugf("[node autoscale] skipping already running for: %s", groupMetadata)
				continue
			}
			retErr = fmt.Errorf("error scaling group %s: %s", groupMetadata, err.Error())
			return
		}
		if event == nil {
			event, err = a.createRebalanceEvent(groupMetadata, nodes, rebalanceFilter)
			if err != nil {
				log.Errorf("[node autoscale] unable to create rebalance event: %s", err.Error())
			}
		}
		if event != nil {
			buf := safe.NewBuffer(nil)
			_, err := a.provisioner.rebalanceContainersByFilter(buf, nil, rebalanceFilter, false)
			if err != nil {
				log.Errorf("[node autoscale] unable to rebalance containers: %s - log: %s", err.Error(), buf.String())
			}
			event.update(err)
		}
	}
	return
}

func (a *memoryScaler) scale(groupMetadata string, nodes []*cluster.Node) (*autoScaleEvent, error) {
	return nil, nil
}

func (a *countScaler) scale(groupMetadata string, nodes []*cluster.Node) (*autoScaleEvent, error) {
	totalCount, _, err := a.provisioner.containerGapInNodes(nodes)
	if err != nil {
		return nil, fmt.Errorf("couldn't find containers from nodes: %s", err)
	}
	freeSlots := (len(nodes) * a.maxContainerCount) - totalCount
	if freeSlots >= 0 {
		return nil, nil
	}
	event, err := newAutoScaleEvent(groupMetadata, "add")
	if err != nil {
		return nil, err
	}
	log.Debugf("[node autoscale] adding a new machine, metadata value: %s, free slots: %d", groupMetadata, freeSlots)
	err = a.addNode(nodes)
	if err != nil {
		event.update(err)
		return nil, err
	}
	return event, nil
}

func (a *autoScaleConfig) createRebalanceEvent(groupMetadata string, nodes []*cluster.Node, rebalanceFilter map[string]string) (*autoScaleEvent, error) {
	_, gap, err := a.provisioner.containerGapInNodes(nodes)
	buf := safe.NewBuffer(nil)
	dryProvisioner, err := a.provisioner.rebalanceContainersByFilter(buf, nil, rebalanceFilter, true)
	if err != nil {
		return nil, fmt.Errorf("unable to run dry rebalance to check if rebalance is needed: %s - log: %s", err, buf.String())
	}
	_, gapAfter, err := dryProvisioner.containerGapInNodes(nodes)
	if err != nil {
		return nil, fmt.Errorf("couldn't find containers from rebalanced nodes: %s", err)
	}
	if math.Abs((float64)(gap-gapAfter)) <= 2.0 {
		return nil, nil
	}
	log.Debugf("[node autoscale] running rebalance, metadata value: %s, gap before: %d, gap after: %d", groupMetadata, gap, gapAfter)
	event, err := newAutoScaleEvent(groupMetadata, "rebalance")
	if err != nil {
		if err == errAutoScaleRunning {
			log.Debugf("[node autoscale] skipping already running for: %s", groupMetadata)
			return nil, nil
		}
		return nil, fmt.Errorf("unable to create auto scale event: %s", err)
	}
	return event, nil
}

func (a *autoScaleConfig) addNode(modelNodes []*cluster.Node) error {
	metadata, err := chooseMetadataFromNodes(modelNodes)
	if err != nil {
		return err
	}
	_, hasIaas := metadata["iaas"]
	if !hasIaas {
		return fmt.Errorf("no IaaS information in nodes metadata: %#v", metadata)
	}
	machine, err := iaas.CreateMachineForIaaS(metadata["iaas"], metadata)
	if err != nil {
		return fmt.Errorf("unable to create machine: %s", err.Error())
	}
	newAddr := machine.FormatNodeAddress()
	log.Debugf("[node autoscale] new machine created: %s - Waiting for docker to start...", newAddr)
	_, err = a.provisioner.getCluster().WaitAndRegister(newAddr, metadata, a.waitTimeNewMachine)
	if err != nil {
		machine.Destroy()
		return fmt.Errorf("error registering new node %s: %s", newAddr, err.Error())
	}
	log.Debugf("[node autoscale] new machine created: %s - started!", newAddr)
	return nil
}

func splitMetadata(nodesMetadata []map[string]string) (metaWithFrequencyList, map[string]string, error) {
	common := make(map[string]string)
	exclusive := make([]map[string]string, len(nodesMetadata))
	for i := range nodesMetadata {
		metadata := nodesMetadata[i]
		for k, v := range metadata {
			isExclusive := false
			for j := range nodesMetadata {
				if i == j {
					continue
				}
				otherMetadata := nodesMetadata[j]
				if v != otherMetadata[k] {
					isExclusive = true
					break
				}
			}
			if isExclusive {
				if exclusive[i] == nil {
					exclusive[i] = make(map[string]string)
				}
				exclusive[i][k] = v
			} else {
				common[k] = v
			}
		}
	}
	var group metaWithFrequencyList
	sameMap := make(map[int]bool)
	for i := range exclusive {
		freq := 1
		for j := range exclusive {
			if i == j {
				continue
			}
			diffCount := 0
			for k, v := range exclusive[i] {
				if exclusive[j][k] != v {
					diffCount++
				}
			}
			if diffCount > 0 && diffCount < len(exclusive[i]) {
				return nil, nil, fmt.Errorf("unbalanced metadata for node group: %v vs %v", exclusive[i], exclusive[j])
			}
			if diffCount == 0 {
				sameMap[j] = true
				freq++
			}
		}
		if !sameMap[i] && exclusive[i] != nil {
			group = append(group, metaWithFrequency{metadata: exclusive[i], freq: freq})
		}
	}
	return group, common, nil
}

func chooseMetadataFromNodes(modelNodes []*cluster.Node) (map[string]string, error) {
	metadataList := make([]map[string]string, len(modelNodes))
	for i, n := range modelNodes {
		metadataList[i] = n.CleanMetadata()
	}
	exclusiveList, baseMetadata, err := splitMetadata(metadataList)
	if err != nil {
		return nil, err
	}
	var chosenExclusive map[string]string
	if exclusiveList != nil {
		sort.Sort(exclusiveList)
		chosenExclusive = exclusiveList[0].metadata
	}
	for k, v := range chosenExclusive {
		baseMetadata[k] = v
	}
	return baseMetadata, nil
}

func (p *dockerProvisioner) containerGapInNodes(nodes []*cluster.Node) (int, int, error) {
	maxCount := 0
	minCount := 0
	totalCount := 0
	for _, n := range nodes {
		contCount, err := p.countRunningContainersByHost(urlToHost(n.Address))
		if err != nil {
			return 0, 0, err
		}
		if contCount > maxCount {
			maxCount = contCount
		}
		if minCount == 0 || contCount < minCount {
			minCount = contCount
		}
		totalCount += contCount
	}
	return totalCount, maxCount - minCount, nil
}
<|endoftext|>


<commit_before>package ansible

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"strings"

	"github.com/mitchellh/packer/packer"
	"golang.org/x/crypto/ssh"
)

// An adapter satisfies SSH requests (from an Ansible client) by delegating SSH
// exec and subsystem commands to a packer.Communicator.
type adapter struct {
	done    <-chan struct{}
	l       net.Listener
	config  *ssh.ServerConfig
	sftpCmd string
	ui      packer.Ui
	comm    packer.Communicator
}

func newAdapter(done <-chan struct{}, l net.Listener, config *ssh.ServerConfig, sftpCmd string, ui packer.Ui, comm packer.Communicator) *adapter {
	return &adapter{
		done:    done,
		l:       l,
		config:  config,
		sftpCmd: sftpCmd,
		ui:      ui,
		comm:    comm,
	}
}

func (c *adapter) Serve() {
	c.ui.Say(fmt.Sprintf("SSH proxy: serving on %s", c.l.Addr()))

	for {
		// Accept will return if either the underlying connection is closed or if a connection is made.
		// after returning, check to see if c.done can be received. If so, then Accept() returned because
		// the connection has been closed.
		conn, err := c.l.Accept()
		select {
		case <-c.done:
			return
		default:
			if err != nil {
				c.ui.Error(fmt.Sprintf("listen.Accept failed: %v", err))
				continue
			}
			go func(conn net.Conn) {
				if err := c.Handle(conn, c.ui); err != nil {
					c.ui.Error(err.Error())
				}
			}(conn)
		}
	}
}

func (c *adapter) Handle(conn net.Conn, ui packer.Ui) error {
	c.ui.Message("SSH proxy: accepted connection")
	_, chans, reqs, err := ssh.NewServerConn(conn, c.config)
	if err != nil {
		return errors.New("failed to handshake")
	}

	// discard all global requests
	go ssh.DiscardRequests(reqs)

	// Service the incoming NewChannels
	for newChannel := range chans {
		if newChannel.ChannelType() != "session" {
			newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}

		go func(ch ssh.NewChannel) {
			if err := c.handleSession(ch); err != nil {
				c.ui.Error(err.Error())
			}
		}(newChannel)
	}

	return nil
}

func (c *adapter) handleSession(newChannel ssh.NewChannel) error {
	channel, requests, err := newChannel.Accept()
	if err != nil {
		return err
	}
	defer channel.Close()

	done := make(chan struct{})

	// Sessions have requests such as "pty-req", "shell", "env", and "exec".
	// see RFC 4254, section 6
	go func(in <-chan *ssh.Request) {
		env := make([]envRequestPayload, 4)
		for req := range in {
			switch req.Type {
			case "pty-req":
				log.Println("ansible provisioner pty-req request")
				// accept pty-req requests, but don't actually do anything. Necessary for OpenSSH and sudo.
				req.Reply(true, nil)

			case "env":
				req, err := newEnvRequest(req)
				if err != nil {
					c.ui.Error(err.Error())
					req.Reply(false, nil)
					continue
				}
				env = append(env, req.Payload)
				log.Printf("new env request: %s", req.Payload)
				req.Reply(true, nil)
			case "exec":
				req, err := newExecRequest(req)
				if err != nil {
					c.ui.Error(err.Error())
					req.Reply(false, nil)
					close(done)
					continue
				}

				log.Printf("new exec request: %s", req.Payload)

				if len(req.Payload) == 0 {
					req.Reply(false, nil)
					close(done)
					return
				}

				go func(channel ssh.Channel) {
					exit := c.exec(string(req.Payload), channel, channel, channel.Stderr())

					exitStatus := make([]byte, 4)
					binary.BigEndian.PutUint32(exitStatus, uint32(exit))
					channel.SendRequest("exit-status", false, exitStatus)
					close(done)
				}(channel)
				req.Reply(true, nil)
			case "subsystem":
				req, err := newSubsystemRequest(req)
				if err != nil {
					c.ui.Error(err.Error())
					req.Reply(false, nil)
					continue
				}

				log.Printf("new subsystem request: %s", req.Payload)
				switch req.Payload {
				case "sftp":
					sftpCmd := c.sftpCmd
					if len(sftpCmd) == 0 {
						sftpCmd = "/usr/lib/sftp-server -e"
					}
					cmd := &packer.RemoteCmd{
						Stdin:   channel,
						Stdout:  channel,
						Stderr:  channel.Stderr(),
						Command: sftpCmd,
					}

					c.ui.Say("starting sftp subsystem")
					if err := c.comm.Start(cmd); err != nil {
						c.ui.Error(err.Error())
						req.Reply(false, nil)
						close(done)
						return
					}

					req.Reply(true, nil)
					go func() {
						cmd.Wait()
						close(done)
					}()
				default:
					c.ui.Error(fmt.Sprintf("unsupported subsystem requested: %s", req.Payload))
					req.Reply(false, nil)
				}
			default:
				c.ui.Message(fmt.Sprintf("rejecting %s request", req.Type))
				req.Reply(false, nil)
			}
		}
	}(requests)

	<-done
	return nil
}

func (c *adapter) Shutdown() {
	c.l.Close()
}

func (c *adapter) exec(command string, in io.Reader, out io.Writer, err io.Writer) int {
	var exitStatus int
	switch {
	case strings.HasPrefix(command, "scp ") && serveSCP(command[4:]):
		err := c.scpExec(command[4:], in, out, err)
		if err != nil {
			log.Println(err)
			exitStatus = 1
		}
	default:
		exitStatus = c.remoteExec(command, in, out, err)
	}
	return exitStatus
}

func serveSCP(args string) bool {
	opts, _ := scpOptions(args)
	return bytes.IndexAny(opts, "tf") >= 0
}

func (c *adapter) scpExec(args string, in io.Reader, out io.Writer, err io.Writer) error {
	opts, rest := scpOptions(args)

	if i := bytes.IndexByte(opts, 't'); i >= 0 {
		return scpUploadSession(opts, rest, in, out, c.comm)
	}

	if i := bytes.IndexByte(opts, 'f'); i >= 0 {
		return scpDownloadSession(opts, rest, in, out, c.comm)
	}
	return errors.New("no scp mode specified")
}

func (c *adapter) remoteExec(command string, in io.Reader, out io.Writer, err io.Writer) int {
	cmd := &packer.RemoteCmd{
		Stdin:   in,
		Stdout:  out,
		Stderr:  err,
		Command: command,
	}

	if err := c.comm.Start(cmd); err != nil {
		c.ui.Error(err.Error())
		return cmd.ExitStatus
	}

	cmd.Wait()

	return cmd.ExitStatus
}

type envRequest struct {
	*ssh.Request
	Payload envRequestPayload
}

type envRequestPayload struct {
	Name  string
	Value string
}

func (p envRequestPayload) String() string {
	return fmt.Sprintf("%s=%s", p.Name, p.Value)
}

func newEnvRequest(raw *ssh.Request) (*envRequest, error) {
	r := new(envRequest)
	r.Request = raw

	if err := ssh.Unmarshal(raw.Payload, &r.Payload); err != nil {
		return nil, err
	}

	return r, nil
}

func sshString(buf io.Reader) (string, error) {
	var size uint32
	err := binary.Read(buf, binary.BigEndian, &size)
	if err != nil {
		return "", err
	}

	b := make([]byte, size)
	err = binary.Read(buf, binary.BigEndian, b)
	if err != nil {
		return "", err
	}
	return string(b), nil
}

type execRequest struct {
	*ssh.Request
	Payload execRequestPayload
}

type execRequestPayload string

func (p execRequestPayload) String() string {
	return string(p)
}

func newExecRequest(raw *ssh.Request) (*execRequest, error) {
	r := new(execRequest)
	r.Request = raw
	buf := bytes.NewReader(r.Request.Payload)

	var err error
	var payload string
	if payload, err = sshString(buf); err != nil {
		return nil, err
	}

	r.Payload = execRequestPayload(payload)
	return r, nil
}

type subsystemRequest struct {
	*ssh.Request
	Payload subsystemRequestPayload
}

type subsystemRequestPayload string

func (p subsystemRequestPayload) String() string {
	return string(p)
}

func newSubsystemRequest(raw *ssh.Request) (*subsystemRequest, error) {
	r := new(subsystemRequest)
	r.Request = raw
	buf := bytes.NewReader(r.Request.Payload)

	var err error
	var payload string
	if payload, err = sshString(buf); err != nil {
		return nil, err
	}

	r.Payload = subsystemRequestPayload(payload)
	return r, nil
}
<commit_msg>refactor sftp subsystem request handling<commit_after>package ansible

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"strings"

	"github.com/mitchellh/packer/packer"
	"golang.org/x/crypto/ssh"
)

// An adapter satisfies SSH requests (from an Ansible client) by delegating SSH
// exec and subsystem commands to a packer.Communicator.
type adapter struct {
	done    <-chan struct{}
	l       net.Listener
	config  *ssh.ServerConfig
	sftpCmd string
	ui      packer.Ui
	comm    packer.Communicator
}

func newAdapter(done <-chan struct{}, l net.Listener, config *ssh.ServerConfig, sftpCmd string, ui packer.Ui, comm packer.Communicator) *adapter {
	return &adapter{
		done:    done,
		l:       l,
		config:  config,
		sftpCmd: sftpCmd,
		ui:      ui,
		comm:    comm,
	}
}

func (c *adapter) Serve() {
	c.ui.Say(fmt.Sprintf("SSH proxy: serving on %s", c.l.Addr()))

	for {
		// Accept will return if either the underlying connection is closed or if a connection is made.
		// after returning, check to see if c.done can be received. If so, then Accept() returned because
		// the connection has been closed.
		conn, err := c.l.Accept()
		select {
		case <-c.done:
			return
		default:
			if err != nil {
				c.ui.Error(fmt.Sprintf("listen.Accept failed: %v", err))
				continue
			}
			go func(conn net.Conn) {
				if err := c.Handle(conn, c.ui); err != nil {
					c.ui.Error(err.Error())
				}
			}(conn)
		}
	}
}

func (c *adapter) Handle(conn net.Conn, ui packer.Ui) error {
	c.ui.Message("SSH proxy: accepted connection")
	_, chans, reqs, err := ssh.NewServerConn(conn, c.config)
	if err != nil {
		return errors.New("failed to handshake")
	}

	// discard all global requests
	go ssh.DiscardRequests(reqs)

	// Service the incoming NewChannels
	for newChannel := range chans {
		if newChannel.ChannelType() != "session" {
			newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}

		go func(ch ssh.NewChannel) {
			if err := c.handleSession(ch); err != nil {
				c.ui.Error(err.Error())
			}
		}(newChannel)
	}

	return nil
}

func (c *adapter) handleSession(newChannel ssh.NewChannel) error {
	channel, requests, err := newChannel.Accept()
	if err != nil {
		return err
	}
	defer channel.Close()

	done := make(chan struct{})

	// Sessions have requests such as "pty-req", "shell", "env", and "exec".
	// see RFC 4254, section 6
	go func(in <-chan *ssh.Request) {
		env := make([]envRequestPayload, 4)
		for req := range in {
			switch req.Type {
			case "pty-req":
				log.Println("ansible provisioner pty-req request")
				// accept pty-req requests, but don't actually do anything. Necessary for OpenSSH and sudo.
				req.Reply(true, nil)

			case "env":
				req, err := newEnvRequest(req)
				if err != nil {
					c.ui.Error(err.Error())
					req.Reply(false, nil)
					continue
				}
				env = append(env, req.Payload)
				log.Printf("new env request: %s", req.Payload)
				req.Reply(true, nil)
			case "exec":
				req, err := newExecRequest(req)
				if err != nil {
					c.ui.Error(err.Error())
					req.Reply(false, nil)
					close(done)
					continue
				}

				log.Printf("new exec request: %s", req.Payload)

				if len(req.Payload) == 0 {
					req.Reply(false, nil)
					close(done)
					return
				}

				go func(channel ssh.Channel) {
					exit := c.exec(string(req.Payload), channel, channel, channel.Stderr())

					exitStatus := make([]byte, 4)
					binary.BigEndian.PutUint32(exitStatus, uint32(exit))
					channel.SendRequest("exit-status", false, exitStatus)
					close(done)
				}(channel)
				req.Reply(true, nil)
			case "subsystem":
				req, err := newSubsystemRequest(req)
				if err != nil {
					c.ui.Error(err.Error())
					req.Reply(false, nil)
					continue
				}

				log.Printf("new subsystem request: %s", req.Payload)
				switch req.Payload {
				case "sftp":
					sftpCmd := c.sftpCmd
					if len(sftpCmd) == 0 {
						sftpCmd = "/usr/lib/sftp-server -e"
					}

					c.ui.Say("starting sftp subsystem")
					go func() {
						_ = c.remoteExec(sftpCmd, channel, channel, channel.Stderr())
						close(done)
					}()
					req.Reply(true, nil)
				default:
					c.ui.Error(fmt.Sprintf("unsupported subsystem requested: %s", req.Payload))
					req.Reply(false, nil)
				}
			default:
				c.ui.Message(fmt.Sprintf("rejecting %s request", req.Type))
				req.Reply(false, nil)
			}
		}
	}(requests)

	<-done
	return nil
}

func (c *adapter) Shutdown() {
	c.l.Close()
}

func (c *adapter) exec(command string, in io.Reader, out io.Writer, err io.Writer) int {
	var exitStatus int
	switch {
	case strings.HasPrefix(command, "scp ") && serveSCP(command[4:]):
		err := c.scpExec(command[4:], in, out, err)
		if err != nil {
			log.Println(err)
			exitStatus = 1
		}
	default:
		exitStatus = c.remoteExec(command, in, out, err)
	}
	return exitStatus
}

func serveSCP(args string) bool {
	opts, _ := scpOptions(args)
	return bytes.IndexAny(opts, "tf") >= 0
}

func (c *adapter) scpExec(args string, in io.Reader, out io.Writer, err io.Writer) error {
	opts, rest := scpOptions(args)

	if i := bytes.IndexByte(opts, 't'); i >= 0 {
		return scpUploadSession(opts, rest, in, out, c.comm)
	}

	if i := bytes.IndexByte(opts, 'f'); i >= 0 {
		return scpDownloadSession(opts, rest, in, out, c.comm)
	}
	return errors.New("no scp mode specified")
}

func (c *adapter) remoteExec(command string, in io.Reader, out io.Writer, err io.Writer) int {
	cmd := &packer.RemoteCmd{
		Stdin:   in,
		Stdout:  out,
		Stderr:  err,
		Command: command,
	}

	if err := c.comm.Start(cmd); err != nil {
		c.ui.Error(err.Error())
		return cmd.ExitStatus
	}

	cmd.Wait()

	return cmd.ExitStatus
}

type envRequest struct {
	*ssh.Request
	Payload envRequestPayload
}

type envRequestPayload struct {
	Name  string
	Value string
}

func (p envRequestPayload) String() string {
	return fmt.Sprintf("%s=%s", p.Name, p.Value)
}

func newEnvRequest(raw *ssh.Request) (*envRequest, error) {
	r := new(envRequest)
	r.Request = raw

	if err := ssh.Unmarshal(raw.Payload, &r.Payload); err != nil {
		return nil, err
	}

	return r, nil
}

func sshString(buf io.Reader) (string, error) {
	var size uint32
	err := binary.Read(buf, binary.BigEndian, &size)
	if err != nil {
		return "", err
	}

	b := make([]byte, size)
	err = binary.Read(buf, binary.BigEndian, b)
	if err != nil {
		return "", err
	}
	return string(b), nil
}

type execRequest struct {
	*ssh.Request
	Payload execRequestPayload
}

type execRequestPayload string

func (p execRequestPayload) String() string {
	return string(p)
}

func newExecRequest(raw *ssh.Request) (*execRequest, error) {
	r := new(execRequest)
	r.Request = raw
	buf := bytes.NewReader(r.Request.Payload)

	var err error
	var payload string
	if payload, err = sshString(buf); err != nil {
		return nil, err
	}

	r.Payload = execRequestPayload(payload)
	return r, nil
}

type subsystemRequest struct {
	*ssh.Request
	Payload subsystemRequestPayload
}

type subsystemRequestPayload string

func (p subsystemRequestPayload) String() string {
	return string(p)
}

func newSubsystemRequest(raw *ssh.Request) (*subsystemRequest, error) {
	r := new(subsystemRequest)
	r.Request = raw
	buf := bytes.NewReader(r.Request.Payload)

	var err error
	var payload string
	if payload, err = sshString(buf); err != nil {
		return nil, err
	}

	r.Payload = subsystemRequestPayload(payload)
	return r, nil
}
<|endoftext|>


<commit_before>package cli

import (
	"context"
	"flag"
	"fmt"
	"io"
	"io/ioutil"

	"github.com/elpinal/coco3/config"
	"github.com/elpinal/coco3/eval"
	"github.com/elpinal/coco3/gate"
	"github.com/elpinal/coco3/parser"
)

type CLI struct {
	In  io.Reader
	Out io.Writer
	Err io.Writer

	config.Config

	exitCh chan int
	doneCh chan struct{} // to ensure exiting just after exitCh received
}

func (c *CLI) Run(args []string) int {
	c.exitCh = make(chan int)
	c.doneCh = make(chan struct{})

	f := flag.NewFlagSet("coco3", flag.ContinueOnError)
	f.SetOutput(c.Err)
	f.Usage = func() {
		c.Err.Write([]byte("coco3 is a shell.\n"))
		c.Err.Write([]byte("Usage:\n"))
		f.PrintDefaults()
	}

	flagC := f.String("c", "", "take first argument as a command to execute")
	if err := f.Parse(args); err != nil {
		return 2
	}

	if len(c.Config.StartUpCommand) > 0 {
		done := make(chan struct{})
		go func() {
			if err := c.execute(c.Config.StartUpCommand); err != nil {
				fmt.Fprintln(c.Err, err)
				c.exitCh <- 1
			}
			close(done)
		}()
		select {
		case code := <-c.exitCh:
			return code
		case <-done:
		}
	}

	if *flagC != "" {
		go func() {
			if err := c.execute([]byte(*flagC)); err != nil {
				fmt.Fprintln(c.Err, err)
				c.exitCh <- 1
				return
			}
			c.exitCh <- 0
		}()
		return <-c.exitCh
	}

	if len(f.Args()) > 0 {
		defer func() {
			close(c.doneCh)
		}()
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		go c.runFiles(ctx, f.Args())
		return <-c.exitCh
	}

	conf := &c.Config
	conf.Init()
	g := gate.New(conf, c.In, c.Out, c.Err)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func(ctx context.Context) {
		for {
			if err := c.interact(ctx, g); err != nil {
				fmt.Fprintln(c.Err, err)
				g.Clear()
			}
			select {
			case <-ctx.Done():
				return
			default:
			}
		}
	}(ctx)
	return <-c.exitCh
}

func (c *CLI) interact(ctx context.Context, g gate.Gate) error {
	for {
		old, err := enterRowMode()
		if err != nil {
			return err
		}
		r, err := g.Read()
		if err != nil {
			return err
		}
		c.Out.Write([]byte{'\n'})
		if err := exitRowMode(old); err != nil {
			return err
		}
		if err := c.execute([]byte(string(r))); err != nil {
			return err
		}
		g.Clear()
		select {
		case <-ctx.Done():
			return nil
		default:
		}
	}
}

func (c *CLI) execute(b []byte) error {
	f, err := parser.ParseSrc(b)
	if err != nil {
		return err
	}
	e := eval.New(c.In, c.Out, c.Err)
	err = e.Eval(f.Lines)
	select {
	case code := <-e.ExitCh:
		c.exitCh <- code
		<-c.doneCh
	default:
	}
	return err
}

func (c *CLI) runFiles(ctx context.Context, files []string) {
	for _, file := range files {
		b, err := ioutil.ReadFile(file)
		if err != nil {
			fmt.Fprintln(c.Err, err)
			c.exitCh <- 1
			return
		}
		if err := c.execute(b); err != nil {
			fmt.Fprintln(c.Err, err)
			c.exitCh <- 1
			return
		}
		select {
		case <-ctx.Done():
			return
		default:
		}
	}
	c.exitCh <- 0
}
<commit_msg>Close c.doneCh every time exiting from CLI.Run<commit_after>package cli

import (
	"context"
	"flag"
	"fmt"
	"io"
	"io/ioutil"

	"github.com/elpinal/coco3/config"
	"github.com/elpinal/coco3/eval"
	"github.com/elpinal/coco3/gate"
	"github.com/elpinal/coco3/parser"
)

type CLI struct {
	In  io.Reader
	Out io.Writer
	Err io.Writer

	config.Config

	exitCh chan int
	doneCh chan struct{} // to ensure exiting just after exitCh received
}

func (c *CLI) Run(args []string) int {
	c.exitCh = make(chan int)
	c.doneCh = make(chan struct{})
	defer func() {
		close(c.doneCh)
	}()

	f := flag.NewFlagSet("coco3", flag.ContinueOnError)
	f.SetOutput(c.Err)
	f.Usage = func() {
		c.Err.Write([]byte("coco3 is a shell.\n"))
		c.Err.Write([]byte("Usage:\n"))
		f.PrintDefaults()
	}

	flagC := f.String("c", "", "take first argument as a command to execute")
	if err := f.Parse(args); err != nil {
		return 2
	}

	if len(c.Config.StartUpCommand) > 0 {
		done := make(chan struct{})
		go func() {
			if err := c.execute(c.Config.StartUpCommand); err != nil {
				fmt.Fprintln(c.Err, err)
				c.exitCh <- 1
			}
			close(done)
		}()
		select {
		case code := <-c.exitCh:
			return code
		case <-done:
		}
	}

	if *flagC != "" {
		go func() {
			if err := c.execute([]byte(*flagC)); err != nil {
				fmt.Fprintln(c.Err, err)
				c.exitCh <- 1
				return
			}
			c.exitCh <- 0
		}()
		return <-c.exitCh
	}

	if len(f.Args()) > 0 {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		go c.runFiles(ctx, f.Args())
		return <-c.exitCh
	}

	conf := &c.Config
	conf.Init()
	g := gate.New(conf, c.In, c.Out, c.Err)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func(ctx context.Context) {
		for {
			if err := c.interact(ctx, g); err != nil {
				fmt.Fprintln(c.Err, err)
				g.Clear()
			}
			select {
			case <-ctx.Done():
				return
			default:
			}
		}
	}(ctx)
	return <-c.exitCh
}

func (c *CLI) interact(ctx context.Context, g gate.Gate) error {
	for {
		old, err := enterRowMode()
		if err != nil {
			return err
		}
		r, err := g.Read()
		if err != nil {
			return err
		}
		c.Out.Write([]byte{'\n'})
		if err := exitRowMode(old); err != nil {
			return err
		}
		if err := c.execute([]byte(string(r))); err != nil {
			return err
		}
		g.Clear()
		select {
		case <-ctx.Done():
			return nil
		default:
		}
	}
}

func (c *CLI) execute(b []byte) error {
	f, err := parser.ParseSrc(b)
	if err != nil {
		return err
	}
	e := eval.New(c.In, c.Out, c.Err)
	err = e.Eval(f.Lines)
	select {
	case code := <-e.ExitCh:
		c.exitCh <- code
		<-c.doneCh
	default:
	}
	return err
}

func (c *CLI) runFiles(ctx context.Context, files []string) {
	for _, file := range files {
		b, err := ioutil.ReadFile(file)
		if err != nil {
			fmt.Fprintln(c.Err, err)
			c.exitCh <- 1
			return
		}
		if err := c.execute(b); err != nil {
			fmt.Fprintln(c.Err, err)
			c.exitCh <- 1
			return
		}
		select {
		case <-ctx.Done():
			return
		default:
		}
	}
	c.exitCh <- 0
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cli

import (
	"fmt"
	"math/rand"
	"os"
	"time"

	"github.com/spf13/cobra"

	cliflag "k8s.io/component-base/cli/flag"
	"k8s.io/component-base/logs"
	"k8s.io/klog/v2"
)

// Run provides the common boilerplate code around executing a cobra command.
// For example, it ensures that logging is set up properly. Logging
// flags get added to the command line if not added already. Flags get normalized
// so that help texts show them with hyphens. Underscores are accepted
// as alternative for the command parameters.
func Run(cmd *cobra.Command) int {
	rand.Seed(time.Now().UnixNano())
	defer logs.FlushLogs()

	cmd.SetGlobalNormalizationFunc(cliflag.WordSepNormalizeFunc)

	// When error printing is enabled for the Cobra command, a flag parse
	// error gets printed first, then optionally the often long usage
	// text. This is very unreadable in a console because the last few
	// lines that will be visible on screen don't include the error.
	//
	// The recommendation from #sig-cli was to print the usage text, then
	// the error. We implement this consistently for all commands here.
	// However, we don't want to print the usage text when command
	// execution fails for other reasons than parsing. We detect this via
	// the FlagParseError callback.
	//
	// Some commands, like kubectl, already deal with this themselves.
	// We don't change the behavior for those.
	if !cmd.SilenceUsage {
		cmd.SilenceUsage = true
		cmd.SetFlagErrorFunc(func(c *cobra.Command, err error) error {
			// Re-enable usage printing.
			c.SilenceUsage = false
			return err
		})
	}

	// In all cases error printing is done below.
	cmd.SilenceErrors = true

	// This is idempotent.
	logs.AddFlags(cmd.PersistentFlags())

	// Inject logs.InitLogs after command line parsing into one of the
	// PersistentPre* functions.
	logsInitialized := false
	switch {
	case cmd.PersistentPreRun != nil:
		pre := cmd.PersistentPreRun
		cmd.PersistentPreRun = func(cmd *cobra.Command, args []string) {
			logs.InitLogs()
			logsInitialized = true
			pre(cmd, args)
		}
	case cmd.PersistentPreRunE != nil:
		pre := cmd.PersistentPreRunE
		cmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
			logs.InitLogs()
			logsInitialized = true
			return pre(cmd, args)
		}
	default:
		cmd.PersistentPreRun = func(cmd *cobra.Command, args []string) {
			logs.InitLogs()
			logsInitialized = true
		}
	}

	if err := cmd.Execute(); err != nil {
		// If the error is about flag parsing, then printing that error
		// with the decoration that klog would add ("E0923
		// 23:02:03.219216 4168816 run.go:61] unknown shorthand flag")
		// is less readable. Using klog.Fatal is even worse because it
		// dumps a stack trace that isn't about the error.
		//
		// But if it is some other error encountered at runtime, then
		// we want to log it as error, at least in most commands because
		// their output is a log event stream.
		//
		// We can distinguish these two cases depending on whether
		// we got to logs.InitLogs() above.
		//
		// This heuristic might be problematic for command line
		// tools like kubectl where the output is carefully controlled
		// and not a log by default. It works because kubectl has
		// its own error handling once a command runs.
		//
		// The usage of klog is problematic also because we don't know
		// whether the command has managed to configure it. This cannot
		// be checked right now, but may become possible when the early
		// logging proposal from
		// https://github.com/kubernetes/enhancements/pull/3078
		// ("contextual logging") is implemented.
		if !logsInitialized {
			fmt.Fprintf(os.Stderr, "Error: %v\n", err)
		} else {
			klog.ErrorS(err, "command failed")
		}
		return 1
	}
	return 0
}
<commit_msg>cli: let kubectl handle error printing<commit_after>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cli

import (
	"fmt"
	"math/rand"
	"os"
	"time"

	"github.com/spf13/cobra"

	cliflag "k8s.io/component-base/cli/flag"
	"k8s.io/component-base/logs"
	"k8s.io/klog/v2"
)

// Run provides the common boilerplate code around executing a cobra command.
// For example, it ensures that logging is set up properly. Logging
// flags get added to the command line if not added already. Flags get normalized
// so that help texts show them with hyphens. Underscores are accepted
// as alternative for the command parameters.
//
// Run tries to be smart about how to print errors that are returned by the
// command: before logging is known to be set up, it prints them as plain text
// to stderr. This covers command line flag parse errors and unknown commands.
// Afterwards it logs them. This covers runtime errors.
//
// Commands like kubectl where logging is not normally part of the runtime output
// should use RunNoErrOutput instead and deal with the returned error themselves.
func Run(cmd *cobra.Command) int {
	if logsInitialized, err := run(cmd); err != nil {
		// If the error is about flag parsing, then printing that error
		// with the decoration that klog would add ("E0923
		// 23:02:03.219216 4168816 run.go:61] unknown shorthand flag")
		// is less readable. Using klog.Fatal is even worse because it
		// dumps a stack trace that isn't about the error.
		//
		// But if it is some other error encountered at runtime, then
		// we want to log it as error, at least in most commands because
		// their output is a log event stream.
		//
		// We can distinguish these two cases depending on whether
		// we got to logs.InitLogs() above.
		//
		// This heuristic might be problematic for command line
		// tools like kubectl where the output is carefully controlled
		// and not a log by default. They should use RunNoErrOutput
		// instead.
		//
		// The usage of klog is problematic also because we don't know
		// whether the command has managed to configure it. This cannot
		// be checked right now, but may become possible when the early
		// logging proposal from
		// https://github.com/kubernetes/enhancements/pull/3078
		// ("contextual logging") is implemented.
		if !logsInitialized {
			fmt.Fprintf(os.Stderr, "Error: %v\n", err)
		} else {
			klog.ErrorS(err, "command failed")
		}
		return 1
	}
	return 0
}

// RunNoErrOutput is a version of Run which returns the cobra command error
// instead of printing it.
func RunNoErrOutput(cmd *cobra.Command) error {
	_, err := run(cmd)
	return err
}

func run(cmd *cobra.Command) (logsInitialized bool, err error) {
	rand.Seed(time.Now().UnixNano())
	defer logs.FlushLogs()

	cmd.SetGlobalNormalizationFunc(cliflag.WordSepNormalizeFunc)

	// When error printing is enabled for the Cobra command, a flag parse
	// error gets printed first, then optionally the often long usage
	// text. This is very unreadable in a console because the last few
	// lines that will be visible on screen don't include the error.
	//
	// The recommendation from #sig-cli was to print the usage text, then
	// the error. We implement this consistently for all commands here.
	// However, we don't want to print the usage text when command
	// execution fails for other reasons than parsing. We detect this via
	// the FlagParseError callback.
	//
	// Some commands, like kubectl, already deal with this themselves.
	// We don't change the behavior for those.
	if !cmd.SilenceUsage {
		cmd.SilenceUsage = true
		cmd.SetFlagErrorFunc(func(c *cobra.Command, err error) error {
			// Re-enable usage printing.
			c.SilenceUsage = false
			return err
		})
	}

	// In all cases error printing is done below.
	cmd.SilenceErrors = true

	// This is idempotent.
	logs.AddFlags(cmd.PersistentFlags())

	// Inject logs.InitLogs after command line parsing into one of the
	// PersistentPre* functions.
	switch {
	case cmd.PersistentPreRun != nil:
		pre := cmd.PersistentPreRun
		cmd.PersistentPreRun = func(cmd *cobra.Command, args []string) {
			logs.InitLogs()
			logsInitialized = true
			pre(cmd, args)
		}
	case cmd.PersistentPreRunE != nil:
		pre := cmd.PersistentPreRunE
		cmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
			logs.InitLogs()
			logsInitialized = true
			return pre(cmd, args)
		}
	default:
		cmd.PersistentPreRun = func(cmd *cobra.Command, args []string) {
			logs.InitLogs()
			logsInitialized = true
		}
	}

	err = cmd.Execute()
	return
}
<|endoftext|>


<commit_before>package shared

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"gopkg.in/robfig/cron.v2"
)

type ContainerAction string

const (
	Stop     ContainerAction = "stop"
	Start    ContainerAction = "start"
	Restart  ContainerAction = "restart"
	Freeze   ContainerAction = "freeze"
	Unfreeze ContainerAction = "unfreeze"
)

func IsInt64(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s", value)
	}

	return nil
}

func IsUint8(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseUint(value, 10, 8)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s. Must be between 0 and 255", value)
	}

	return nil
}

func IsUint32(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseUint(value, 10, 32)
	if err != nil {
		return fmt.Errorf("Invalid value for uint32: %s: %v", value, err)
	}

	return nil
}

func IsPriority(value string) error {
	if value == "" {
		return nil
	}

	valueInt, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s", value)
	}

	if valueInt < 0 || valueInt > 10 {
		return fmt.Errorf("Invalid value for a limit '%s'. Must be between 0 and 10", value)
	}

	return nil
}

func IsBool(value string) error {
	if value == "" {
		return nil
	}

	if !StringInSlice(strings.ToLower(value), []string{"true", "false", "yes", "no", "1", "0", "on", "off"}) {
		return fmt.Errorf("Invalid value for a boolean: %s", value)
	}

	return nil
}

func IsOneOf(value string, valid []string) error {
	if value == "" {
		return nil
	}

	if !StringInSlice(value, valid) {
		return fmt.Errorf("Invalid value: %s (not one of %s)", value, valid)
	}

	return nil
}

func IsAny(value string) error {
	return nil
}

// IsRootDiskDevice returns true if the given device representation is
// configured as root disk for a container. It typically get passed a specific
// entry of api.Container.Devices.
func IsRootDiskDevice(device map[string]string) bool {
	if device["type"] == "disk" && device["path"] == "/" && device["source"] == "" {
		return true
	}

	return false
}

// GetRootDiskDevice returns the container device that is configured as root disk
func GetRootDiskDevice(devices map[string]map[string]string) (string, map[string]string, error) {
	var devName string
	var dev map[string]string

	for n, d := range devices {
		if IsRootDiskDevice(d) {
			if devName != "" {
				return "", nil, fmt.Errorf("More than one root device found")
			}

			devName = n
			dev = d
		}
	}

	if devName != "" {
		return devName, dev, nil
	}

	return "", nil, fmt.Errorf("No root device could be found")
}

// KnownContainerConfigKeys maps all fully defined, well-known config keys
// to an appropriate checker function, which validates whether or not a
// given value is syntactically legal.
var KnownContainerConfigKeys = map[string]func(value string) error{
	"boot.autostart":             IsBool,
	"boot.autostart.delay":       IsInt64,
	"boot.autostart.priority":    IsInt64,
	"boot.stop.priority":         IsInt64,
	"boot.host_shutdown_timeout": IsInt64,

	"limits.cpu": func(value string) error {
		if value == "" {
			return nil
		}

		// Validate the character set
		match, _ := regexp.MatchString("^[-,0-9]*$", value)
		if !match {
			return fmt.Errorf("Invalid CPU limit syntax")
		}

		// Validate first character
		if strings.HasPrefix(value, "-") || strings.HasPrefix(value, ",") {
			return fmt.Errorf("CPU limit can't start with a separator")
		}

		// Validate last character
		if strings.HasSuffix(value, "-") || strings.HasSuffix(value, ",") {
			return fmt.Errorf("CPU limit can't end with a separator")
		}

		return nil
	},
	"limits.cpu.allowance": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			// Percentage based allocation
			_, err := strconv.Atoi(strings.TrimSuffix(value, "%"))
			if err != nil {
				return err
			}

			return nil
		}

		// Time based allocation
		fields := strings.SplitN(value, "/", 2)
		if len(fields) != 2 {
			return fmt.Errorf("Invalid allowance: %s", value)
		}

		_, err := strconv.Atoi(strings.TrimSuffix(fields[0], "ms"))
		if err != nil {
			return err
		}

		_, err = strconv.Atoi(strings.TrimSuffix(fields[1], "ms"))
		if err != nil {
			return err
		}

		return nil
	},
	"limits.cpu.priority": IsPriority,

	"limits.disk.priority": IsPriority,

	"limits.memory": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			_, err := strconv.ParseInt(strings.TrimSuffix(value, "%"), 10, 64)
			if err != nil {
				return err
			}

			return nil
		}

		_, err := ParseByteSizeString(value)
		if err != nil {
			return err
		}

		return nil
	},
	"limits.memory.enforce": func(value string) error {
		return IsOneOf(value, []string{"soft", "hard"})
	},
	"limits.memory.swap":          IsBool,
	"limits.memory.swap.priority": IsPriority,

	"limits.network.priority": IsPriority,

	"limits.processes": IsInt64,

	"linux.kernel_modules": IsAny,

	"migration.incremental.memory":            IsBool,
	"migration.incremental.memory.iterations": IsUint32,
	"migration.incremental.memory.goal":       IsUint32,

	"nvidia.runtime":             IsBool,
	"nvidia.driver.capabilities": IsAny,
	"nvidia.require.cuda":        IsAny,
	"nvidia.require.driver":      IsAny,

	"security.nesting":       IsBool,
	"security.privileged":    IsBool,
	"security.devlxd":        IsBool,
	"security.devlxd.images": IsBool,

	"security.protection.delete": IsBool,
	"security.protection.shift":  IsBool,

	"security.idmap.base":     IsUint32,
	"security.idmap.isolated": IsBool,
	"security.idmap.size":     IsUint32,

	"security.syscalls.blacklist_default": IsBool,
	"security.syscalls.blacklist_compat":  IsBool,
	"security.syscalls.blacklist":         IsAny,
	"security.syscalls.whitelist":         IsAny,

	"snapshots.schedule": func(value string) error {
		if value == "" {
			return nil
		}

		if len(strings.Split(value, " ")) != 5 {
			return fmt.Errorf("Schedule must be of the form: <minute> <hour> <day-of-month> <month> <day-of-week>")
		}

		_, err := cron.Parse(fmt.Sprintf("* %s", value))
		if err != nil {
			return errors.Wrap(err, "Error parsing schedule")
		}

		return nil
	},
	"snapshots.schedule.stopped": IsBool,
	"snapshots.pattern":          IsAny,
	"snapshots.expiry": func(value string) error {
		// Validate expression
		_, err := GetSnapshotExpiry(time.Time{}, value)
		return err
	},

	// Caller is responsible for full validation of any raw.* value
	"raw.apparmor": IsAny,
	"raw.lxc":      IsAny,
	"raw.seccomp":  IsAny,
	"raw.idmap":    IsAny,

	"volatile.apply_template":   IsAny,
	"volatile.base_image":       IsAny,
	"volatile.last_state.idmap": IsAny,
	"volatile.last_state.power": IsAny,
	"volatile.idmap.base":       IsAny,
	"volatile.idmap.current":    IsAny,
	"volatile.idmap.next":       IsAny,
	"volatile.apply_quota":      IsAny,
}

// ConfigKeyChecker returns a function that will check whether or not
// a provide value is valid for the associate config key.  Returns an
// error if the key is not known.  The checker function only performs
// syntactic checking of the value, semantic and usage checking must
// be done by the caller.  User defined keys are always considered to
// be valid, e.g. user.* and environment.* keys.
func ConfigKeyChecker(key string) (func(value string) error, error) {
	if f, ok := KnownContainerConfigKeys[key]; ok {
		return f, nil
	}

	if strings.HasPrefix(key, "volatile.") {
		if strings.HasSuffix(key, ".hwaddr") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".name") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".host_name") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".mtu") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".created") {
			return IsAny, nil
		}
	}

	if strings.HasPrefix(key, "environment.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "user.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "image.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "limits.kernel.") &&
		(len(key) > len("limits.kernel.")) {
		return IsAny, nil
	}

	return nil, fmt.Errorf("Unknown configuration key: %s", key)
}
<commit_msg>shared/container: Adds volatile key suffixes for SR-IOV<commit_after>package shared

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"gopkg.in/robfig/cron.v2"
)

type ContainerAction string

const (
	Stop     ContainerAction = "stop"
	Start    ContainerAction = "start"
	Restart  ContainerAction = "restart"
	Freeze   ContainerAction = "freeze"
	Unfreeze ContainerAction = "unfreeze"
)

func IsInt64(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s", value)
	}

	return nil
}

func IsUint8(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseUint(value, 10, 8)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s. Must be between 0 and 255", value)
	}

	return nil
}

func IsUint32(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseUint(value, 10, 32)
	if err != nil {
		return fmt.Errorf("Invalid value for uint32: %s: %v", value, err)
	}

	return nil
}

func IsPriority(value string) error {
	if value == "" {
		return nil
	}

	valueInt, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s", value)
	}

	if valueInt < 0 || valueInt > 10 {
		return fmt.Errorf("Invalid value for a limit '%s'. Must be between 0 and 10", value)
	}

	return nil
}

func IsBool(value string) error {
	if value == "" {
		return nil
	}

	if !StringInSlice(strings.ToLower(value), []string{"true", "false", "yes", "no", "1", "0", "on", "off"}) {
		return fmt.Errorf("Invalid value for a boolean: %s", value)
	}

	return nil
}

func IsOneOf(value string, valid []string) error {
	if value == "" {
		return nil
	}

	if !StringInSlice(value, valid) {
		return fmt.Errorf("Invalid value: %s (not one of %s)", value, valid)
	}

	return nil
}

func IsAny(value string) error {
	return nil
}

// IsRootDiskDevice returns true if the given device representation is
// configured as root disk for a container. It typically get passed a specific
// entry of api.Container.Devices.
func IsRootDiskDevice(device map[string]string) bool {
	if device["type"] == "disk" && device["path"] == "/" && device["source"] == "" {
		return true
	}

	return false
}

// GetRootDiskDevice returns the container device that is configured as root disk
func GetRootDiskDevice(devices map[string]map[string]string) (string, map[string]string, error) {
	var devName string
	var dev map[string]string

	for n, d := range devices {
		if IsRootDiskDevice(d) {
			if devName != "" {
				return "", nil, fmt.Errorf("More than one root device found")
			}

			devName = n
			dev = d
		}
	}

	if devName != "" {
		return devName, dev, nil
	}

	return "", nil, fmt.Errorf("No root device could be found")
}

// KnownContainerConfigKeys maps all fully defined, well-known config keys
// to an appropriate checker function, which validates whether or not a
// given value is syntactically legal.
var KnownContainerConfigKeys = map[string]func(value string) error{
	"boot.autostart":             IsBool,
	"boot.autostart.delay":       IsInt64,
	"boot.autostart.priority":    IsInt64,
	"boot.stop.priority":         IsInt64,
	"boot.host_shutdown_timeout": IsInt64,

	"limits.cpu": func(value string) error {
		if value == "" {
			return nil
		}

		// Validate the character set
		match, _ := regexp.MatchString("^[-,0-9]*$", value)
		if !match {
			return fmt.Errorf("Invalid CPU limit syntax")
		}

		// Validate first character
		if strings.HasPrefix(value, "-") || strings.HasPrefix(value, ",") {
			return fmt.Errorf("CPU limit can't start with a separator")
		}

		// Validate last character
		if strings.HasSuffix(value, "-") || strings.HasSuffix(value, ",") {
			return fmt.Errorf("CPU limit can't end with a separator")
		}

		return nil
	},
	"limits.cpu.allowance": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			// Percentage based allocation
			_, err := strconv.Atoi(strings.TrimSuffix(value, "%"))
			if err != nil {
				return err
			}

			return nil
		}

		// Time based allocation
		fields := strings.SplitN(value, "/", 2)
		if len(fields) != 2 {
			return fmt.Errorf("Invalid allowance: %s", value)
		}

		_, err := strconv.Atoi(strings.TrimSuffix(fields[0], "ms"))
		if err != nil {
			return err
		}

		_, err = strconv.Atoi(strings.TrimSuffix(fields[1], "ms"))
		if err != nil {
			return err
		}

		return nil
	},
	"limits.cpu.priority": IsPriority,

	"limits.disk.priority": IsPriority,

	"limits.memory": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			_, err := strconv.ParseInt(strings.TrimSuffix(value, "%"), 10, 64)
			if err != nil {
				return err
			}

			return nil
		}

		_, err := ParseByteSizeString(value)
		if err != nil {
			return err
		}

		return nil
	},
	"limits.memory.enforce": func(value string) error {
		return IsOneOf(value, []string{"soft", "hard"})
	},
	"limits.memory.swap":          IsBool,
	"limits.memory.swap.priority": IsPriority,

	"limits.network.priority": IsPriority,

	"limits.processes": IsInt64,

	"linux.kernel_modules": IsAny,

	"migration.incremental.memory":            IsBool,
	"migration.incremental.memory.iterations": IsUint32,
	"migration.incremental.memory.goal":       IsUint32,

	"nvidia.runtime":             IsBool,
	"nvidia.driver.capabilities": IsAny,
	"nvidia.require.cuda":        IsAny,
	"nvidia.require.driver":      IsAny,

	"security.nesting":       IsBool,
	"security.privileged":    IsBool,
	"security.devlxd":        IsBool,
	"security.devlxd.images": IsBool,

	"security.protection.delete": IsBool,
	"security.protection.shift":  IsBool,

	"security.idmap.base":     IsUint32,
	"security.idmap.isolated": IsBool,
	"security.idmap.size":     IsUint32,

	"security.syscalls.blacklist_default": IsBool,
	"security.syscalls.blacklist_compat":  IsBool,
	"security.syscalls.blacklist":         IsAny,
	"security.syscalls.whitelist":         IsAny,

	"snapshots.schedule": func(value string) error {
		if value == "" {
			return nil
		}

		if len(strings.Split(value, " ")) != 5 {
			return fmt.Errorf("Schedule must be of the form: <minute> <hour> <day-of-month> <month> <day-of-week>")
		}

		_, err := cron.Parse(fmt.Sprintf("* %s", value))
		if err != nil {
			return errors.Wrap(err, "Error parsing schedule")
		}

		return nil
	},
	"snapshots.schedule.stopped": IsBool,
	"snapshots.pattern":          IsAny,
	"snapshots.expiry": func(value string) error {
		// Validate expression
		_, err := GetSnapshotExpiry(time.Time{}, value)
		return err
	},

	// Caller is responsible for full validation of any raw.* value
	"raw.apparmor": IsAny,
	"raw.lxc":      IsAny,
	"raw.seccomp":  IsAny,
	"raw.idmap":    IsAny,

	"volatile.apply_template":   IsAny,
	"volatile.base_image":       IsAny,
	"volatile.last_state.idmap": IsAny,
	"volatile.last_state.power": IsAny,
	"volatile.idmap.base":       IsAny,
	"volatile.idmap.current":    IsAny,
	"volatile.idmap.next":       IsAny,
	"volatile.apply_quota":      IsAny,
}

// ConfigKeyChecker returns a function that will check whether or not
// a provide value is valid for the associate config key.  Returns an
// error if the key is not known.  The checker function only performs
// syntactic checking of the value, semantic and usage checking must
// be done by the caller.  User defined keys are always considered to
// be valid, e.g. user.* and environment.* keys.
func ConfigKeyChecker(key string) (func(value string) error, error) {
	if f, ok := KnownContainerConfigKeys[key]; ok {
		return f, nil
	}

	if strings.HasPrefix(key, "volatile.") {
		if strings.HasSuffix(key, ".hwaddr") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".name") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".host_name") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".mtu") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".created") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".id") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".vlan") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".spoofcheck") {
			return IsAny, nil
		}
	}

	if strings.HasPrefix(key, "environment.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "user.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "image.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "limits.kernel.") &&
		(len(key) > len("limits.kernel.")) {
		return IsAny, nil
	}

	return nil, fmt.Errorf("Unknown configuration key: %s", key)
}
<|endoftext|>


<commit_before>package ff

import (
	"fmt"
)

// Represents the entire commandline for ffmpeg/ffprobe
type Command struct {
	Path    string
	Input   File
	Outputs []File
}

// You can pass in only input (for ffprobe)
// You can pass in one or multiple ouptuts
// Returns a new Command structure
func NewCommand(path string, input File, outputs ...File) (cmd *Command, err error) {
	if path == "" {
		return nil, fmt.Errorf("Cannot create a Command with no path")
	}
	if input == nil {
		return nil, fmt.Errorf("Cannot create a Command with no input")
	}
	cmd = &Command{
		Path:  path,
		Input: input,
	}

	for _, output := range outputs {
		if output != nil {
			cmd.Outputs = append(cmd.Outputs, output)
		}
	}

	return cmd, nil
}

// Returns a []string slice of how the ffmpeg/ffprobe call should be represented
// Does not include the command Path before it
func (c *Command) Slice() (results []string) {
	results = []string{}
	results = append(results, c.Input.Slice()...)
	for _, output := range c.Outputs {
		results = append(results, output.Slice()...)
	}
	return
}
<commit_msg>command: add 'license block' and Doc string for the package.<commit_after>// Copyright 2015 Dave Gradwell
// Under BSD-style license (see LICENSE file)

// Package ff implements an ffmpeg (or ffprobe) command-line parameter builder
package ff

import (
	"fmt"
)

// Represents the entire commandline for ffmpeg/ffprobe
type Command struct {
	Path    string
	Input   File
	Outputs []File
}

// You can pass in only input (for ffprobe)
// You can pass in one or multiple ouptuts
// Returns a new Command structure
func NewCommand(path string, input File, outputs ...File) (cmd *Command, err error) {
	if path == "" {
		return nil, fmt.Errorf("Cannot create a Command with no path")
	}
	if input == nil {
		return nil, fmt.Errorf("Cannot create a Command with no input")
	}
	cmd = &Command{
		Path:  path,
		Input: input,
	}

	for _, output := range outputs {
		if output != nil {
			cmd.Outputs = append(cmd.Outputs, output)
		}
	}

	return cmd, nil
}

// Returns a []string slice of how the ffmpeg/ffprobe call should be represented
// Does not include the command Path before it
func (c *Command) Slice() (results []string) {
	results = []string{}
	results = append(results, c.Input.Slice()...)
	for _, output := range c.Outputs {
		results = append(results, output.Slice()...)
	}
	return
}
<|endoftext|>


<commit_before>package sctl

import (
	"log"
	"os/exec"
	"runtime/debug"
	"strings"
)

// Command Holds the main command and argument to be executed against os/exec
type Command struct {
	Main string   `json:"main"`
	Args []string `json:"args"`
}

// Execute Executes a given command against os/exec and return the output and potential errors
func (command Command) Execute() (string, error) {
	out, err := exec.Command(command.Main, command.Args...).Output()
	if err != nil {
		log.Println("Failed: " + command.ToString())
		log.Println(string(debug.Stack()))
	}
	return string(out), err
}

// ToString returns the string representation of the command
func (command Command) ToString() string {
	return command.Main + " " + strings.Join(command.Args, " ")
}

// DockerCommand creates a general docker command
func DockerCommand(args []string) Command {
	return Command{
		Main: "docker",
		Args: args,
	}
}

// MinionCommand Holds a command and the minon it should be sent to
type MinionCommand struct {
	Minion  Node    `json:"node"`
	Command Command `json:"command"`
}
<commit_msg>command: Changed execute to older version<commit_after>package sctl

import (
	"bytes"
	"os/exec"
	"strings"
)

// Command Holds the main command and argument to be executed against os/exec
type Command struct {
	Main string   `json:"main"`
	Args []string `json:"args"`
}

// Execute Executes a given command against os/exec and return the output and potential errors
func (command Command) Execute() (string, error) {
	cmd := exec.Command(command.Main, command.Args...)
	var out, errOut bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &errOut
	err := cmd.Run()
	if err != nil {
		return errOut.String(), err
	}
	return out.String(), nil
}

// ToString returns the string representation of the command
func (command Command) ToString() string {
	return command.Main + " " + strings.Join(command.Args, " ")
}

// DockerCommand creates a general docker command
func DockerCommand(args []string) Command {
	return Command{
		Main: "docker",
		Args: args,
	}
}

// MinionCommand Holds a command and the minon it should be sent to
type MinionCommand struct {
	Minion  Node    `json:"node"`
	Command Command `json:"command"`
}
<|endoftext|>


<commit_before>//
// DISCLAIMER
//
// Copyright 2017 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package driver

import (
	"context"
	"time"
)

// Cluster provides access to cluster wide specific operations.
// To use this interface, an ArangoDB cluster is required.
type Cluster interface {
	// Get the cluster configuration & health
	Health(ctx context.Context) (ClusterHealth, error)

	// Get the inventory of the cluster containing all collections (with entire details) of a database.
	DatabaseInventory(ctx context.Context, db Database) (DatabaseInventory, error)

	// MoveShard moves a single shard of the given collection from server `fromServer` to
	// server `toServer`.
	MoveShard(ctx context.Context, col Collection, shard ShardID, fromServer, toServer ServerID) error

	// CleanOutServer triggers activities to clean out a DBServer.
	CleanOutServer(ctx context.Context, serverID string) error

	// IsCleanedOut checks if the dbserver with given ID has been cleaned out.
	IsCleanedOut(ctx context.Context, serverID string) (bool, error)

	// RemoveServer is a low-level option to remove a server from a cluster.
	// This function is suitable for servers of type coordinator or dbserver.
	// The use of `ClientServerAdmin.Shutdown` is highly recommended above this function.
	RemoveServer(ctx context.Context, serverID ServerID) error
}

// ServerID identifies an arangod server in a cluster.
type ServerID string

// ClusterHealth contains health information for all servers in a cluster.
type ClusterHealth struct {
	// Unique identifier of the entire cluster.
	// This ID is created when the cluster was first created.
	ID string `json:"ClusterId"`
	// Health per server
	Health map[ServerID]ServerHealth `json:"Health"`
}

// ServerSyncStatus describes the servers sync status
type ServerSyncStatus string

const (
	ServerSyncStatusUnknown   ServerSyncStatus = "UNKNOWN"
	ServerSyncStatusUndefined ServerSyncStatus = "UNDEFINED"
	ServerSyncStatusStartup   ServerSyncStatus = "STARTUP"
	ServerSyncStatusStopping  ServerSyncStatus = "STOPPING"
	ServerSyncStatusStopped   ServerSyncStatus = "STOPPED"
	ServerSyncStatusServing   ServerSyncStatus = "SERVING"
	ServerSyncStatusShutdown  ServerSyncStatus = "SHUTDOWN"
)

// ServerHealth contains health information of a single server in a cluster.
type ServerHealth struct {
	Endpoint            string           `json:"Endpoint"`
	LastHeartbeatAcked  time.Time        `json:"LastHeartbeatAcked"`
	LastHeartbeatSent   time.Time        `json:"LastHeartbeatSent"`
	LastHeartbeatStatus string           `json:"LastHeartbeatStatus"`
	Role                ServerRole       `json:"Role"`
	ShortName           string           `json:"ShortName"`
	Status              ServerStatus     `json:"Status"`
	CanBeDeleted        bool             `json:"CanBeDeleted"`
	HostID              string           `json:"Host,omitempty"`
	Version             Version          `json:"Version,omitempty"`
	Engine              EngineType       `json:"Engine,omitempty"`
	SyncStatus          ServerSyncStatus `json:"SyncStatus,omitempty"`

	// Only for Coordinators
	AdvertisedEndpoint *string `json:"AdvertisedEndpoint,omitempty"`

	// Only for Agents
	Leader  *string `json:"Leader,omitempty"`
	Leading *bool   `json:"Leading,omitempty"`
}

// ServerStatus describes the health status of a server
type ServerStatus string

const (
	// ServerStatusGood indicates server is in good state
	ServerStatusGood ServerStatus = "GOOD"
	// ServerStatusBad indicates server has missed 1 heartbeat
	ServerStatusBad ServerStatus = "BAD"
	// ServerStatusFailed indicates server has been declared failed by the supervision, this happens after about 15s being bad.
	ServerStatusFailed ServerStatus = "FAILED"
)

// DatabaseInventory describes a detailed state of the collections & shards of a specific database within a cluster.
type DatabaseInventory struct {
	// Details of all collections
	Collections []InventoryCollection `json:"collections,omitempty"`
	// Details of all views
	Views []InventoryView `json:"views,omitempty"`
}

// IsReady returns true if the IsReady flag of all collections is set.
func (i DatabaseInventory) IsReady() bool {
	for _, c := range i.Collections {
		if !c.IsReady {
			return false
		}
	}
	return true
}

// PlanVersion returns the plan version of the first collection in the given inventory.
func (i DatabaseInventory) PlanVersion() int64 {
	if len(i.Collections) == 0 {
		return 0
	}
	return i.Collections[0].PlanVersion
}

// CollectionByName returns the InventoryCollection with given name.
// Return false if not found.
func (i DatabaseInventory) CollectionByName(name string) (InventoryCollection, bool) {
	for _, c := range i.Collections {
		if c.Parameters.Name == name {
			return c, true
		}
	}
	return InventoryCollection{}, false
}

// ViewByName returns the InventoryView with given name.
// Return false if not found.
func (i DatabaseInventory) ViewByName(name string) (InventoryView, bool) {
	for _, v := range i.Views {
		if v.Name == name {
			return v, true
		}
	}
	return InventoryView{}, false
}

// InventoryCollection is a single element of a DatabaseInventory, containing all information
// of a specific collection.
type InventoryCollection struct {
	Parameters  InventoryCollectionParameters `json:"parameters"`
	Indexes     []InventoryIndex              `json:"indexes,omitempty"`
	PlanVersion int64                         `json:"planVersion,omitempty"`
	IsReady     bool                          `json:"isReady,omitempty"`
}

// IndexByFieldsAndType returns the InventoryIndex with given fields & type.
// Return false if not found.
func (i InventoryCollection) IndexByFieldsAndType(fields []string, indexType string) (InventoryIndex, bool) {
	for _, idx := range i.Indexes {
		if idx.Type == indexType && idx.FieldsEqual(fields) {
			return idx, true
		}
	}
	return InventoryIndex{}, false
}

// InventoryCollectionParameters contains all configuration parameters of a collection in a database inventory.
type InventoryCollectionParameters struct {
	Deleted             bool             `json:"deleted,omitempty"`
	DoCompact           bool             `json:"doCompact,omitempty"`
	ID                  string           `json:"id,omitempty"`
	IndexBuckets        int              `json:"indexBuckets,omitempty"`
	Indexes             []InventoryIndex `json:"indexes,omitempty"`
	IsSmart             bool             `json:"isSmart,omitempty"`
	SmartGraphAttribute string           `json:"smartGraphAttribute,omitempty"`
	IsSystem            bool             `json:"isSystem,omitempty"`
	IsVolatile          bool             `json:"isVolatile,omitempty"`
	JournalSize         int64            `json:"journalSize,omitempty"`
	KeyOptions          struct {
		Type          string `json:"type,omitempty"`
		AllowUserKeys bool   `json:"allowUserKeys,omitempty"`
		LastValue     int64  `json:"lastValue,omitempty"`
	} `json:"keyOptions"`
	Name                 string                 `json:"name,omitempty"`
	NumberOfShards       int                    `json:"numberOfShards,omitempty"`
	Path                 string                 `json:"path,omitempty"`
	PlanID               string                 `json:"planId,omitempty"`
	ReplicationFactor    int                    `json:"replicationFactor,omitempty"`
	ShardKeys            []string               `json:"shardKeys,omitempty"`
	Shards               map[ShardID][]ServerID `json:"shards,omitempty"`
	Status               CollectionStatus       `json:"status,omitempty"`
	Type                 CollectionType         `json:"type,omitempty"`
	WaitForSync          bool                   `json:"waitForSync,omitempty"`
	DistributeShardsLike string                 `json:"distributeShardsLike,omitempty"`
}

// IsSatellite returns true if the collection is a satellite collection
func (icp *InventoryCollectionParameters) IsSatellite() bool {
	return icp.ReplicationFactor == ReplicationFactorSatellite
}

// ShardID is an internal identifier of a specific shard
type ShardID string

// InventoryIndex contains all configuration parameters of a single index of a collection in a database inventory.
type InventoryIndex struct {
	ID          string   `json:"id,omitempty"`
	Type        string   `json:"type,omitempty"`
	Fields      []string `json:"fields,omitempty"`
	Unique      bool     `json:"unique"`
	Sparse      bool     `json:"sparse"`
	Deduplicate bool     `json:"deduplicate"`
	MinLength   int      `json:"minLength,omitempty"`
	GeoJSON     bool     `json:"geoJson,omitempty"`
}

// FieldsEqual returns true when the given fields list equals the
// Fields list in the InventoryIndex.
// The order of fields is irrelevant.
func (i InventoryIndex) FieldsEqual(fields []string) bool {
	return stringSliceEqualsIgnoreOrder(i.Fields, fields)
}

// InventoryView is a single element of a DatabaseInventory, containing all information
// of a specific view.
type InventoryView struct {
	Name     string   `json:"name,omitempty"`
	Deleted  bool     `json:"deleted,omitempty"`
	ID       string   `json:"id,omitempty"`
	IsSystem bool     `json:"isSystem,omitempty"`
	PlanID   string   `json:"planId,omitempty"`
	Type     ViewType `json:"type,omitempty"`
	// Include all properties from an arangosearch view.
	ArangoSearchViewProperties
}

// stringSliceEqualsIgnoreOrder returns true when the given lists contain the same elements.
// The order of elements is irrelevant.
func stringSliceEqualsIgnoreOrder(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	bMap := make(map[string]struct{})
	for _, x := range b {
		bMap[x] = struct{}{}
	}
	for _, x := range a {
		if _, found := bMap[x]; !found {
			return false
		}
	}
	return true
}
<commit_msg>Added AllInSync field. (#173)<commit_after>//
// DISCLAIMER
//
// Copyright 2017 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package driver

import (
	"context"
	"time"
)

// Cluster provides access to cluster wide specific operations.
// To use this interface, an ArangoDB cluster is required.
type Cluster interface {
	// Get the cluster configuration & health
	Health(ctx context.Context) (ClusterHealth, error)

	// Get the inventory of the cluster containing all collections (with entire details) of a database.
	DatabaseInventory(ctx context.Context, db Database) (DatabaseInventory, error)

	// MoveShard moves a single shard of the given collection from server `fromServer` to
	// server `toServer`.
	MoveShard(ctx context.Context, col Collection, shard ShardID, fromServer, toServer ServerID) error

	// CleanOutServer triggers activities to clean out a DBServer.
	CleanOutServer(ctx context.Context, serverID string) error

	// IsCleanedOut checks if the dbserver with given ID has been cleaned out.
	IsCleanedOut(ctx context.Context, serverID string) (bool, error)

	// RemoveServer is a low-level option to remove a server from a cluster.
	// This function is suitable for servers of type coordinator or dbserver.
	// The use of `ClientServerAdmin.Shutdown` is highly recommended above this function.
	RemoveServer(ctx context.Context, serverID ServerID) error
}

// ServerID identifies an arangod server in a cluster.
type ServerID string

// ClusterHealth contains health information for all servers in a cluster.
type ClusterHealth struct {
	// Unique identifier of the entire cluster.
	// This ID is created when the cluster was first created.
	ID string `json:"ClusterId"`
	// Health per server
	Health map[ServerID]ServerHealth `json:"Health"`
}

// ServerSyncStatus describes the servers sync status
type ServerSyncStatus string

const (
	ServerSyncStatusUnknown   ServerSyncStatus = "UNKNOWN"
	ServerSyncStatusUndefined ServerSyncStatus = "UNDEFINED"
	ServerSyncStatusStartup   ServerSyncStatus = "STARTUP"
	ServerSyncStatusStopping  ServerSyncStatus = "STOPPING"
	ServerSyncStatusStopped   ServerSyncStatus = "STOPPED"
	ServerSyncStatusServing   ServerSyncStatus = "SERVING"
	ServerSyncStatusShutdown  ServerSyncStatus = "SHUTDOWN"
)

// ServerHealth contains health information of a single server in a cluster.
type ServerHealth struct {
	Endpoint            string           `json:"Endpoint"`
	LastHeartbeatAcked  time.Time        `json:"LastHeartbeatAcked"`
	LastHeartbeatSent   time.Time        `json:"LastHeartbeatSent"`
	LastHeartbeatStatus string           `json:"LastHeartbeatStatus"`
	Role                ServerRole       `json:"Role"`
	ShortName           string           `json:"ShortName"`
	Status              ServerStatus     `json:"Status"`
	CanBeDeleted        bool             `json:"CanBeDeleted"`
	HostID              string           `json:"Host,omitempty"`
	Version             Version          `json:"Version,omitempty"`
	Engine              EngineType       `json:"Engine,omitempty"`
	SyncStatus          ServerSyncStatus `json:"SyncStatus,omitempty"`

	// Only for Coordinators
	AdvertisedEndpoint *string `json:"AdvertisedEndpoint,omitempty"`

	// Only for Agents
	Leader  *string `json:"Leader,omitempty"`
	Leading *bool   `json:"Leading,omitempty"`
}

// ServerStatus describes the health status of a server
type ServerStatus string

const (
	// ServerStatusGood indicates server is in good state
	ServerStatusGood ServerStatus = "GOOD"
	// ServerStatusBad indicates server has missed 1 heartbeat
	ServerStatusBad ServerStatus = "BAD"
	// ServerStatusFailed indicates server has been declared failed by the supervision, this happens after about 15s being bad.
	ServerStatusFailed ServerStatus = "FAILED"
)

// DatabaseInventory describes a detailed state of the collections & shards of a specific database within a cluster.
type DatabaseInventory struct {
	// Details of all collections
	Collections []InventoryCollection `json:"collections,omitempty"`
	// Details of all views
	Views []InventoryView `json:"views,omitempty"`
}

// IsReady returns true if the IsReady flag of all collections is set.
func (i DatabaseInventory) IsReady() bool {
	for _, c := range i.Collections {
		if !c.IsReady {
			return false
		}
	}
	return true
}

// PlanVersion returns the plan version of the first collection in the given inventory.
func (i DatabaseInventory) PlanVersion() int64 {
	if len(i.Collections) == 0 {
		return 0
	}
	return i.Collections[0].PlanVersion
}

// CollectionByName returns the InventoryCollection with given name.
// Return false if not found.
func (i DatabaseInventory) CollectionByName(name string) (InventoryCollection, bool) {
	for _, c := range i.Collections {
		if c.Parameters.Name == name {
			return c, true
		}
	}
	return InventoryCollection{}, false
}

// ViewByName returns the InventoryView with given name.
// Return false if not found.
func (i DatabaseInventory) ViewByName(name string) (InventoryView, bool) {
	for _, v := range i.Views {
		if v.Name == name {
			return v, true
		}
	}
	return InventoryView{}, false
}

// InventoryCollection is a single element of a DatabaseInventory, containing all information
// of a specific collection.
type InventoryCollection struct {
	Parameters  InventoryCollectionParameters `json:"parameters"`
	Indexes     []InventoryIndex              `json:"indexes,omitempty"`
	PlanVersion int64                         `json:"planVersion,omitempty"`
	IsReady     bool                          `json:"isReady,omitempty"`
	AllInSync   bool                          `json:"allInSync,omitempty"`
}

// IndexByFieldsAndType returns the InventoryIndex with given fields & type.
// Return false if not found.
func (i InventoryCollection) IndexByFieldsAndType(fields []string, indexType string) (InventoryIndex, bool) {
	for _, idx := range i.Indexes {
		if idx.Type == indexType && idx.FieldsEqual(fields) {
			return idx, true
		}
	}
	return InventoryIndex{}, false
}

// InventoryCollectionParameters contains all configuration parameters of a collection in a database inventory.
type InventoryCollectionParameters struct {
	Deleted             bool             `json:"deleted,omitempty"`
	DoCompact           bool             `json:"doCompact,omitempty"`
	ID                  string           `json:"id,omitempty"`
	IndexBuckets        int              `json:"indexBuckets,omitempty"`
	Indexes             []InventoryIndex `json:"indexes,omitempty"`
	IsSmart             bool             `json:"isSmart,omitempty"`
	SmartGraphAttribute string           `json:"smartGraphAttribute,omitempty"`
	IsSystem            bool             `json:"isSystem,omitempty"`
	IsVolatile          bool             `json:"isVolatile,omitempty"`
	JournalSize         int64            `json:"journalSize,omitempty"`
	KeyOptions          struct {
		Type          string `json:"type,omitempty"`
		AllowUserKeys bool   `json:"allowUserKeys,omitempty"`
		LastValue     int64  `json:"lastValue,omitempty"`
	} `json:"keyOptions"`
	Name                 string                 `json:"name,omitempty"`
	NumberOfShards       int                    `json:"numberOfShards,omitempty"`
	Path                 string                 `json:"path,omitempty"`
	PlanID               string                 `json:"planId,omitempty"`
	ReplicationFactor    int                    `json:"replicationFactor,omitempty"`
	ShardKeys            []string               `json:"shardKeys,omitempty"`
	Shards               map[ShardID][]ServerID `json:"shards,omitempty"`
	Status               CollectionStatus       `json:"status,omitempty"`
	Type                 CollectionType         `json:"type,omitempty"`
	WaitForSync          bool                   `json:"waitForSync,omitempty"`
	DistributeShardsLike string                 `json:"distributeShardsLike,omitempty"`
}

// IsSatellite returns true if the collection is a satellite collection
func (icp *InventoryCollectionParameters) IsSatellite() bool {
	return icp.ReplicationFactor == ReplicationFactorSatellite
}

// ShardID is an internal identifier of a specific shard
type ShardID string

// InventoryIndex contains all configuration parameters of a single index of a collection in a database inventory.
type InventoryIndex struct {
	ID          string   `json:"id,omitempty"`
	Type        string   `json:"type,omitempty"`
	Fields      []string `json:"fields,omitempty"`
	Unique      bool     `json:"unique"`
	Sparse      bool     `json:"sparse"`
	Deduplicate bool     `json:"deduplicate"`
	MinLength   int      `json:"minLength,omitempty"`
	GeoJSON     bool     `json:"geoJson,omitempty"`
}

// FieldsEqual returns true when the given fields list equals the
// Fields list in the InventoryIndex.
// The order of fields is irrelevant.
func (i InventoryIndex) FieldsEqual(fields []string) bool {
	return stringSliceEqualsIgnoreOrder(i.Fields, fields)
}

// InventoryView is a single element of a DatabaseInventory, containing all information
// of a specific view.
type InventoryView struct {
	Name     string   `json:"name,omitempty"`
	Deleted  bool     `json:"deleted,omitempty"`
	ID       string   `json:"id,omitempty"`
	IsSystem bool     `json:"isSystem,omitempty"`
	PlanID   string   `json:"planId,omitempty"`
	Type     ViewType `json:"type,omitempty"`
	// Include all properties from an arangosearch view.
	ArangoSearchViewProperties
}

// stringSliceEqualsIgnoreOrder returns true when the given lists contain the same elements.
// The order of elements is irrelevant.
func stringSliceEqualsIgnoreOrder(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	bMap := make(map[string]struct{})
	for _, x := range b {
		bMap[x] = struct{}{}
	}
	for _, x := range a {
		if _, found := bMap[x]; !found {
			return false
		}
	}
	return true
}
<|endoftext|>