<commit_before>package k8s

import (
	"fmt"
	"time"

	"math/rand"

	"github.com/jtblin/kube2iam"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	v1 "k8s.io/client-go/pkg/api/v1"
	selector "k8s.io/client-go/pkg/fields"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
)

const (
	podIPIndexName     = "byPodIP"
	namespaceIndexName = "byName"
)

// Client represents a kubernetes client.
type Client struct {
	*kubernetes.Clientset
	namespaceController *cache.Controller
	namespaceIndexer    cache.Indexer
	podController       *cache.Controller
	podIndexer          cache.Indexer
	nodeName            string
}

// Returns a cache.ListWatch that gets all changes to pods.
func (k8s *Client) createPodLW() *cache.ListWatch {
	fieldSelector := selector.Everything()
	if k8s.nodeName != "" {
		fieldSelector = selector.OneTermEqualSelector("spec.nodeName", k8s.nodeName)
	}
	return cache.NewListWatchFromClient(k8s.CoreV1().RESTClient(), "pods", v1.NamespaceAll, fieldSelector)
}

// WatchForPods watches for pod changes.
func (k8s *Client) WatchForPods(podEventLogger cache.ResourceEventHandler, resyncPeriod time.Duration) cache.InformerSynced {
	randomDelay := time.Duration(rand.Intn(60)) * time.Second
	resyncPeriod = resyncPeriod + randomDelay
	k8s.podIndexer, k8s.podController = cache.NewIndexerInformer(
		k8s.createPodLW(),
		&v1.Pod{},
		resyncPeriod,
		podEventLogger,
		cache.Indexers{podIPIndexName: kube2iam.PodIPIndexFunc},
	)
	go k8s.podController.Run(wait.NeverStop)
	return k8s.podController.HasSynced
}

// returns a cache.ListWatch of namespaces.
func (k8s *Client) createNamespaceLW() *cache.ListWatch {
	return cache.NewListWatchFromClient(k8s.CoreV1().RESTClient(), "namespaces", v1.NamespaceAll, selector.Everything())
}

// WatchForNamespaces watches for namespaces changes.
func (k8s *Client) WatchForNamespaces(nsEventLogger cache.ResourceEventHandler, resyncPeriod time.Duration) cache.InformerSynced {
	randomDelay := time.Duration(rand.Intn(60)) * time.Second
	resyncPeriod = resyncPeriod + randomDelay
	k8s.namespaceIndexer, k8s.namespaceController = cache.NewIndexerInformer(
		k8s.createNamespaceLW(),
		&v1.Namespace{},
		resyncPeriod,
		nsEventLogger,
		cache.Indexers{namespaceIndexName: kube2iam.NamespaceIndexFunc},
	)
	go k8s.namespaceController.Run(wait.NeverStop)
	return k8s.namespaceController.HasSynced
}

// ListPodIPs returns the underlying set of pods being managed/indexed
func (k8s *Client) ListPodIPs() []string {
	// Decided to simply dump this and leave it up to consumer
	// as k8s package currently doesn't need to be concerned about what's
	// a signficant annotation to process, that is left up to store/server
	return k8s.podIndexer.ListIndexFuncValues(podIPIndexName)
}

// ListNamespaces returns the underlying set of namespaces being managed/indexed
func (k8s *Client) ListNamespaces() []string {
	return k8s.namespaceIndexer.ListIndexFuncValues(namespaceIndexName)
}

// PodByIP provides the representation of the pod itself being cached keyed off of it's IP
// Returns an error if there are multiple pods attempting to be keyed off of the same IP
// (Which happens when they of type `hostNetwork: true`)
func (k8s *Client) PodByIP(IP string) (*v1.Pod, error) {
	pods, err := k8s.podIndexer.ByIndex(podIPIndexName, IP)
	if err != nil {
		return nil, err
	}

	if len(pods) == 0 {
		return nil, fmt.Errorf("pod with specificed IP not found")
	}

	if len(pods) == 1 {
		return pods[0].(*v1.Pod), nil
	}

	//This happens with `hostNetwork: true` pods
	podNames := make([]string, len(pods))
	for i, pod := range pods {
		podNames[i] = pod.(*v1.Pod).ObjectMeta.Name
	}
	return nil, fmt.Errorf("%d pods (%v) with the ip %s indexed", len(pods), podNames, IP)
}

// NamespaceByName retrieves a namespace by it's given name.
// Returns an error if there are no namespaces available
func (k8s *Client) NamespaceByName(namespaceName string) (*v1.Namespace, error) {
	namespace, err := k8s.namespaceIndexer.ByIndex(namespaceIndexName, namespaceName)
	if err != nil {
		return nil, err
	}

	if len(namespace) == 0 {
		return nil, fmt.Errorf("namespace was not found")
	}

	return namespace[0].(*v1.Namespace), nil
}

// NewClient returns a new kubernetes client.
func NewClient(host, token, nodeName string, insecure bool) (*Client, error) {
	var config *rest.Config
	var err error
	if host != "" && token != "" {
		config = &rest.Config{
			Host:        host,
			BearerToken: token,
			Insecure:    insecure,
		}
	} else {
		config, err = rest.InClusterConfig()
		if err != nil {
			return nil, err
		}
	}
	client, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, err
	}
	return &Client{Clientset: client, nodeName: nodeName}, nil
}
<commit_msg>Revert "add random delay"<commit_after>package k8s

import (
	"fmt"
	"time"

	"github.com/jtblin/kube2iam"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	v1 "k8s.io/client-go/pkg/api/v1"
	selector "k8s.io/client-go/pkg/fields"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
)

const (
	podIPIndexName     = "byPodIP"
	namespaceIndexName = "byName"
)

// Client represents a kubernetes client.
type Client struct {
	*kubernetes.Clientset
	namespaceController *cache.Controller
	namespaceIndexer    cache.Indexer
	podController       *cache.Controller
	podIndexer          cache.Indexer
	nodeName            string
}

// Returns a cache.ListWatch that gets all changes to pods.
func (k8s *Client) createPodLW() *cache.ListWatch {
	fieldSelector := selector.Everything()
	if k8s.nodeName != "" {
		fieldSelector = selector.OneTermEqualSelector("spec.nodeName", k8s.nodeName)
	}
	return cache.NewListWatchFromClient(k8s.CoreV1().RESTClient(), "pods", v1.NamespaceAll, fieldSelector)
}

// WatchForPods watches for pod changes.
func (k8s *Client) WatchForPods(podEventLogger cache.ResourceEventHandler, resyncPeriod time.Duration) cache.InformerSynced {
	k8s.podIndexer, k8s.podController = cache.NewIndexerInformer(
		k8s.createPodLW(),
		&v1.Pod{},
		resyncPeriod,
		podEventLogger,
		cache.Indexers{podIPIndexName: kube2iam.PodIPIndexFunc},
	)
	go k8s.podController.Run(wait.NeverStop)
	return k8s.podController.HasSynced
}

// returns a cache.ListWatch of namespaces.
func (k8s *Client) createNamespaceLW() *cache.ListWatch {
	return cache.NewListWatchFromClient(k8s.CoreV1().RESTClient(), "namespaces", v1.NamespaceAll, selector.Everything())
}

// WatchForNamespaces watches for namespaces changes.
func (k8s *Client) WatchForNamespaces(nsEventLogger cache.ResourceEventHandler, resyncPeriod time.Duration) cache.InformerSynced {
	k8s.namespaceIndexer, k8s.namespaceController = cache.NewIndexerInformer(
		k8s.createNamespaceLW(),
		&v1.Namespace{},
		resyncPeriod,
		nsEventLogger,
		cache.Indexers{namespaceIndexName: kube2iam.NamespaceIndexFunc},
	)
	go k8s.namespaceController.Run(wait.NeverStop)
	return k8s.namespaceController.HasSynced
}

// ListPodIPs returns the underlying set of pods being managed/indexed
func (k8s *Client) ListPodIPs() []string {
	// Decided to simply dump this and leave it up to consumer
	// as k8s package currently doesn't need to be concerned about what's
	// a signficant annotation to process, that is left up to store/server
	return k8s.podIndexer.ListIndexFuncValues(podIPIndexName)
}

// ListNamespaces returns the underlying set of namespaces being managed/indexed
func (k8s *Client) ListNamespaces() []string {
	return k8s.namespaceIndexer.ListIndexFuncValues(namespaceIndexName)
}

// PodByIP provides the representation of the pod itself being cached keyed off of it's IP
// Returns an error if there are multiple pods attempting to be keyed off of the same IP
// (Which happens when they of type `hostNetwork: true`)
func (k8s *Client) PodByIP(IP string) (*v1.Pod, error) {
	pods, err := k8s.podIndexer.ByIndex(podIPIndexName, IP)
	if err != nil {
		return nil, err
	}

	if len(pods) == 0 {
		return nil, fmt.Errorf("pod with specificed IP not found")
	}

	if len(pods) == 1 {
		return pods[0].(*v1.Pod), nil
	}

	//This happens with `hostNetwork: true` pods
	podNames := make([]string, len(pods))
	for i, pod := range pods {
		podNames[i] = pod.(*v1.Pod).ObjectMeta.Name
	}
	return nil, fmt.Errorf("%d pods (%v) with the ip %s indexed", len(pods), podNames, IP)
}

// NamespaceByName retrieves a namespace by it's given name.
// Returns an error if there are no namespaces available
func (k8s *Client) NamespaceByName(namespaceName string) (*v1.Namespace, error) {
	namespace, err := k8s.namespaceIndexer.ByIndex(namespaceIndexName, namespaceName)
	if err != nil {
		return nil, err
	}

	if len(namespace) == 0 {
		return nil, fmt.Errorf("namespace was not found")
	}

	return namespace[0].(*v1.Namespace), nil
}

// NewClient returns a new kubernetes client.
func NewClient(host, token, nodeName string, insecure bool) (*Client, error) {
	var config *rest.Config
	var err error
	if host != "" && token != "" {
		config = &rest.Config{
			Host:        host,
			BearerToken: token,
			Insecure:    insecure,
		}
	} else {
		config, err = rest.InClusterConfig()
		if err != nil {
			return nil, err
		}
	}
	client, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, err
	}
	return &Client{Clientset: client, nodeName: nodeName}, nil
}
<|endoftext|>


<commit_before>package expr

import (
	"encoding/json"
	"fmt"
	"reflect"
	"regexp"
	"runtime"

	"github.com/MiniProfiler/go/miniprofiler"
	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/tsaf/expr/parse"
)

type state struct {
	*Expr
	context opentsdb.Context
	queries []opentsdb.Request
}

func (e *state) addRequest(r opentsdb.Request) {
	e.queries = append(e.queries, r)
}

var ErrUnknownOp = fmt.Errorf("expr: unknown op type")

type Expr struct {
	*parse.Tree
}

func (e *Expr) MarshalJSON() ([]byte, error) {
	return json.Marshal(e.String())
}

func New(expr string) (*Expr, error) {
	t, err := parse.Parse(expr, Builtins)
	if err != nil {
		return nil, err
	}
	e := &Expr{
		Tree: t,
	}
	return e, nil
}

// Execute applies a parse expression to the specified OpenTSDB context,
// and returns one result per group. T may be nil to ignore timings.
func (e *Expr) Execute(c opentsdb.Context, T miniprofiler.Timer) (r []*Result, queries []opentsdb.Request, err error) {
	defer errRecover(&err)
	s := &state{
		Expr:    e,
		context: c,
	}
	if T == nil {
		T = new(miniprofiler.Profile)
	}
	T.Step("expr execute", func(T miniprofiler.Timer) {
		r = s.walk(e.Tree.Root, T)
	})
	queries = s.queries
	return
}

// errRecover is the handler that turns panics into returns from the top
// level of Parse.
func errRecover(errp *error) {
	e := recover()
	if e != nil {
		switch err := e.(type) {
		case runtime.Error:
			panic(e)
		case error:
			*errp = err
		default:
			panic(e)
		}
	}
}

type Value interface {
	Type() parse.FuncType
	Value() interface{}
}

type Number float64

func (n Number) Type() parse.FuncType { return parse.TYPE_NUMBER }
func (n Number) Value() interface{}   { return n }

type Scalar float64

func (s Scalar) Type() parse.FuncType { return parse.TYPE_SCALAR }
func (s Scalar) Value() interface{}   { return s }

type Series map[string]opentsdb.Point

func (s Series) Type() parse.FuncType { return parse.TYPE_SERIES }
func (s Series) Value() interface{}   { return s }

type Result struct {
	Computations
	Value
	Group opentsdb.TagSet
}

type Computations []Computation

type Computation struct {
	Text  string
	Value Number
}

var RGroup_Re = regexp.MustCompile("{[^}]+}")

func (r *Result) AddComputation(text string, result Number) {
	text = RGroup_Re.ReplaceAllString(text, r.Group.String())
	r.Computations = append(r.Computations, Computation{text, result})
}

type Union struct {
	Computations
	A, B  Value
	Group opentsdb.TagSet
}

// wrap creates a new Result with a nil group and given value.
func wrap(v float64) []*Result {
	return []*Result{
		{
			Value: Scalar(v),
			Group: nil,
		},
	}
}

func (u *Union) ExtendComputations(o *Result) {
	u.Computations = append(u.Computations, o.Computations...)
}

// union returns the combination of a and b where one is a strict subset of the
// other.
func union(a, b []*Result) []*Union {
	var us []*Union
	for _, ra := range a {
		for _, rb := range b {
			u := &Union{
				A: ra.Value,
				B: rb.Value,
			}
			setGroup := false
			if ra.Group.Equal(rb.Group) || len(ra.Group) == 0 || len(rb.Group) == 0 {
				g := ra.Group
				if len(ra.Group) == 0 {
					g = rb.Group
				}
				u.Group = g
				setGroup = true
			} else if ra.Group.Subset(rb.Group) {
				u.Group = rb.Group
				setGroup = true
			} else if rb.Group.Subset(ra.Group) {
				u.Group = ra.Group
				setGroup = true
			}
			if setGroup {
				u.ExtendComputations(ra)
				u.ExtendComputations(rb)
				us = append(us, u)
			}
		}
	}
	return us
}

func (e *state) walk(node parse.Node, T miniprofiler.Timer) []*Result {
	switch node := node.(type) {
	case *parse.NumberNode:
		return wrap(node.Float64)
	case *parse.BinaryNode:
		return e.walkBinary(node, T)
	case *parse.UnaryNode:
		return e.walkUnary(node, T)
	case *parse.FuncNode:
		return e.walkFunc(node, T)
	default:
		panic(fmt.Errorf("expr: unknown node type"))
	}
}

func (e *state) walkBinary(node *parse.BinaryNode, T miniprofiler.Timer) []*Result {
	ar := e.walk(node.Args[0], T)
	br := e.walk(node.Args[1], T)
	var res []*Result
	u := union(ar, br)
	for _, v := range u {
		var value Value
		r := Result{
			Group:        v.Group,
			Computations: v.Computations,
		}
		switch at := v.A.(type) {
		case Scalar:
			switch bt := v.B.(type) {
			case Scalar:
				n := Scalar(operate(node.OpStr, float64(at), float64(bt)))
				r.AddComputation(node.String(), Number(n))
				value = n
			case Number:
				n := Number(operate(node.OpStr, float64(at), float64(bt)))
				r.AddComputation(node.String(), n)
				value = n
			case Series:
				s := make(Series)
				for k, v := range bt {
					s[k] = opentsdb.Point(operate(node.OpStr, float64(at), float64(v)))
				}
				value = s
			default:
				panic(ErrUnknownOp)
			}
		case Number:
			switch bt := v.B.(type) {
			case Scalar:
				n := Number(operate(node.OpStr, float64(at), float64(bt)))
				r.AddComputation(node.String(), Number(n))
				value = n
			case Number:
				n := Number(operate(node.OpStr, float64(at), float64(bt)))
				r.AddComputation(node.String(), n)
				value = n
			case Series:
				s := make(Series)
				for k, v := range bt {
					s[k] = opentsdb.Point(operate(node.OpStr, float64(at), float64(v)))
				}
				value = s
			default:
				panic(ErrUnknownOp)
			}
		case Series:
			switch bt := v.B.(type) {
			case Number, Scalar:
				bv := reflect.ValueOf(bt).Float()
				s := make(Series)
				for k, v := range at {
					s[k] = opentsdb.Point(operate(node.OpStr, float64(v), bv))
				}
				value = s
			default:
				panic(ErrUnknownOp)
			}
		default:
			panic(ErrUnknownOp)
		}
		r.Value = value
		res = append(res, &r)
	}
	return res
}

func operate(op string, a, b float64) (r float64) {
	switch op {
	case "+":
		r = a + b
	case "*":
		r = a * b
	case "-":
		r = a - b
	case "/":
		r = a / b
	case "==":
		if a == b {
			r = 1
		} else {
			r = 0
		}
	case ">":
		if a > b {
			r = 1
		} else {
			r = 0
		}
	case "!=":
		if a != b {
			r = 1
		} else {
			r = 0
		}
	case "<":
		if a < b {
			r = 1
		} else {
			r = 0
		}
	case ">=":
		if a >= b {
			r = 1
		} else {
			r = 0
		}
	case "<=":
		if a <= b {
			r = 1
		} else {
			r = 0
		}
	case "||":
		if a != 0 || b != 0 {
			r = 1
		} else {
			r = 0
		}
	case "&&":
		if a != 0 && b != 0 {
			r = 1
		} else {
			r = 0
		}
	default:
		panic(fmt.Errorf("expr: unknown operator %s", op))
	}
	return
}

func (e *state) walkUnary(node *parse.UnaryNode, T miniprofiler.Timer) []*Result {
	a := e.walk(node.Arg, T)
	for _, r := range a {
		switch rt := r.Value.(type) {
		case Scalar:
			r.Value = Scalar(uoperate(node.OpStr, float64(rt)))
		case Number:
			r.Value = Number(uoperate(node.OpStr, float64(rt)))
		case Series:
			s := make(Series)
			for k, v := range rt {
				s[k] = opentsdb.Point(uoperate(node.OpStr, float64(v)))
			}
			r.Value = s
		default:
			panic(ErrUnknownOp)
		}
	}
	return a
}

func uoperate(op string, a float64) (r float64) {
	switch op {
	case "!":
		if a == 0 {
			r = 1
		} else {
			r = 0
		}
	case "-":
		r = -a
	default:
		panic(fmt.Errorf("expr: unknown operator %s", op))
	}
	return
}

func (e *state) walkFunc(node *parse.FuncNode, T miniprofiler.Timer) []*Result {
	f := reflect.ValueOf(node.F.F)
	var in []reflect.Value
	for _, a := range node.Args {
		var v interface{}
		switch t := a.(type) {
		case *parse.StringNode:
			v = t.Text
		case *parse.NumberNode:
			v = t.Float64
		case *parse.FuncNode:
			v = extractScalar(e.walkFunc(t, T))
		case *parse.UnaryNode:
			v = extractScalar(e.walkUnary(t, T))
		case *parse.BinaryNode:
			v = extractScalar(e.walkBinary(t, T))
		default:
			panic(fmt.Errorf("expr: unknown func arg type"))
		}
		in = append(in, reflect.ValueOf(v))
	}
	fr := f.Call(append([]reflect.Value{reflect.ValueOf(e), reflect.ValueOf(T)}, in...))
	res := fr[0].Interface().([]*Result)
	if len(fr) > 1 && !fr[1].IsNil() {
		err := fr[1].Interface().(error)
		if err != nil {
			panic(err)
		}
	}
	if node.Return() == parse.TYPE_NUMBER {
		for _, r := range res {
			r.AddComputation(node.String(), r.Value.(Number))
		}
	}
	return res
}

// extractScalar will return a float64 if res contains exactly one scalar.
func extractScalar(res []*Result) interface{} {
	if len(res) == 1 && res[0].Type() == parse.TYPE_SCALAR {
		return float64(res[0].Value.Value().(Scalar))
	}
	return res
}
<commit_msg>Minor refactor<commit_after>package expr

import (
	"encoding/json"
	"fmt"
	"reflect"
	"regexp"
	"runtime"

	"github.com/MiniProfiler/go/miniprofiler"
	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/tsaf/expr/parse"
)

type state struct {
	*Expr
	context opentsdb.Context
	queries []opentsdb.Request
}

func (e *state) addRequest(r opentsdb.Request) {
	e.queries = append(e.queries, r)
}

var ErrUnknownOp = fmt.Errorf("expr: unknown op type")

type Expr struct {
	*parse.Tree
}

func (e *Expr) MarshalJSON() ([]byte, error) {
	return json.Marshal(e.String())
}

func New(expr string) (*Expr, error) {
	t, err := parse.Parse(expr, Builtins)
	if err != nil {
		return nil, err
	}
	e := &Expr{
		Tree: t,
	}
	return e, nil
}

// Execute applies a parse expression to the specified OpenTSDB context,
// and returns one result per group. T may be nil to ignore timings.
func (e *Expr) Execute(c opentsdb.Context, T miniprofiler.Timer) (r []*Result, queries []opentsdb.Request, err error) {
	defer errRecover(&err)
	s := &state{
		Expr:    e,
		context: c,
	}
	if T == nil {
		T = new(miniprofiler.Profile)
	}
	T.Step("expr execute", func(T miniprofiler.Timer) {
		r = s.walk(e.Tree.Root, T)
	})
	queries = s.queries
	return
}

// errRecover is the handler that turns panics into returns from the top
// level of Parse.
func errRecover(errp *error) {
	e := recover()
	if e != nil {
		switch err := e.(type) {
		case runtime.Error:
			panic(e)
		case error:
			*errp = err
		default:
			panic(e)
		}
	}
}

type Value interface {
	Type() parse.FuncType
	Value() interface{}
}

type Number float64

func (n Number) Type() parse.FuncType { return parse.TYPE_NUMBER }
func (n Number) Value() interface{}   { return n }

type Scalar float64

func (s Scalar) Type() parse.FuncType { return parse.TYPE_SCALAR }
func (s Scalar) Value() interface{}   { return s }

type Series map[string]opentsdb.Point

func (s Series) Type() parse.FuncType { return parse.TYPE_SERIES }
func (s Series) Value() interface{}   { return s }

type Result struct {
	Computations
	Value
	Group opentsdb.TagSet
}

type Computations []Computation

type Computation struct {
	Text  string
	Value Number
}

var RGroup_Re = regexp.MustCompile("{[^}]+}")

func (r *Result) AddComputation(text string, result Number) {
	text = RGroup_Re.ReplaceAllString(text, r.Group.String())
	r.Computations = append(r.Computations, Computation{text, result})
}

type Union struct {
	Computations
	A, B  Value
	Group opentsdb.TagSet
}

// wrap creates a new Result with a nil group and given value.
func wrap(v float64) []*Result {
	return []*Result{
		{
			Value: Scalar(v),
			Group: nil,
		},
	}
}

func (u *Union) ExtendComputations(o *Result) {
	u.Computations = append(u.Computations, o.Computations...)
}

// union returns the combination of a and b where one is a strict subset of the
// other.
func union(a, b []*Result) []*Union {
	var us []*Union
	for _, ra := range a {
		for _, rb := range b {
			u := &Union{
				A: ra.Value,
				B: rb.Value,
			}
			if ra.Group.Equal(rb.Group) || len(ra.Group) == 0 || len(rb.Group) == 0 {
				g := ra.Group
				if len(ra.Group) == 0 {
					g = rb.Group
				}
				u.Group = g
			} else if ra.Group.Subset(rb.Group) {
				u.Group = rb.Group
			} else if rb.Group.Subset(ra.Group) {
				u.Group = ra.Group
			} else {
				continue
			}
			u.ExtendComputations(ra)
			u.ExtendComputations(rb)
			us = append(us, u)
		}
	}
	return us
}

func (e *state) walk(node parse.Node, T miniprofiler.Timer) []*Result {
	switch node := node.(type) {
	case *parse.NumberNode:
		return wrap(node.Float64)
	case *parse.BinaryNode:
		return e.walkBinary(node, T)
	case *parse.UnaryNode:
		return e.walkUnary(node, T)
	case *parse.FuncNode:
		return e.walkFunc(node, T)
	default:
		panic(fmt.Errorf("expr: unknown node type"))
	}
}

func (e *state) walkBinary(node *parse.BinaryNode, T miniprofiler.Timer) []*Result {
	ar := e.walk(node.Args[0], T)
	br := e.walk(node.Args[1], T)
	var res []*Result
	u := union(ar, br)
	for _, v := range u {
		var value Value
		r := Result{
			Group:        v.Group,
			Computations: v.Computations,
		}
		switch at := v.A.(type) {
		case Scalar:
			switch bt := v.B.(type) {
			case Scalar:
				n := Scalar(operate(node.OpStr, float64(at), float64(bt)))
				r.AddComputation(node.String(), Number(n))
				value = n
			case Number:
				n := Number(operate(node.OpStr, float64(at), float64(bt)))
				r.AddComputation(node.String(), n)
				value = n
			case Series:
				s := make(Series)
				for k, v := range bt {
					s[k] = opentsdb.Point(operate(node.OpStr, float64(at), float64(v)))
				}
				value = s
			default:
				panic(ErrUnknownOp)
			}
		case Number:
			switch bt := v.B.(type) {
			case Scalar:
				n := Number(operate(node.OpStr, float64(at), float64(bt)))
				r.AddComputation(node.String(), Number(n))
				value = n
			case Number:
				n := Number(operate(node.OpStr, float64(at), float64(bt)))
				r.AddComputation(node.String(), n)
				value = n
			case Series:
				s := make(Series)
				for k, v := range bt {
					s[k] = opentsdb.Point(operate(node.OpStr, float64(at), float64(v)))
				}
				value = s
			default:
				panic(ErrUnknownOp)
			}
		case Series:
			switch bt := v.B.(type) {
			case Number, Scalar:
				bv := reflect.ValueOf(bt).Float()
				s := make(Series)
				for k, v := range at {
					s[k] = opentsdb.Point(operate(node.OpStr, float64(v), bv))
				}
				value = s
			default:
				panic(ErrUnknownOp)
			}
		default:
			panic(ErrUnknownOp)
		}
		r.Value = value
		res = append(res, &r)
	}
	return res
}

func operate(op string, a, b float64) (r float64) {
	switch op {
	case "+":
		r = a + b
	case "*":
		r = a * b
	case "-":
		r = a - b
	case "/":
		r = a / b
	case "==":
		if a == b {
			r = 1
		} else {
			r = 0
		}
	case ">":
		if a > b {
			r = 1
		} else {
			r = 0
		}
	case "!=":
		if a != b {
			r = 1
		} else {
			r = 0
		}
	case "<":
		if a < b {
			r = 1
		} else {
			r = 0
		}
	case ">=":
		if a >= b {
			r = 1
		} else {
			r = 0
		}
	case "<=":
		if a <= b {
			r = 1
		} else {
			r = 0
		}
	case "||":
		if a != 0 || b != 0 {
			r = 1
		} else {
			r = 0
		}
	case "&&":
		if a != 0 && b != 0 {
			r = 1
		} else {
			r = 0
		}
	default:
		panic(fmt.Errorf("expr: unknown operator %s", op))
	}
	return
}

func (e *state) walkUnary(node *parse.UnaryNode, T miniprofiler.Timer) []*Result {
	a := e.walk(node.Arg, T)
	for _, r := range a {
		switch rt := r.Value.(type) {
		case Scalar:
			r.Value = Scalar(uoperate(node.OpStr, float64(rt)))
		case Number:
			r.Value = Number(uoperate(node.OpStr, float64(rt)))
		case Series:
			s := make(Series)
			for k, v := range rt {
				s[k] = opentsdb.Point(uoperate(node.OpStr, float64(v)))
			}
			r.Value = s
		default:
			panic(ErrUnknownOp)
		}
	}
	return a
}

func uoperate(op string, a float64) (r float64) {
	switch op {
	case "!":
		if a == 0 {
			r = 1
		} else {
			r = 0
		}
	case "-":
		r = -a
	default:
		panic(fmt.Errorf("expr: unknown operator %s", op))
	}
	return
}

func (e *state) walkFunc(node *parse.FuncNode, T miniprofiler.Timer) []*Result {
	f := reflect.ValueOf(node.F.F)
	var in []reflect.Value
	for _, a := range node.Args {
		var v interface{}
		switch t := a.(type) {
		case *parse.StringNode:
			v = t.Text
		case *parse.NumberNode:
			v = t.Float64
		case *parse.FuncNode:
			v = extractScalar(e.walkFunc(t, T))
		case *parse.UnaryNode:
			v = extractScalar(e.walkUnary(t, T))
		case *parse.BinaryNode:
			v = extractScalar(e.walkBinary(t, T))
		default:
			panic(fmt.Errorf("expr: unknown func arg type"))
		}
		in = append(in, reflect.ValueOf(v))
	}
	fr := f.Call(append([]reflect.Value{reflect.ValueOf(e), reflect.ValueOf(T)}, in...))
	res := fr[0].Interface().([]*Result)
	if len(fr) > 1 && !fr[1].IsNil() {
		err := fr[1].Interface().(error)
		if err != nil {
			panic(err)
		}
	}
	if node.Return() == parse.TYPE_NUMBER {
		for _, r := range res {
			r.AddComputation(node.String(), r.Value.(Number))
		}
	}
	return res
}

// extractScalar will return a float64 if res contains exactly one scalar.
func extractScalar(res []*Result) interface{} {
	if len(res) == 1 && res[0].Type() == parse.TYPE_SCALAR {
		return float64(res[0].Value.Value().(Scalar))
	}
	return res
}
<|endoftext|>


<commit_before>package services

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
)

var mu sync.Mutex

//Image Image
type Image struct {
	Name     string
	ImageURL string
}

//AddImage AddImage
func (c *CmsService) AddImage(name string, fileData []byte) bool {
	mu.Lock()
	defer mu.Unlock()
	var rtn bool
	c.Log.Debug("image file name in add: ", name)
	var imageName = c.ImagePath + string(filepath.Separator) + name
	c.Log.Debug("image complete file name in add: ", imageName)
	err := ioutil.WriteFile(imageName, fileData, 0644)
	if err == nil {
		rtn = true
	}
	return rtn
}

//GetImageList GetImageList
func (c *CmsService) GetImageList() *[]Image {
	var rtn []Image
	ifiles, err := ioutil.ReadDir(c.ImagePath)
	if err == nil {
		for _, ifile := range ifiles {
			if !ifile.IsDir() {
				//fmt.Println("sfile: ", sfile)
				var imgfile Image
				imgfile.Name = ifile.Name()
				imgfile.ImageURL = ".." + string(filepath.Separator) + ".." + string(filepath.Separator) + "images" + string(filepath.Separator) + ifile.Name()
				c.Log.Debug("image ImageURL in list: ", imgfile.ImageURL)
				rtn = append(rtn, imgfile)
			}
		}
	}
	return &rtn
}

//GetImagePath GetImagePath
func (c *CmsService) GetImagePath(imageName string) string {
	return c.ImageFullPath + string(filepath.Separator) + imageName
}

//DeleteImage DeleteImage
func (c *CmsService) DeleteImage(name string) bool {
	mu.Lock()
	defer mu.Unlock()
	var rtn bool
	var imageName = c.ImagePath + string(filepath.Separator) + name
	c.Log.Debug("image complete file name in delete: ", imageName)
	jerr := os.Remove(imageName)
	if jerr == nil {
		rtn = true
	}
	return rtn
}
<commit_msg>added files<commit_after>package services

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
)

var mu sync.Mutex

//Image Image
type Image struct {
	Name     string
	ImageURL string
}

//AddImage AddImage
func (c *CmsService) AddImage(name string, fileData []byte) bool {
	mu.Lock()
	defer mu.Unlock()
	var rtn bool
	c.Log.Debug("image file name in add: ", name)
	var imageName = c.ImagePath + string(filepath.Separator) + name
	c.Log.Debug("image complete file name in add: ", imageName)
	err := ioutil.WriteFile(imageName, fileData, 0644)
	if err == nil {
		rtn = true
	}
	return rtn
}

//GetImageList GetImageList
func (c *CmsService) GetImageList() *[]Image {
	var rtn []Image
	ifiles, err := ioutil.ReadDir(c.ImagePath)
	if err == nil {
		for _, ifile := range ifiles {
			if !ifile.IsDir() {
				//fmt.Println("sfile: ", sfile)
				var imgfile Image
				imgfile.Name = ifile.Name()
				imgfile.ImageURL = ".." + string(filepath.Separator) + ".." + string(filepath.Separator) + "images" + string(filepath.Separator) + ifile.Name()
				c.Log.Debug("image ImageURL in list: ", imgfile.ImageURL)
				rtn = append(rtn, imgfile)
			}
		}
	}
	return &rtn
}

//GetImagePath GetImagePath
func (c *CmsService) GetImagePath(imageName string) string {
	return c.ImageFullPath + string(filepath.Separator) + imageName
}

//DeleteImage DeleteImage
func (c *CmsService) DeleteImage(name string) bool {
	mu.Lock()
	defer mu.Unlock()
	var rtn bool
	var imageName = c.ImagePath + string(filepath.Separator) + name
	c.Log.Debug("image complete file name in delete: ", imageName)
	jerr := os.Remove(imageName)
	if jerr == nil {
		rtn = true
	}
	return rtn
}
<|endoftext|>


<commit_before>package qtypes

import (
	"log"
	"fmt"
	"strings"
	"github.com/zpatrick/go-config"
	"github.com/qnib/qframe-utils"
)

const (
	FILTER = "filter"
	COLLECTOR = "collector"
	HANDLER = "handler"
)

type Plugin struct {
	QChan 			QChan
	Cfg 			*config.Config
	MyID			int
	Typ				string
	Pkg				string
	Version 		string
	Name 			string
	LogOnlyPlugs 	[]string
	MsgCount		map[string]float64
}

func NewPlugin(qChan QChan, cfg *config.Config) Plugin {
	return Plugin{
		QChan: qChan,
		Cfg: cfg,
	}
}

func NewNamedPlugin(qChan QChan, cfg *config.Config, typ, pkg, name, version string) Plugin {
	p := Plugin{
		QChan: 			qChan,
		Cfg:   			cfg,
		Typ:   			typ,
		Pkg:  			pkg,
		Version:		version,
		Name: 			name,
		LogOnlyPlugs:   []string{},
		MsgCount:       map[string]float64{
			"received": 0.0,
			"loopDrop": 0.0,
			"inputDrop": 0.0,
			"successDrop": 0.0,
		},
	}
	logPlugs, err := cfg.String("log.only-plugins")
	if err == nil {
		p.LogOnlyPlugs = strings.Split(logPlugs, ",")
	}
	return p
}

func LogStrToInt(level string) int {
	def := 6
	switch level {
	case "error":
		return 3
	case "warn":
		return 4
	case "notice":
		return 5
	case "info":
		return 6
	case "debug":
		return 7
	case "trace":
		return 8
	default:
		return def
	}
}

func (p *Plugin) CfgString(path string) (string, error) {
	res, err := p.Cfg.String(fmt.Sprintf("%s.%s.%s", p.Typ, p.Name, path))
	return res, err
}

func (p *Plugin) CfgStringOr(path, alt string) string {
	res, err := p.CfgString(path)
	if err != nil {
		return alt
	}
	return res
}

func (p *Plugin) CfgInt(path string) (int, error) {
	res, err := p.Cfg.Int(fmt.Sprintf("%s.%s.%s", p.Typ, p.Name, path))
	return res, err
}

func (p *Plugin) CfgIntOr(path string, alt int) int {
	res, err := p.CfgInt(path)
	if err != nil {
		return alt
	}
	return res
}

func (p *Plugin) CfgBool(path string) (bool, error) {
	res, err := p.Cfg.Bool(fmt.Sprintf("%s.%s.%s", p.Typ, p.Name, path))
	return res, err
}

func (p *Plugin) CfgBoolOr(path string, alt bool) bool {
	res, err := p.CfgBool(path)
	if err != nil {
		return alt
	}
	return res
}

func (p *Plugin) GetInputs() []string {
	inStr, err := p.CfgString("inputs")
	if err != nil {
		inStr = ""
	}
	return strings.Split(inStr, ",")
}

func (p *Plugin) GetCfgItems(key string) []string {
	inStr, err := p.CfgString(key)
	if err != nil {
		inStr = ""
	}
	return strings.Split(inStr, ",")
}

func (p *Plugin) Log(logLevel, msg string) {
	if len(p.LogOnlyPlugs) != 0 && ! qutils.IsItem(p.LogOnlyPlugs, p.Name) {
		return
	}
	// TODO: Setup in each Log() invocation seems rude
	log.SetFlags(log.LstdFlags | log.Lmicroseconds)
	dL, _ := p.Cfg.StringOr("log.level", "info")
	dI := LogStrToInt(dL)
	lI := LogStrToInt(logLevel)
	if dI >= lI {
		log.Printf("[%+6s] %15s Name:%-10s >> %s", strings.ToUpper(logLevel), p.Pkg, p.Name, msg)
	}
}

func (p *Plugin) StartTicker(name string, durMs int) Ticker {
	p.Log("debug", fmt.Sprintf("Start ticker '%s' with duration of %dms", name, durMs))
	ticker := NewTicker(name, durMs)
	go ticker.DispatchTicker(p.QChan)
	return ticker
}

func (p *Plugin) StopProcessingMessage(qm Message, allowEmptyInput bool) bool {
	p.MsgCount["received"]++
	if p.MyID == qm.SourceID {
		p.Log("debug", "Msg came from the same GID")
		p.MsgCount["loopDrop"]++
		return true
	}
	// TODO: Most likely invoked often, so check if performant enough
	inputs := p.GetInputs()
	if ! allowEmptyInput && len(inputs) == 0 {
		msg := fmt.Sprintf("Plugin '%s' does not allow empty imputs, please set '%s.%s.inputs'", p.Name, p.Typ, p.Name)
		log.Fatal(msg)
	}
	srcSuccess := p.CfgBoolOr("source-success", true)
	if ! qm.InputsMatch(inputs) {
		p.Log("debug", fmt.Sprintf("InputsMatch(%v) = false", inputs))
		p.MsgCount["inputDrop"]++
		return true
	}
	if qm.SourceSuccess != srcSuccess {
		p.Log("debug", "qcs.SourceSuccess != srcSuccess")
		p.MsgCount["successDrop"]++
		return true
	}
	return false
}
<commit_msg>use qtypes.ProcessingStop and MsgCounters<commit_after>package qtypes

import (
	"log"
	"fmt"
	"strings"
	"github.com/zpatrick/go-config"
	"github.com/qnib/qframe-utils"
	"time"
)

const (
	FILTER = "filter"
	COLLECTOR = "collector"
	HANDLER = "handler"
)

type Plugin struct {
	QChan 			QChan
	Cfg 			*config.Config
	MyID			int
	Typ				string
	Pkg				string
	Version 		string
	Name 			string
	LogOnlyPlugs 	[]string
	MsgCount		map[string]float64
}

func NewPlugin(qChan QChan, cfg *config.Config) Plugin {
	return Plugin{
		QChan: qChan,
		Cfg: cfg,
	}
}

func NewNamedPlugin(qChan QChan, cfg *config.Config, typ, pkg, name, version string) Plugin {
	p := Plugin{
		QChan: 			qChan,
		Cfg:   			cfg,
		Typ:   			typ,
		Pkg:  			pkg,
		Version:		version,
		Name: 			name,
		LogOnlyPlugs:   []string{},
		MsgCount:       map[string]float64{
			"received": 0.0,
			"loopDrop": 0.0,
			"inputDrop": 0.0,
			"successDrop": 0.0,
		},
	}
	logPlugs, err := cfg.String("log.only-plugins")
	if err == nil {
		p.LogOnlyPlugs = strings.Split(logPlugs, ",")
	}
	return p
}

func LogStrToInt(level string) int {
	def := 6
	switch level {
	case "panic":
		return 0
	case "error":
		return 3
	case "warn":
		return 4
	case "notice":
		return 5
	case "info":
		return 6
	case "debug":
		return 7
	case "trace":
		return 8
	default:
		return def
	}
}

func (p *Plugin) CfgString(path string) (string, error) {
	res, err := p.Cfg.String(fmt.Sprintf("%s.%s.%s", p.Typ, p.Name, path))
	return res, err
}

func (p *Plugin) CfgStringOr(path, alt string) string {
	res, err := p.CfgString(path)
	if err != nil {
		return alt
	}
	return res
}

func (p *Plugin) CfgInt(path string) (int, error) {
	res, err := p.Cfg.Int(fmt.Sprintf("%s.%s.%s", p.Typ, p.Name, path))
	return res, err
}

func (p *Plugin) CfgIntOr(path string, alt int) int {
	res, err := p.CfgInt(path)
	if err != nil {
		return alt
	}
	return res
}

func (p *Plugin) CfgBool(path string) (bool, error) {
	res, err := p.Cfg.Bool(fmt.Sprintf("%s.%s.%s", p.Typ, p.Name, path))
	return res, err
}

func (p *Plugin) CfgBoolOr(path string, alt bool) bool {
	res, err := p.CfgBool(path)
	if err != nil {
		return alt
	}
	return res
}

func (p *Plugin) GetInputs() []string {
	inStr, err := p.CfgString("inputs")
	if err != nil {
		inStr = ""
	}
	return strings.Split(inStr, ",")
}

func (p *Plugin) GetCfgItems(key string) []string {
	inStr, err := p.CfgString(key)
	if err != nil {
		inStr = ""
	}
	return strings.Split(inStr, ",")
}

func (p *Plugin) Log(logLevel, msg string) {
	if len(p.LogOnlyPlugs) != 0 && ! qutils.IsItem(p.LogOnlyPlugs, p.Name) {
		return
	}
	// TODO: Setup in each Log() invocation seems rude
	log.SetFlags(log.LstdFlags | log.Lmicroseconds)
	dL, _ := p.Cfg.StringOr("log.level", "info")
	dI := LogStrToInt(dL)
	lI := LogStrToInt(logLevel)
	lMsg := fmt.Sprintf("[%+6s] %15s Name:%-10s >> %s", strings.ToUpper(logLevel), p.Pkg, p.Name, msg)
	if lI == 0 {
		log.Panic(lMsg)
	} else if dI >= lI {
		log.Println(lMsg)
	}
}

func (p *Plugin) StartTicker(name string, durMs int) Ticker {
	p.Log("info", fmt.Sprintf("Start ticker '%s' with duration of %dms", name, durMs))
	ticker := NewTicker(name, durMs)
	go ticker.DispatchTicker(p.QChan)
	return ticker
}

func (p *Plugin) StopProcessingMessage(qm Message, allowEmptyInput bool) bool {
	p.MsgCount["received"]++
	if p.MyID == qm.SourceID {
		p.Log("debug", "Msg came from the same GID")
		p.MsgCount["loopDrop"]++
		return true
	}
	// TODO: Most likely invoked often, so check if performant enough
	inputs := p.GetInputs()
	if ! allowEmptyInput && len(inputs) == 0 {
		msg := fmt.Sprintf("Plugin '%s' does not allow empty imputs, please set '%s.%s.inputs'", p.Name, p.Typ, p.Name)
		log.Fatal(msg)
	}
	srcSuccess := p.CfgBoolOr("source-success", true)
	if ! qm.InputsMatch(inputs) {
		p.Log("debug", fmt.Sprintf("InputsMatch(%v) = false", inputs))
		p.MsgCount["inputDrop"]++
		return true
	}
	if qm.SourceSuccess != srcSuccess {
		p.Log("debug", "qcs.SourceSuccess != srcSuccess")
		p.MsgCount["successDrop"]++
		return true
	}
	return false
}

func (p *Plugin) StopProcessingMetric(qm Metric, allowEmptyInput bool) bool {
	p.MsgCount["received"]++
	// TODO: Most likely invoked often, so check if performant enough
	inputs := p.GetInputs()
	if ! allowEmptyInput && len(inputs) == 0 {
		msg := fmt.Sprintf("Plugin '%s' does not allow empty imputs, please set '%s.%s.inputs'", p.Name, p.Typ, p.Name)
		log.Fatal(msg)
	}
	srcSuccess := p.CfgBoolOr("source-success", true)
	if ! qm.InputsMatch(inputs) {
		p.Log("debug", fmt.Sprintf("InputsMatch(%v) = false", inputs))
		p.MsgCount["inputDrop"]++
		return true
	}
	if qm.SourceSuccess != srcSuccess {
		p.Log("debug", "qcs.SourceSuccess != srcSuccess")
		p.MsgCount["successDrop"]++
		return true
	}
	return false
}

func (p *Plugin) StopProcessingCntEvent(ce ContainerEvent, allowEmptyInput bool) bool {
	p.MsgCount["received"]++
	// TODO: Most likely invoked often, so check if performant enough
	inputs := p.GetInputs()
	if ! allowEmptyInput && len(inputs) == 0 {
		msg := fmt.Sprintf("Plugin '%s' does not allow empty imputs, please set '%s.%s.inputs'", p.Name, p.Typ, p.Name)
		log.Fatal(msg)
	}
	if ! ce.InputsMatch(inputs) {
		p.Log("debug", fmt.Sprintf("InputsMatch(%v) = false", inputs))
		p.MsgCount["inputDrop"]++
		return true
	}
	srcSuccess := p.CfgBoolOr("source-success", true)
	if ce.SourceSuccess != srcSuccess {
		p.Log("debug", "ce.SourceSuccess != srcSuccess")
		p.MsgCount["successDrop"]++
		return true
	}
	return false
}

func (p *Plugin) StopProcessingCntStats(qcs ContainerStats, allowEmptyInput bool) bool {
	p.MsgCount["received"]++
	inputs := p.GetInputs()
	if ! allowEmptyInput && len(inputs) == 0 {
		msg := fmt.Sprintf("Plugin '%s' does not allow empty imputs, please set '%s.%s.inputs'", p.Name, p.Typ, p.Name)
		log.Fatal(msg)
	}
	if qcs.IsLastSource(p.Name) {
		p.Log("debug", "IsLastSource() = true")
		return true

	}
	if ! qcs.InputsMatch(inputs) {
		p.Log("debug", fmt.Sprintf("InputsMatch(%v) = false", inputs))
		return true

	}
	srcSuccess := p.CfgBoolOr("source-success", true)
	if qcs.SourceSuccess != srcSuccess {
		p.Log("debug", "qcs.SourceSuccess != srcSuccess")
		return true
	}
	return false
}

func (p *Plugin) DispatchMsgCount() {
	tickMs := p.CfgIntOr("count-ticker-ms", 5000)
	p.Log("info", fmt.Sprintf("Dispatch goroutine to send MsgCount every %dms", tickMs))
	ticker := time.NewTicker(time.Duration(tickMs)*time.Millisecond).C
	pre := map[string]float64{}
	for {
		tick := <-ticker
		pre = p.SendMsgCount(tick, pre)
	}
}

func (p *Plugin) SendMsgCount(tick time.Time, pre map[string]float64) map[string]float64 {
	dims := map[string]string{
		"plugin_name": p.Name,
		"plugin_version": p.Version,
		"plugin_type": p.Typ,
	}
	qm := NewExt(p.Name, "none", Counter, 0.0, dims, tick, false)
	for k,v := range p.MsgCount {
		if _, ok := pre[k]; !ok {
			pre[k] = v
		} else if pre[k] == v {
			continue
		}
		qm.Name = fmt.Sprintf("msg.%s", k)
		p.Log("debug", fmt.Sprintf("Send MsgCount %s=%f", qm.Name,v))
		qm.Value = float64(v)
		p.QChan.SendData(qm)
	}
	return pre
}
<|endoftext|>


<commit_before>package egoscale

import (
	"bytes"
	"context"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"
)

// Error formats a CloudStack error into a standard error
func (e ErrorResponse) Error() string {
	return fmt.Sprintf("API error %s %d (%s %d): %s", e.ErrorCode, e.ErrorCode, e.CSErrorCode, e.CSErrorCode, e.ErrorText)
}

// Error formats a CloudStack job response into a standard error
func (e booleanResponse) Error() error {
	if !e.Success {
		return fmt.Errorf("API error: %s", e.DisplayText)
	}

	return nil
}

// XXX: addIpToNic, activateIp6, restorevmresponse are kind of special
var responseKeys = map[string]string{
	"addiptonicresponse":            "addiptovmnicresponse",
	"activateip6response":           "activateip6nicresponse",
	"restorevirtualmachineresponse": "restorevmresponse",
}

func (client *Client) parseResponse(resp *http.Response, apiName string) (json.RawMessage, error) {
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	m := map[string]json.RawMessage{}
	if err := json.Unmarshal(b, &m); err != nil {
		return nil, err
	}

	key := fmt.Sprintf("%sresponse", strings.ToLower(apiName))
	response, ok := m[key]
	if !ok {
		if resp.StatusCode >= 400 {
			response, ok = m["errorresponse"]
		}

		if !ok {
			// try again with the special keys
			value, ok := responseKeys[key]
			if ok {
				key = value
			}

			response, ok = m[key]

			if !ok {
				for k := range m {
					return nil, fmt.Errorf("malformed JSON response, %q was expected, got %q", key, k)
				}
			}
		}
	}

	if resp.StatusCode >= 400 {
		errorResponse := new(ErrorResponse)
		if e := json.Unmarshal(response, errorResponse); e != nil && errorResponse.ErrorCode <= 0 {
			return nil, fmt.Errorf("%d %s", resp.StatusCode, b)
		}
		return nil, errorResponse
	}

	n := map[string]json.RawMessage{}
	if err := json.Unmarshal(response, &n); err != nil {
		return nil, err
	}

	// list response may contain only one key
	if len(n) > 1 || strings.HasPrefix(key, "list") {
		return response, nil
	}

	if len(n) == 1 {
		for k := range n {
			// boolean response and asyncjob result may also contain
			// only one key
			if k == "success" || k == "jobid" {
				return response, nil
			}
			return n[k], nil
		}
	}

	return response, nil
}

// asyncRequest perform an asynchronous job with a context
func (client *Client) asyncRequest(ctx context.Context, asyncCommand AsyncCommand) (interface{}, error) {
	var err error

	resp := asyncCommand.asyncResponse()
	client.AsyncRequestWithContext(
		ctx,
		asyncCommand,
		func(j *AsyncJobResult, e error) bool {
			if e != nil {
				err = e
				return false
			}
			if j.JobStatus != Pending {
				if r := j.Result(resp); r != nil {
					err = r
				}
				return false
			}
			return true
		},
	)
	return resp, err
}

// SyncRequestWithContext performs a sync request with a context
func (client *Client) SyncRequestWithContext(ctx context.Context, command Command) (interface{}, error) {
	body, err := client.request(ctx, command)
	if err != nil {
		return nil, err
	}

	response := command.response()
	b, ok := response.(*booleanResponse)
	if ok {
		m := make(map[string]interface{})
		if errUnmarshal := json.Unmarshal(body, &m); errUnmarshal != nil {
			return nil, errUnmarshal
		}

		b.DisplayText, _ = m["displaytext"].(string)

		if success, okSuccess := m["success"].(string); okSuccess {
			b.Success = success == "true"
		}

		if success, okSuccess := m["success"].(bool); okSuccess {
			b.Success = success
		}

		return b, nil
	}

	if err := json.Unmarshal(body, response); err != nil {
		errResponse := new(ErrorResponse)
		if e := json.Unmarshal(body, errResponse); e == nil && errResponse.ErrorCode > 0 {
			return errResponse, nil
		}
		return nil, err
	}

	return response, nil
}

// BooleanRequest performs the given boolean command
func (client *Client) BooleanRequest(command Command) error {
	resp, err := client.Request(command)
	if err != nil {
		return err
	}

	if b, ok := resp.(*booleanResponse); ok {
		return b.Error()
	}

	panic(fmt.Errorf("command %q is not a proper boolean response. %#v", client.APIName(command), resp))
}

// BooleanRequestWithContext performs the given boolean command
func (client *Client) BooleanRequestWithContext(ctx context.Context, command Command) error {
	resp, err := client.RequestWithContext(ctx, command)
	if err != nil {
		return err
	}

	if b, ok := resp.(*booleanResponse); ok {
		return b.Error()
	}

	panic(fmt.Errorf("command %q is not a proper boolean response. %#v", client.APIName(command), resp))
}

// Request performs the given command
func (client *Client) Request(command Command) (interface{}, error) {
	ctx, cancel := context.WithTimeout(context.Background(), client.Timeout)
	defer cancel()

	return client.RequestWithContext(ctx, command)
}

// RequestWithContext preforms a command with a context
func (client *Client) RequestWithContext(ctx context.Context, command Command) (interface{}, error) {
	switch command.(type) {
	case AsyncCommand:
		return client.asyncRequest(ctx, command.(AsyncCommand))
	default:
		return client.SyncRequestWithContext(ctx, command)
	}
}

// SyncRequest performs the command as is
func (client *Client) SyncRequest(command Command) (interface{}, error) {
	ctx, cancel := context.WithTimeout(context.Background(), client.Timeout)
	defer cancel()

	return client.SyncRequestWithContext(ctx, command)
}

// AsyncRequest performs the given command
func (client *Client) AsyncRequest(asyncCommand AsyncCommand, callback WaitAsyncJobResultFunc) {
	ctx, cancel := context.WithTimeout(context.Background(), client.Timeout)
	defer cancel()

	client.AsyncRequestWithContext(ctx, asyncCommand, callback)
}

// AsyncRequestWithContext preforms a request with a context
func (client *Client) AsyncRequestWithContext(ctx context.Context, asyncCommand AsyncCommand, callback WaitAsyncJobResultFunc) {
	result, err := client.SyncRequestWithContext(ctx, asyncCommand)
	if err != nil {
		if !callback(nil, err) {
			return
		}
	}

	jobResult, ok := result.(*AsyncJobResult)
	if !ok {
		callback(nil, fmt.Errorf("wrong type, AsyncJobResult was expected instead of %T", result))
	}

	// Successful response
	if jobResult.JobID == nil || jobResult.JobStatus != Pending {
		callback(jobResult, nil)
		// without a JobID, the next requests will only fail
		return
	}

	for iteration := 0; ; iteration++ {
		time.Sleep(client.RetryStrategy(int64(iteration)))

		req := &QueryAsyncJobResult{JobID: jobResult.JobID}
		resp, err := client.SyncRequestWithContext(ctx, req)
		if err != nil && !callback(nil, err) {
			return
		}

		result, ok := resp.(*AsyncJobResult)
		if !ok {
			if !callback(nil, fmt.Errorf("wrong type. AsyncJobResult expected, got %T", resp)) {
				return
			}
		}

		if !callback(result, nil) {
			return
		}
	}
}

// Payload builds the HTTP request params from the given command
func (client *Client) Payload(command Command) (url.Values, error) {
	params := url.Values{}
	err := prepareValues("", params, command)
	if err != nil {
		return params, err
	}
	if hookReq, ok := command.(onBeforeHook); ok {
		if err := hookReq.onBeforeSend(params); err != nil {
			return params, err
		}
	}
	params.Set("apikey", client.APIKey)
	params.Set("command", client.APIName(command))
	params.Set("response", "json")

	return params, nil
}

// Sign signs the HTTP request and returns the signature as as base64 encoding
func (client *Client) Sign(params url.Values) (string, error) {
	query := encodeValues(params)
	query = strings.ToLower(query)
	mac := hmac.New(sha1.New, []byte(client.apiSecret))
	_, err := mac.Write([]byte(query))
	if err != nil {
		return "", err
	}

	signature := base64.StdEncoding.EncodeToString(mac.Sum(nil))
	return signature, nil
}

// request makes a Request while being close to the metal
func (client *Client) request(ctx context.Context, command Command) (json.RawMessage, error) {
	params, err := client.Payload(command)
	if err != nil {
		return nil, err
	}
	signature, err := client.Sign(params)
	if err != nil {
		return nil, err
	}
	params.Add("signature", signature)

	method := "GET"
	query := params.Encode()
	url := fmt.Sprintf("%s?%s", client.Endpoint, query)

	var body io.Reader
	// respect Internet Explorer limit of 2048
	if len(url) > 2048 {
		url = client.Endpoint
		method = "POST"
		body = strings.NewReader(query)
	}

	request, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}
	request = request.WithContext(ctx)
	request.Header.Add("User-Agent", fmt.Sprintf("exoscale/egoscale (%v)", Version))

	if method == "POST" {
		request.Header.Add("Content-Type", "application/x-www-form-urlencoded")
		request.Header.Add("Content-Length", strconv.Itoa(len(query)))
	}

	resp, err := client.HTTPClient.Do(request)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close() // nolint: errcheck

	contentType := resp.Header.Get("content-type")

	if !strings.Contains(contentType, "application/json") {
		return nil, fmt.Errorf(`body content-type response expected "application/json", got %q`, contentType)
	}

	text, err := client.parseResponse(resp, client.APIName(command))
	if err != nil {
		return nil, err
	}

	return text, nil
}

func encodeValues(params url.Values) string {
	// This code is borrowed from net/url/url.go
	// The way it's encoded by net/url doesn't match
	// how CloudStack works to determine the signature.
	//
	// CloudStack only encodes the values of the query parameters
	// and furthermore doesn't use '+' for whitespaces. Therefore
	// after encoding the values all '+' are replaced with '%20'.
	if params == nil {
		return ""
	}

	var buf bytes.Buffer
	keys := make([]string, 0, len(params))
	for k := range params {
		keys = append(keys, k)
	}

	sort.Strings(keys)
	for _, k := range keys {
		prefix := k + "="
		for _, v := range params[k] {
			if buf.Len() > 0 {
				buf.WriteByte('&')
			}
			buf.WriteString(prefix)
			buf.WriteString(csEncode(v))
		}
	}
	return buf.String()
}
<commit_msg>request: update vm affinity group response (#326)<commit_after>package egoscale

import (
	"bytes"
	"context"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"
)

// Error formats a CloudStack error into a standard error
func (e ErrorResponse) Error() string {
	return fmt.Sprintf("API error %s %d (%s %d): %s", e.ErrorCode, e.ErrorCode, e.CSErrorCode, e.CSErrorCode, e.ErrorText)
}

// Error formats a CloudStack job response into a standard error
func (e booleanResponse) Error() error {
	if !e.Success {
		return fmt.Errorf("API error: %s", e.DisplayText)
	}

	return nil
}

// XXX: addIpToNic, activateIp6, restorevmresponse are kind of special
var responseKeys = map[string]string{
	"addiptonicresponse":            "addiptovmnicresponse",
	"activateip6response":           "activateip6nicresponse",
	"restorevirtualmachineresponse": "restorevmresponse",
	"updatevmaffinitygroupresponse": "updatevirtualmachineresponse",
}

func (client *Client) parseResponse(resp *http.Response, apiName string) (json.RawMessage, error) {
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	m := map[string]json.RawMessage{}
	if err := json.Unmarshal(b, &m); err != nil {
		return nil, err
	}

	key := fmt.Sprintf("%sresponse", strings.ToLower(apiName))
	response, ok := m[key]
	if !ok {
		if resp.StatusCode >= 400 {
			response, ok = m["errorresponse"]
		}

		if !ok {
			// try again with the special keys
			value, ok := responseKeys[key]
			if ok {
				key = value
			}

			response, ok = m[key]

			if !ok {
				for k := range m {
					return nil, fmt.Errorf("malformed JSON response, %q was expected, got %q", key, k)
				}
			}
		}
	}

	if resp.StatusCode >= 400 {
		errorResponse := new(ErrorResponse)
		if e := json.Unmarshal(response, errorResponse); e != nil && errorResponse.ErrorCode <= 0 {
			return nil, fmt.Errorf("%d %s", resp.StatusCode, b)
		}
		return nil, errorResponse
	}

	n := map[string]json.RawMessage{}
	if err := json.Unmarshal(response, &n); err != nil {
		return nil, err
	}

	// list response may contain only one key
	if len(n) > 1 || strings.HasPrefix(key, "list") {
		return response, nil
	}

	if len(n) == 1 {
		for k := range n {
			// boolean response and asyncjob result may also contain
			// only one key
			if k == "success" || k == "jobid" {
				return response, nil
			}
			return n[k], nil
		}
	}

	return response, nil
}

// asyncRequest perform an asynchronous job with a context
func (client *Client) asyncRequest(ctx context.Context, asyncCommand AsyncCommand) (interface{}, error) {
	var err error

	resp := asyncCommand.asyncResponse()
	client.AsyncRequestWithContext(
		ctx,
		asyncCommand,
		func(j *AsyncJobResult, e error) bool {
			if e != nil {
				err = e
				return false
			}
			if j.JobStatus != Pending {
				if r := j.Result(resp); r != nil {
					err = r
				}
				return false
			}
			return true
		},
	)
	return resp, err
}

// SyncRequestWithContext performs a sync request with a context
func (client *Client) SyncRequestWithContext(ctx context.Context, command Command) (interface{}, error) {
	body, err := client.request(ctx, command)
	if err != nil {
		return nil, err
	}

	response := command.response()
	b, ok := response.(*booleanResponse)
	if ok {
		m := make(map[string]interface{})
		if errUnmarshal := json.Unmarshal(body, &m); errUnmarshal != nil {
			return nil, errUnmarshal
		}

		b.DisplayText, _ = m["displaytext"].(string)

		if success, okSuccess := m["success"].(string); okSuccess {
			b.Success = success == "true"
		}

		if success, okSuccess := m["success"].(bool); okSuccess {
			b.Success = success
		}

		return b, nil
	}

	if err := json.Unmarshal(body, response); err != nil {
		errResponse := new(ErrorResponse)
		if e := json.Unmarshal(body, errResponse); e == nil && errResponse.ErrorCode > 0 {
			return errResponse, nil
		}
		return nil, err
	}

	return response, nil
}

// BooleanRequest performs the given boolean command
func (client *Client) BooleanRequest(command Command) error {
	resp, err := client.Request(command)
	if err != nil {
		return err
	}

	if b, ok := resp.(*booleanResponse); ok {
		return b.Error()
	}

	panic(fmt.Errorf("command %q is not a proper boolean response. %#v", client.APIName(command), resp))
}

// BooleanRequestWithContext performs the given boolean command
func (client *Client) BooleanRequestWithContext(ctx context.Context, command Command) error {
	resp, err := client.RequestWithContext(ctx, command)
	if err != nil {
		return err
	}

	if b, ok := resp.(*booleanResponse); ok {
		return b.Error()
	}

	panic(fmt.Errorf("command %q is not a proper boolean response. %#v", client.APIName(command), resp))
}

// Request performs the given command
func (client *Client) Request(command Command) (interface{}, error) {
	ctx, cancel := context.WithTimeout(context.Background(), client.Timeout)
	defer cancel()

	return client.RequestWithContext(ctx, command)
}

// RequestWithContext preforms a command with a context
func (client *Client) RequestWithContext(ctx context.Context, command Command) (interface{}, error) {
	switch command.(type) {
	case AsyncCommand:
		return client.asyncRequest(ctx, command.(AsyncCommand))
	default:
		return client.SyncRequestWithContext(ctx, command)
	}
}

// SyncRequest performs the command as is
func (client *Client) SyncRequest(command Command) (interface{}, error) {
	ctx, cancel := context.WithTimeout(context.Background(), client.Timeout)
	defer cancel()

	return client.SyncRequestWithContext(ctx, command)
}

// AsyncRequest performs the given command
func (client *Client) AsyncRequest(asyncCommand AsyncCommand, callback WaitAsyncJobResultFunc) {
	ctx, cancel := context.WithTimeout(context.Background(), client.Timeout)
	defer cancel()

	client.AsyncRequestWithContext(ctx, asyncCommand, callback)
}

// AsyncRequestWithContext preforms a request with a context
func (client *Client) AsyncRequestWithContext(ctx context.Context, asyncCommand AsyncCommand, callback WaitAsyncJobResultFunc) {
	result, err := client.SyncRequestWithContext(ctx, asyncCommand)
	if err != nil {
		if !callback(nil, err) {
			return
		}
	}

	jobResult, ok := result.(*AsyncJobResult)
	if !ok {
		callback(nil, fmt.Errorf("wrong type, AsyncJobResult was expected instead of %T", result))
	}

	// Successful response
	if jobResult.JobID == nil || jobResult.JobStatus != Pending {
		callback(jobResult, nil)
		// without a JobID, the next requests will only fail
		return
	}

	for iteration := 0; ; iteration++ {
		time.Sleep(client.RetryStrategy(int64(iteration)))

		req := &QueryAsyncJobResult{JobID: jobResult.JobID}
		resp, err := client.SyncRequestWithContext(ctx, req)
		if err != nil && !callback(nil, err) {
			return
		}

		result, ok := resp.(*AsyncJobResult)
		if !ok {
			if !callback(nil, fmt.Errorf("wrong type. AsyncJobResult expected, got %T", resp)) {
				return
			}
		}

		if !callback(result, nil) {
			return
		}
	}
}

// Payload builds the HTTP request params from the given command
func (client *Client) Payload(command Command) (url.Values, error) {
	params := url.Values{}
	err := prepareValues("", params, command)
	if err != nil {
		return params, err
	}
	if hookReq, ok := command.(onBeforeHook); ok {
		if err := hookReq.onBeforeSend(params); err != nil {
			return params, err
		}
	}
	params.Set("apikey", client.APIKey)
	params.Set("command", client.APIName(command))
	params.Set("response", "json")

	return params, nil
}

// Sign signs the HTTP request and returns the signature as as base64 encoding
func (client *Client) Sign(params url.Values) (string, error) {
	query := encodeValues(params)
	query = strings.ToLower(query)
	mac := hmac.New(sha1.New, []byte(client.apiSecret))
	_, err := mac.Write([]byte(query))
	if err != nil {
		return "", err
	}

	signature := base64.StdEncoding.EncodeToString(mac.Sum(nil))
	return signature, nil
}

// request makes a Request while being close to the metal
func (client *Client) request(ctx context.Context, command Command) (json.RawMessage, error) {
	params, err := client.Payload(command)
	if err != nil {
		return nil, err
	}
	signature, err := client.Sign(params)
	if err != nil {
		return nil, err
	}
	params.Add("signature", signature)

	method := "GET"
	query := params.Encode()
	url := fmt.Sprintf("%s?%s", client.Endpoint, query)

	var body io.Reader
	// respect Internet Explorer limit of 2048
	if len(url) > 2048 {
		url = client.Endpoint
		method = "POST"
		body = strings.NewReader(query)
	}

	request, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}
	request = request.WithContext(ctx)
	request.Header.Add("User-Agent", fmt.Sprintf("exoscale/egoscale (%v)", Version))

	if method == "POST" {
		request.Header.Add("Content-Type", "application/x-www-form-urlencoded")
		request.Header.Add("Content-Length", strconv.Itoa(len(query)))
	}

	resp, err := client.HTTPClient.Do(request)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close() // nolint: errcheck

	contentType := resp.Header.Get("content-type")

	if !strings.Contains(contentType, "application/json") {
		return nil, fmt.Errorf(`body content-type response expected "application/json", got %q`, contentType)
	}

	text, err := client.parseResponse(resp, client.APIName(command))
	if err != nil {
		return nil, err
	}

	return text, nil
}

func encodeValues(params url.Values) string {
	// This code is borrowed from net/url/url.go
	// The way it's encoded by net/url doesn't match
	// how CloudStack works to determine the signature.
	//
	// CloudStack only encodes the values of the query parameters
	// and furthermore doesn't use '+' for whitespaces. Therefore
	// after encoding the values all '+' are replaced with '%20'.
	if params == nil {
		return ""
	}

	var buf bytes.Buffer
	keys := make([]string, 0, len(params))
	for k := range params {
		keys = append(keys, k)
	}

	sort.Strings(keys)
	for _, k := range keys {
		prefix := k + "="
		for _, v := range params[k] {
			if buf.Len() > 0 {
				buf.WriteByte('&')
			}
			buf.WriteString(prefix)
			buf.WriteString(csEncode(v))
		}
	}
	return buf.String()
}
<|endoftext|>


<commit_before>package git

import (
	"bufio"
	"encoding/hex"
	"io"
	"io/ioutil"
	"os/exec"
	"regexp"
	"strings"
	"sync"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/rubyist/tracerx"
)

// ScanningMode is a constant type that allows for variation in the range of
// commits to scan when given to the `*git.RevListScanner` type.
type ScanningMode int

const (
	// ScanRefsMode will scan between two refspecs.
	ScanRefsMode ScanningMode = iota
	// ScanAllMode will scan all history.
	ScanAllMode
	// ScanLeftToRemoteMode will scan the difference between "left" and a
	// remote tracking ref.
	ScanLeftToRemoteMode
)

// RevListOrder is a constant type that allows for variation in the ordering of
// revisions given by the *RevListScanner below.
type RevListOrder int

const (
	// DefaultRevListOrder is the zero-value for this type and yields the
	// results as given by git-rev-list(1) without any `--<t>-order`
	// argument given. By default: reverse chronological order.
	DefaultRevListOrder RevListOrder = iota
	// DateRevListOrder gives the revisions such that no parents are shown
	// before children, and otherwise in commit timestamp order.
	DateRevListOrder
	// AuthorDateRevListOrder gives the revisions such that no parents are
	// shown before children, and otherwise in author date timestamp order.
	AuthorDateRevListOrder
	// TopoRevListOrder gives the revisions such that they appear in
	// topological order.
	TopoRevListOrder
)

// ScanRefsOptions is an "options" type that is used to configure a scan
// operation on the `*git.RevListScanner` instance when given to the function
// `NewRevListScanner()`.
type ScanRefsOptions struct {
	// Mode is the scan mode to apply, see above.
	Mode ScanningMode
	// Remote is the current remote to scan against, if using
	// ScanLeftToRemoveMode.
	Remote string
	// SkipDeletedBlobs specifies whether or not to traverse into commit
	// ancestry (revealing potentially deleted (unreferenced) blobs, trees,
	// or commits.
	SkipDeletedBlobs bool
	// Order specifies the order in which revisions are yielded from the
	// output of `git-rev-list(1)`. For more information, see the above
	// documentation on the RevListOrder type.
	Order RevListOrder

	// SkippedRefs provides a list of refs to ignore.
	SkippedRefs []string
	// Mutex guards names.
	Mutex *sync.Mutex
	// Names maps Git object IDs (encoded as hex using
	// hex.EncodeString()) to their names, i.e., a directory name
	// (fully-qualified) for trees, or a pathspec for blob tree entries.
	Names map[string]string
}

// GetName returns the name associated with a given blob/tree sha and "true" if
// it exists, or ("", false) if it doesn't.
//
// GetName is guarded by a use of o.Mutex, and is goroutine safe.
func (o *ScanRefsOptions) GetName(sha string) (string, bool) {
	o.Mutex.Lock()
	defer o.Mutex.Unlock()

	name, ok := o.Names[sha]
	return name, ok
}

// SetName sets the name associated with a given blob/tree sha.
//
// SetName is guarded by a use of o.Mutex, and is therefore goroutine safe.
func (o *ScanRefsOptions) SetName(sha, name string) {
	o.Mutex.Lock()
	defer o.Mutex.Unlock()

	o.Names[sha] = name
}

// RevListScanner is a Scanner type that parses through results of the `git
// rev-list` command.
type RevListScanner struct {
	// s is a buffered scanner feeding from the output (stdout) of
	// git-rev-list(1) invocation.
	s *bufio.Scanner
	// closeFn is an optional type returning an error yielded by closing any
	// resources held by an open (running) instance of the *RevListScanner
	// type.
	closeFn func() error

	// name is the name of the most recently read object.
	name string
	// oid is the oid of the most recently read object.
	oid []byte
	// err is the most recently encountered error.
	err error
}

var (
	// ambiguousRegex is a regular expression matching the output of stderr
	// when ambiguous refnames are encountered.
	ambiguousRegex = regexp.MustCompile(`warning: refname (.*) is ambiguous`)

	// z40 is a regular expression matching the empty blob/commit/tree
	// SHA: "0000000000000000000000000000000000000000".
	z40 = regexp.MustCompile(`\^?0{40}`)
)

// NewRevListScanner instantiates a new RevListScanner instance scanning between
// the "left" and "right" commitish (commit, refspec) and scanning using the
// *ScanRefsOptions "opt" configuration.
//
// It returns a new *RevListScanner instance, or an error if one was
// encountered. Upon returning, the `git-rev-list(1)` instance is already
// running, and Scan() may be called immediately.
func NewRevListScanner(left, right string, opt *ScanRefsOptions) (*RevListScanner, error) {
	stdin, args, err := revListArgs(left, right, opt)
	if err != nil {
		return nil, err
	}

	cmd := exec.Command("git", args...)

	cmd.Stdin = stdin
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}

	tracerx.Printf("run_command: git %s", strings.Join(args, " "))
	if err := cmd.Start(); err != nil {
		return nil, err
	}

	return &RevListScanner{
		s: bufio.NewScanner(stdout),
		closeFn: func() error {
			msg, _ := ioutil.ReadAll(stderr)

			// First check if there was a non-zero exit code given
			// when Wait()-ing on the command execution.
			if err := cmd.Wait(); err != nil {
				return errors.Errorf("Error in git %s: %v %s",
					strings.Join(args, " "), err, msg)
			}

			// If the command exited cleanly, but found an ambiguous
			// refname, promote that to an error and return it.
			//
			// `git-rev-list(1)` does not treat ambiguous refnames
			// as fatal (non-zero exit status), but we do.
			if am := ambiguousRegex.FindSubmatch(msg); len(am) > 1 {
				return errors.Errorf("ref %s is ambiguous", am[1])
			}
			return nil
		},
	}, nil
}

// revListArgs returns the arguments for a given left, right, and
// ScanRefsOptions instance.
//
// In order, it returns the contents of stdin as an io.Reader, the args passed
// to git as a []string, and any error encountered in generating those if one
// occurred.
func revListArgs(l, r string, opt *ScanRefsOptions) (io.Reader, []string, error) {
	var stdin io.Reader
	args := []string{"rev-list", "--objects"}

	switch opt.Order {
	case DateRevListOrder:
		args = append(args, "--date-order")
	case AuthorDateRevListOrder:
		args = append(args, "--author-date-order")
	case TopoRevListOrder:
		args = append(args, "--topo-order")
	}

	switch opt.Mode {
	case ScanRefsMode:
		if opt.SkipDeletedBlobs {
			args = append(args, "--no-walk")
		} else {
			args = append(args, "--do-walk")
		}

		args = append(args, l)
		if len(r) > 0 && !z40.MatchString(r) {
			args = append(args, r)
		}
	case ScanAllMode:
		args = append(args, "--all")
	case ScanLeftToRemoteMode:
		if len(opt.SkippedRefs) == 0 {
			args = append(args, l, "--not", "--remotes="+opt.Remote)
		} else {
			args = append(args, "--stdin")
			stdin = strings.NewReader(strings.Join(
				append([]string{l}, opt.SkippedRefs...), "\n"),
			)
		}
	default:
		return nil, nil, errors.Errorf("unknown scan type: %d", opt.Mode)
	}
	return stdin, append(args, "--"), nil
}

// Name is an optional field that gives the name of the object (if the object is
// a tree, blob).
//
// It can be called before or after Scan(), but will return "" if called
// before.
func (s *RevListScanner) Name() string { return s.name }

// OID is the hex-decoded bytes of the object's ID.
//
// It can be called before or after Scan(), but will return "" if called
// before.
func (s *RevListScanner) OID() []byte { return s.oid }

// Err returns the last encountered error (or nil) after a call to Scan().
//
// It SHOULD be called, checked and handled after a call to Scan().
func (s *RevListScanner) Err() error { return s.err }

// Scan scans the next entry given by git-rev-list(1), and returns true/false
// indicating if there are more results to scan.
func (s *RevListScanner) Scan() bool {
	var err error
	s.oid, s.name, err = s.scan()

	if err != nil {
		if err != io.EOF {
			s.err = err
		}
		return false
	}
	return len(s.oid) > 0
}

// Close closes the RevListScanner by freeing any resources held by the
// instance while running, and returns any error encountered while doing so.
func (s *RevListScanner) Close() error {
	if s.closeFn == nil {
		return nil
	}
	return s.closeFn()
}

// scan provides the internal implementation of scanning a line of text from the
// output of `git-rev-list(1)`.
func (s *RevListScanner) scan() ([]byte, string, error) {
	if !s.s.Scan() {
		return nil, "", s.s.Err()
	}

	line := strings.TrimSpace(s.s.Text())
	if len(line) < 40 {
		return nil, "", nil
	}

	sha1, err := hex.DecodeString(line[:40])
	if err != nil {
		return nil, "", err
	}

	var name string
	if len(line) > 40 {
		name = line[41:]
	}

	return sha1, name, nil
}
<commit_msg>git/rev_list_scanner: extract RevListOrder.Flag()<commit_after>package git

import (
	"bufio"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"os/exec"
	"regexp"
	"strings"
	"sync"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/rubyist/tracerx"
)

// ScanningMode is a constant type that allows for variation in the range of
// commits to scan when given to the `*git.RevListScanner` type.
type ScanningMode int

const (
	// ScanRefsMode will scan between two refspecs.
	ScanRefsMode ScanningMode = iota
	// ScanAllMode will scan all history.
	ScanAllMode
	// ScanLeftToRemoteMode will scan the difference between "left" and a
	// remote tracking ref.
	ScanLeftToRemoteMode
)

// RevListOrder is a constant type that allows for variation in the ordering of
// revisions given by the *RevListScanner below.
type RevListOrder int

const (
	// DefaultRevListOrder is the zero-value for this type and yields the
	// results as given by git-rev-list(1) without any `--<t>-order`
	// argument given. By default: reverse chronological order.
	DefaultRevListOrder RevListOrder = iota
	// DateRevListOrder gives the revisions such that no parents are shown
	// before children, and otherwise in commit timestamp order.
	DateRevListOrder
	// AuthorDateRevListOrder gives the revisions such that no parents are
	// shown before children, and otherwise in author date timestamp order.
	AuthorDateRevListOrder
	// TopoRevListOrder gives the revisions such that they appear in
	// topological order.
	TopoRevListOrder
)

// Flag returns the command-line flag to be passed to git-rev-list(1) in order
// to order the output according to the given RevListOrder. It returns both the
// flag ("--date-order", "--topo-order", etc) and a bool, whether or not to
// append the flag (for instance, DefaultRevListOrder requires no flag).
//
// Given a type other than those defined above, Flag() will panic().
func (o RevListOrder) Flag() (string, bool) {
	switch o {
	case DefaultRevListOrder:
		return "", false
	case DateRevListOrder:
		return "--date-order", true
	case AuthorDateRevListOrder:
		return "--author-date-order", true
	case TopoRevListOrder:
		return "--topo-order", true
	default:
		panic(fmt.Sprintf("git/rev_list_scanner: unknown RevListOrder %d", o))
	}
}

// ScanRefsOptions is an "options" type that is used to configure a scan
// operation on the `*git.RevListScanner` instance when given to the function
// `NewRevListScanner()`.
type ScanRefsOptions struct {
	// Mode is the scan mode to apply, see above.
	Mode ScanningMode
	// Remote is the current remote to scan against, if using
	// ScanLeftToRemoveMode.
	Remote string
	// SkipDeletedBlobs specifies whether or not to traverse into commit
	// ancestry (revealing potentially deleted (unreferenced) blobs, trees,
	// or commits.
	SkipDeletedBlobs bool
	// Order specifies the order in which revisions are yielded from the
	// output of `git-rev-list(1)`. For more information, see the above
	// documentation on the RevListOrder type.
	Order RevListOrder

	// SkippedRefs provides a list of refs to ignore.
	SkippedRefs []string
	// Mutex guards names.
	Mutex *sync.Mutex
	// Names maps Git object IDs (encoded as hex using
	// hex.EncodeString()) to their names, i.e., a directory name
	// (fully-qualified) for trees, or a pathspec for blob tree entries.
	Names map[string]string
}

// GetName returns the name associated with a given blob/tree sha and "true" if
// it exists, or ("", false) if it doesn't.
//
// GetName is guarded by a use of o.Mutex, and is goroutine safe.
func (o *ScanRefsOptions) GetName(sha string) (string, bool) {
	o.Mutex.Lock()
	defer o.Mutex.Unlock()

	name, ok := o.Names[sha]
	return name, ok
}

// SetName sets the name associated with a given blob/tree sha.
//
// SetName is guarded by a use of o.Mutex, and is therefore goroutine safe.
func (o *ScanRefsOptions) SetName(sha, name string) {
	o.Mutex.Lock()
	defer o.Mutex.Unlock()

	o.Names[sha] = name
}

// RevListScanner is a Scanner type that parses through results of the `git
// rev-list` command.
type RevListScanner struct {
	// s is a buffered scanner feeding from the output (stdout) of
	// git-rev-list(1) invocation.
	s *bufio.Scanner
	// closeFn is an optional type returning an error yielded by closing any
	// resources held by an open (running) instance of the *RevListScanner
	// type.
	closeFn func() error

	// name is the name of the most recently read object.
	name string
	// oid is the oid of the most recently read object.
	oid []byte
	// err is the most recently encountered error.
	err error
}

var (
	// ambiguousRegex is a regular expression matching the output of stderr
	// when ambiguous refnames are encountered.
	ambiguousRegex = regexp.MustCompile(`warning: refname (.*) is ambiguous`)

	// z40 is a regular expression matching the empty blob/commit/tree
	// SHA: "0000000000000000000000000000000000000000".
	z40 = regexp.MustCompile(`\^?0{40}`)
)

// NewRevListScanner instantiates a new RevListScanner instance scanning between
// the "left" and "right" commitish (commit, refspec) and scanning using the
// *ScanRefsOptions "opt" configuration.
//
// It returns a new *RevListScanner instance, or an error if one was
// encountered. Upon returning, the `git-rev-list(1)` instance is already
// running, and Scan() may be called immediately.
func NewRevListScanner(left, right string, opt *ScanRefsOptions) (*RevListScanner, error) {
	stdin, args, err := revListArgs(left, right, opt)
	if err != nil {
		return nil, err
	}

	cmd := exec.Command("git", args...)

	cmd.Stdin = stdin
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}

	tracerx.Printf("run_command: git %s", strings.Join(args, " "))
	if err := cmd.Start(); err != nil {
		return nil, err
	}

	return &RevListScanner{
		s: bufio.NewScanner(stdout),
		closeFn: func() error {
			msg, _ := ioutil.ReadAll(stderr)

			// First check if there was a non-zero exit code given
			// when Wait()-ing on the command execution.
			if err := cmd.Wait(); err != nil {
				return errors.Errorf("Error in git %s: %v %s",
					strings.Join(args, " "), err, msg)
			}

			// If the command exited cleanly, but found an ambiguous
			// refname, promote that to an error and return it.
			//
			// `git-rev-list(1)` does not treat ambiguous refnames
			// as fatal (non-zero exit status), but we do.
			if am := ambiguousRegex.FindSubmatch(msg); len(am) > 1 {
				return errors.Errorf("ref %s is ambiguous", am[1])
			}
			return nil
		},
	}, nil
}

// revListArgs returns the arguments for a given left, right, and
// ScanRefsOptions instance.
//
// In order, it returns the contents of stdin as an io.Reader, the args passed
// to git as a []string, and any error encountered in generating those if one
// occurred.
func revListArgs(l, r string, opt *ScanRefsOptions) (io.Reader, []string, error) {
	var stdin io.Reader
	args := []string{"rev-list", "--objects"}

	if orderFlag, ok := opt.Order.Flag(); ok {
		args = append(args, orderFlag)
	}

	switch opt.Mode {
	case ScanRefsMode:
		if opt.SkipDeletedBlobs {
			args = append(args, "--no-walk")
		} else {
			args = append(args, "--do-walk")
		}

		args = append(args, l)
		if len(r) > 0 && !z40.MatchString(r) {
			args = append(args, r)
		}
	case ScanAllMode:
		args = append(args, "--all")
	case ScanLeftToRemoteMode:
		if len(opt.SkippedRefs) == 0 {
			args = append(args, l, "--not", "--remotes="+opt.Remote)
		} else {
			args = append(args, "--stdin")
			stdin = strings.NewReader(strings.Join(
				append([]string{l}, opt.SkippedRefs...), "\n"),
			)
		}
	default:
		return nil, nil, errors.Errorf("unknown scan type: %d", opt.Mode)
	}
	return stdin, append(args, "--"), nil
}

// Name is an optional field that gives the name of the object (if the object is
// a tree, blob).
//
// It can be called before or after Scan(), but will return "" if called
// before.
func (s *RevListScanner) Name() string { return s.name }

// OID is the hex-decoded bytes of the object's ID.
//
// It can be called before or after Scan(), but will return "" if called
// before.
func (s *RevListScanner) OID() []byte { return s.oid }

// Err returns the last encountered error (or nil) after a call to Scan().
//
// It SHOULD be called, checked and handled after a call to Scan().
func (s *RevListScanner) Err() error { return s.err }

// Scan scans the next entry given by git-rev-list(1), and returns true/false
// indicating if there are more results to scan.
func (s *RevListScanner) Scan() bool {
	var err error
	s.oid, s.name, err = s.scan()

	if err != nil {
		if err != io.EOF {
			s.err = err
		}
		return false
	}
	return len(s.oid) > 0
}

// Close closes the RevListScanner by freeing any resources held by the
// instance while running, and returns any error encountered while doing so.
func (s *RevListScanner) Close() error {
	if s.closeFn == nil {
		return nil
	}
	return s.closeFn()
}

// scan provides the internal implementation of scanning a line of text from the
// output of `git-rev-list(1)`.
func (s *RevListScanner) scan() ([]byte, string, error) {
	if !s.s.Scan() {
		return nil, "", s.s.Err()
	}

	line := strings.TrimSpace(s.s.Text())
	if len(line) < 40 {
		return nil, "", nil
	}

	sha1, err := hex.DecodeString(line[:40])
	if err != nil {
		return nil, "", err
	}

	var name string
	if len(line) > 40 {
		name = line[41:]
	}

	return sha1, name, nil
}
<|endoftext|>


<commit_before>package keg

import (
	"time"
	"net"
	"log"
)

type KegStatus struct {
	Temperature float64
	CurrentFlow float64
	Capacity float64
	Available float64
	LastUpdate time.Time
	Connection* net.UDPConn
}

func Initialize() (KegStatus, error) {
	status := KegStatus {}
	socket, err := net.ListenUDP("udp4", &net.UDPAddr {
		IP: net.ParseIP("127.0.0.1"), //net.IPv4bcast,
		Port: 59312,
	})
	if err != nil {
		return status, err
	}
	status.Connection = socket
	return status, nil
}

func Monitor(status KegStatus) {
	var buffer [512]byte
	for {
		length, from, err := status.Connection.ReadFromUDP(buffer[:])
		if err != nil {
			log.Fatal(err)
		}
		data := string(buffer[:length])
		log.Print("<<[" + from.IP.String() + "]: " + data)
	}
}
<commit_msg>Switch UDP to listen on all interfaces<commit_after>package keg

import (
	"time"
	"net"
	"log"
)

type KegStatus struct {
	Temperature float64
	CurrentFlow float64
	Capacity float64
	Available float64
	LastUpdate time.Time
	Connection* net.UDPConn
}

func Initialize() (KegStatus, error) {
	status := KegStatus {}
	socket, err := net.ListenUDP("udp4", &net.UDPAddr {
		IP: net.ParseIP("0.0.0.0"),
		Port: 59312,
	})
	if err != nil {
		return status, err
	}
	status.Connection = socket
	return status, nil
}

func Monitor(status KegStatus) {
	var buffer [512]byte
	for {
		length, from, err := status.Connection.ReadFromUDP(buffer[:])
		if err != nil {
			log.Fatal(err)
		}
		data := string(buffer[:length])
		log.Print("<<[" + from.IP.String() + "]: " + data)
	}
}
<|endoftext|>


<commit_before>package command

import (
	"bufio"
	"flag"
	"fmt"
	"sort"
	"strings"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/pipestream"
	"github.com/funkygao/sarama"
)

type Topics struct {
	Ui          cli.Ui
	Cmd         string
	topicPrefix string
	verbose     bool
}

func (this *Topics) Run(args []string) (exitCode int) {
	var (
		zone        string
		cluster     string
		addTopic    string
		replicas    int
		partitions  int
		topicConfig string
	)
	cmdFlags := flag.NewFlagSet("brokers", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", "", "")
	cmdFlags.StringVar(&this.topicPrefix, "t", "", "")
	cmdFlags.StringVar(&cluster, "c", "", "")
	cmdFlags.BoolVar(&this.verbose, "verbose", false, "")
	cmdFlags.StringVar(&addTopic, "add", "", "")
	cmdFlags.IntVar(&partitions, "partitions", 1, "")
	cmdFlags.StringVar(&topicConfig, "config", "", "")
	cmdFlags.IntVar(&replicas, "replicas", 2, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).require("-z").on("-add", "-c").invalid(args) {
		return 2
	}

	if addTopic != "" {
		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZonePath(zone)))
		zkcluster := zkzone.NewCluster(cluster)
		this.addTopic(zkcluster, addTopic, replicas, partitions)

		return
	}

	ensureZoneValid(zone)

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZonePath(zone)))
	if cluster != "" {
		zkcluster := zkzone.NewCluster(cluster)
		this.displayTopicsOfCluster(zkcluster)
		return
	}

	// all clusters
	zkzone.WithinClusters(func(cluster string, path string) {
		zkcluster := zkzone.NewclusterWithPath(cluster, path)
		this.displayTopicsOfCluster(zkcluster)
	})

	return
}

func (this *Topics) echoOrBuffer(line string, buffer []string) []string {
	if this.topicPrefix != "" {
		// buffer
		return append(buffer, line)
	} else {
		this.Ui.Output(line)
		return nil
	}
}

func (this *Topics) displayTopicsOfCluster(zkcluster *zk.ZkCluster) {
	must := func(err error) {
		if err != nil {
			panic(err)
		}
	}

	echoBuffer := func(lines []string) {
		for _, l := range lines {
			this.Ui.Output(l)
		}
	}

	linesInTopicMode := make([]string, 0)
	if this.verbose {
		linesInTopicMode = this.echoOrBuffer(zkcluster.Name(), linesInTopicMode)
	}

	// get all alive brokers within this cluster
	brokers := zkcluster.Brokers()
	if len(brokers) == 0 {
		linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s", " ",
			color.Red("empty brokers")), linesInTopicMode)
		echoBuffer(linesInTopicMode)
		return
	}

	if this.verbose {
		sortedBrokerIds := make([]string, 0, len(brokers))
		for brokerId, _ := range brokers {
			sortedBrokerIds = append(sortedBrokerIds, brokerId)
		}
		sort.Strings(sortedBrokerIds)
		for _, brokerId := range sortedBrokerIds {
			linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s %s", " ",
				color.Green(brokerId), brokers[brokerId]), linesInTopicMode)
		}
	}

	// find 1st broker in the cluster
	// each broker in the cluster has same metadata
	var broker0 *zk.Broker
	for _, broker := range brokers {
		broker0 = broker
		break
	}

	kfk, err := sarama.NewClient([]string{broker0.Addr()}, sarama.NewConfig())
	if err != nil {
		if this.verbose {
			linesInTopicMode = this.echoOrBuffer(color.Yellow("%5s%s %s", " ",
				broker0.Addr(), err.Error()), linesInTopicMode)
		}

		return
	}
	defer kfk.Close()

	topics, err := kfk.Topics()
	must(err)
	if len(topics) == 0 {
		if this.topicPrefix == "" && this.verbose {
			linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%5s%s", " ",
				color.Magenta("no topics")), linesInTopicMode)
			echoBuffer(linesInTopicMode)
		}

		return
	}

	hasTopicMatched := false
	for _, topic := range topics {
		if this.topicPrefix != "" && !strings.Contains(topic, this.topicPrefix) {
			continue
		}

		hasTopicMatched = true
		if this.verbose {
			linesInTopicMode = this.echoOrBuffer(strings.Repeat(" ", 4)+color.Blue(topic), linesInTopicMode)
		}

		// get partitions and check if some dead
		alivePartitions, err := kfk.WritablePartitions(topic)
		must(err)
		partions, err := kfk.Partitions(topic)
		must(err)
		if len(alivePartitions) != len(partions) {
			linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("topic[%s] has %s partitions: %+v/%+v",
				alivePartitions, color.Red("dead"), partions), linesInTopicMode)
		}

		replicas, err := kfk.Replicas(topic, partions[0])
		must(err)

		if !this.verbose {
			linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%30s %s %3dP %dR",
				zkcluster.Name(),
				color.Blue("%50s", topic),
				len(partions), len(replicas)), linesInTopicMode)
			continue
		}

		for _, partitionID := range alivePartitions {
			leader, err := kfk.Leader(topic, partitionID)
			must(err)

			replicas, err := kfk.Replicas(topic, partitionID)
			must(err)

			isr := zkcluster.Isr(topic, partitionID)
			//isr, err := kfk.Isr(topic, partitionID)

			underReplicated := false
			if len(isr) != len(replicas) {
				underReplicated = true
			}

			latestOffset, err := kfk.GetOffset(topic, partitionID,
				sarama.OffsetNewest)
			must(err)

			oldestOffset, err := kfk.GetOffset(topic, partitionID,
				sarama.OffsetOldest)
			must(err)

			if !underReplicated {
				linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%8d Leader:%d Replicas:%+v Isr:%+v Offset:%d Num:%d",
					partitionID, leader.ID(), replicas, isr,
					latestOffset, latestOffset-oldestOffset), linesInTopicMode)
			} else {
				// use red for alert
				linesInTopicMode = this.echoOrBuffer(color.Red("%8d Leader:%d Replicas:%+v Isr:%+v Offset:%d Num:%d",
					partitionID, leader.ID(), replicas, isr,
					latestOffset, latestOffset-oldestOffset), linesInTopicMode)
			}

		}
	}

	if this.topicPrefix != "" {
		if hasTopicMatched {
			echoBuffer(linesInTopicMode)
		}

	} else {
		echoBuffer(linesInTopicMode)
	}
}

func (this *Topics) addTopic(zkcluster *zk.ZkCluster, topic string, replicas,
	partitions int) error {
	this.Ui.Info(fmt.Sprintf("creating kafka topic: %s", topic))

	zkAddrs := zkcluster.ZkAddrs()

	cmd := pipestream.New(fmt.Sprintf("%s/bin/kafka-topics.sh", ctx.KafkaHome()),
		fmt.Sprintf("--zookeeper %s", zkAddrs),
		fmt.Sprintf("--create"),
		fmt.Sprintf("--topic %s", topic),
		fmt.Sprintf("--partitions %d", partitions),
		fmt.Sprintf("--replication-factor %d", replicas),
	)
	err := cmd.Open()
	if err != nil {
		return err
	}

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	var errmsg string
	var line string
	for scanner.Scan() {
		line = scanner.Text()

		this.Ui.Info(line)
		if strings.HasPrefix(line, "Error") {
			errmsg = line
		}
	}
	err = scanner.Err()
	if err != nil {
		return err
	}
	cmd.Close()

	if errmsg != "" {
		return fmt.Errorf("%s", errmsg)
	}

	this.Ui.Output(fmt.Sprintf("\tzookeeper.connect: %s", zkAddrs))
	this.Ui.Output(fmt.Sprintf("\t      broker list: %s",
		strings.Join(zkcluster.BrokerList(), ",")))
	return nil
}

func (*Topics) Synopsis() string {
	return "Manage topics & partitions of a zone"
}

func (this *Topics) Help() string {
	help := fmt.Sprintf(`
Usage: %s topics -z zone [options]

	Manage topics & partitions of a zone

Options:

  -c cluster

  -t topic prefix
  	Only show topics like this give topic.

  -config k=v
  	Config a topic. TODO

  -add topic
  	Add a topic to a kafka cluster.

  -partitions n
  	Default 1

  -replicas n
  	Default 2

  -verbose
`, this.Cmd)
	return strings.TrimSpace(help)
}
<commit_msg>topics subcommand output header<commit_after>package command

import (
	"bufio"
	"flag"
	"fmt"
	"sort"
	"strings"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/pipestream"
	"github.com/funkygao/sarama"
)

type Topics struct {
	Ui          cli.Ui
	Cmd         string
	topicPrefix string
	verbose     bool
}

func (this *Topics) Run(args []string) (exitCode int) {
	var (
		zone        string
		cluster     string
		addTopic    string
		replicas    int
		partitions  int
		topicConfig string
	)
	cmdFlags := flag.NewFlagSet("brokers", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", "", "")
	cmdFlags.StringVar(&this.topicPrefix, "t", "", "")
	cmdFlags.StringVar(&cluster, "c", "", "")
	cmdFlags.BoolVar(&this.verbose, "verbose", false, "")
	cmdFlags.StringVar(&addTopic, "add", "", "")
	cmdFlags.IntVar(&partitions, "partitions", 1, "")
	cmdFlags.StringVar(&topicConfig, "config", "", "")
	cmdFlags.IntVar(&replicas, "replicas", 2, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).require("-z").on("-add", "-c").invalid(args) {
		return 2
	}

	if addTopic != "" {
		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZonePath(zone)))
		zkcluster := zkzone.NewCluster(cluster)
		this.addTopic(zkcluster, addTopic, replicas, partitions)

		return
	}

	ensureZoneValid(zone)

	if !this.verbose {
		// output header
		this.Ui.Output(fmt.Sprintf("%30s %50s", "cluster", "topic"))
		this.Ui.Output(fmt.Sprintf("%30s %50s",
			strings.Repeat("-", 30), strings.Repeat("-", 50)))
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZonePath(zone)))
	if cluster != "" {
		zkcluster := zkzone.NewCluster(cluster)
		this.displayTopicsOfCluster(zkcluster)
		return
	}

	// all clusters
	zkzone.WithinClusters(func(cluster string, path string) {
		zkcluster := zkzone.NewclusterWithPath(cluster, path)
		this.displayTopicsOfCluster(zkcluster)
	})

	return
}

func (this *Topics) echoOrBuffer(line string, buffer []string) []string {
	if this.topicPrefix != "" {
		// buffer
		return append(buffer, line)
	} else {
		this.Ui.Output(line)
		return nil
	}
}

func (this *Topics) displayTopicsOfCluster(zkcluster *zk.ZkCluster) {
	must := func(err error) {
		if err != nil {
			panic(err)
		}
	}

	echoBuffer := func(lines []string) {
		for _, l := range lines {
			this.Ui.Output(l)
		}
	}

	linesInTopicMode := make([]string, 0)
	if this.verbose {
		linesInTopicMode = this.echoOrBuffer(zkcluster.Name(), linesInTopicMode)
	}

	// get all alive brokers within this cluster
	brokers := zkcluster.Brokers()
	if len(brokers) == 0 {
		linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s", " ",
			color.Red("empty brokers")), linesInTopicMode)
		echoBuffer(linesInTopicMode)
		return
	}

	if this.verbose {
		sortedBrokerIds := make([]string, 0, len(brokers))
		for brokerId, _ := range brokers {
			sortedBrokerIds = append(sortedBrokerIds, brokerId)
		}
		sort.Strings(sortedBrokerIds)
		for _, brokerId := range sortedBrokerIds {
			linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s %s", " ",
				color.Green(brokerId), brokers[brokerId]), linesInTopicMode)
		}
	}

	// find 1st broker in the cluster
	// each broker in the cluster has same metadata
	var broker0 *zk.Broker
	for _, broker := range brokers {
		broker0 = broker
		break
	}

	kfk, err := sarama.NewClient([]string{broker0.Addr()}, sarama.NewConfig())
	if err != nil {
		if this.verbose {
			linesInTopicMode = this.echoOrBuffer(color.Yellow("%5s%s %s", " ",
				broker0.Addr(), err.Error()), linesInTopicMode)
		}

		return
	}
	defer kfk.Close()

	topics, err := kfk.Topics()
	must(err)
	if len(topics) == 0 {
		if this.topicPrefix == "" && this.verbose {
			linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%5s%s", " ",
				color.Magenta("no topics")), linesInTopicMode)
			echoBuffer(linesInTopicMode)
		}

		return
	}

	hasTopicMatched := false
	for _, topic := range topics {
		if this.topicPrefix != "" && !strings.Contains(topic, this.topicPrefix) {
			continue
		}

		hasTopicMatched = true
		if this.verbose {
			linesInTopicMode = this.echoOrBuffer(strings.Repeat(" ", 4)+color.Blue(topic), linesInTopicMode)
		}

		// get partitions and check if some dead
		alivePartitions, err := kfk.WritablePartitions(topic)
		must(err)
		partions, err := kfk.Partitions(topic)
		must(err)
		if len(alivePartitions) != len(partions) {
			linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("topic[%s] has %s partitions: %+v/%+v",
				alivePartitions, color.Red("dead"), partions), linesInTopicMode)
		}

		replicas, err := kfk.Replicas(topic, partions[0])
		must(err)

		if !this.verbose {
			linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%30s %s %3dP %dR",
				zkcluster.Name(),
				color.Blue("%50s", topic),
				len(partions), len(replicas)), linesInTopicMode)
			continue
		}

		for _, partitionID := range alivePartitions {
			leader, err := kfk.Leader(topic, partitionID)
			must(err)

			replicas, err := kfk.Replicas(topic, partitionID)
			must(err)

			isr := zkcluster.Isr(topic, partitionID)
			//isr, err := kfk.Isr(topic, partitionID)

			underReplicated := false
			if len(isr) != len(replicas) {
				underReplicated = true
			}

			latestOffset, err := kfk.GetOffset(topic, partitionID,
				sarama.OffsetNewest)
			must(err)

			oldestOffset, err := kfk.GetOffset(topic, partitionID,
				sarama.OffsetOldest)
			must(err)

			if !underReplicated {
				linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%8d Leader:%d Replicas:%+v Isr:%+v Offset:%d Num:%d",
					partitionID, leader.ID(), replicas, isr,
					latestOffset, latestOffset-oldestOffset), linesInTopicMode)
			} else {
				// use red for alert
				linesInTopicMode = this.echoOrBuffer(color.Red("%8d Leader:%d Replicas:%+v Isr:%+v Offset:%d Num:%d",
					partitionID, leader.ID(), replicas, isr,
					latestOffset, latestOffset-oldestOffset), linesInTopicMode)
			}

		}
	}

	if this.topicPrefix != "" {
		if hasTopicMatched {
			echoBuffer(linesInTopicMode)
		}

	} else {
		echoBuffer(linesInTopicMode)
	}
}

func (this *Topics) addTopic(zkcluster *zk.ZkCluster, topic string, replicas,
	partitions int) error {
	this.Ui.Info(fmt.Sprintf("creating kafka topic: %s", topic))

	zkAddrs := zkcluster.ZkAddrs()

	cmd := pipestream.New(fmt.Sprintf("%s/bin/kafka-topics.sh", ctx.KafkaHome()),
		fmt.Sprintf("--zookeeper %s", zkAddrs),
		fmt.Sprintf("--create"),
		fmt.Sprintf("--topic %s", topic),
		fmt.Sprintf("--partitions %d", partitions),
		fmt.Sprintf("--replication-factor %d", replicas),
	)
	err := cmd.Open()
	if err != nil {
		return err
	}

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	var errmsg string
	var line string
	for scanner.Scan() {
		line = scanner.Text()

		this.Ui.Info(line)
		if strings.HasPrefix(line, "Error") {
			errmsg = line
		}
	}
	err = scanner.Err()
	if err != nil {
		return err
	}
	cmd.Close()

	if errmsg != "" {
		return fmt.Errorf("%s", errmsg)
	}

	this.Ui.Output(fmt.Sprintf("\tzookeeper.connect: %s", zkAddrs))
	this.Ui.Output(fmt.Sprintf("\t      broker list: %s",
		strings.Join(zkcluster.BrokerList(), ",")))
	return nil
}

func (*Topics) Synopsis() string {
	return "Manage topics & partitions of a zone"
}

func (this *Topics) Help() string {
	help := fmt.Sprintf(`
Usage: %s topics -z zone [options]

	Manage topics & partitions of a zone

Options:

  -c cluster

  -t topic prefix
  	Only show topics like this give topic.

  -config k=v
  	Config a topic. TODO

  -add topic
  	Add a topic to a kafka cluster.

  -partitions n
  	Default 1

  -replicas n
  	Default 2

  -verbose
`, this.Cmd)
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>// Copyright 2015 The appc Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"github.com/spf13/cobra"
)

var (
	cmdCopy = &cobra.Command{
		Use:     "copy PATH_ON_HOST PATH_IN_ACI",
		Short:   "Copy a file or directory into the image",
		Example: "acbuild copy nginx.conf /etc/nginx/nginx.conf",
		Run:     runWrapper(runCopy),
	}
)

func init() {
	cmdAcbuild.AddCommand(cmdCopy)
}

func runCopy(cmd *cobra.Command, args []string) (exit int) {
	if len(args) == 0 {
		cmd.Usage()
		return 1
	}
	if len(args) != 2 {
		stderr("copy: incorrect number of arguments")
		return 1
	}

	if debug {
		stderr("Copying host:%s to aci:%s", args[0], args[1])
	}

	a, err := newACBuild()
	if err != nil {
		stderr("%v", err)
		return 1
	}
	err = a.CopyToTarget(args[0], args[1])

	if err != nil {
		stderr("copy: %v", err)
		return getErrorCode(err)
	}

	return 0
}
<commit_msg>Error on trailing path separator in copy command.<commit_after>// Copyright 2015 The appc Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"github.com/spf13/cobra"
	"os"
)

var (
	cmdCopy = &cobra.Command{
		Use:     "copy PATH_ON_HOST PATH_IN_ACI",
		Short:   "Copy a file or directory into the image",
		Example: "acbuild copy nginx.conf /etc/nginx/nginx.conf",
		Run:     runWrapper(runCopy),
	}
)

func init() {
	cmdAcbuild.AddCommand(cmdCopy)
}

func runCopy(cmd *cobra.Command, args []string) (exit int) {
	if len(args) == 0 {
		cmd.Usage()
		return 1
	}
	if len(args) != 2 {
		stderr("copy: incorrect number of arguments")
		return 1
	}

	if debug {
		stderr("Copying host:%s to aci:%s", args[0], args[1])
	}

	if len(args[1]) > 0 && args[1][len(args[1])-1] == os.PathSeparator {
		stderr(`There is a trailing path separator in the second
			operand. Please review the differences between
			this command and copy-to-dir. `)
		return 1
	}

	a, err := newACBuild()
	if err != nil {
		stderr("%v", err)
		return 1
	}
	err = a.CopyToTarget(args[0], args[1])

	if err != nil {
		stderr("copy: %v", err)
		return getErrorCode(err)
	}

	return 0
}
<|endoftext|>