<commit_before>package mux

import (
	"fmt"
	"gondola/cache"
	"gondola/errors"
	"math"
	"net/http"
	"reflect"
	"runtime"
	"strconv"
	"strings"
)

type ContextFinalizer func(*Context)

type Context struct {
	http.ResponseWriter
	R             *http.Request
	submatches    []string
	params        map[string]string
	c             *cache.Cache
	cached        bool
	fromCache     bool
	handlerName   string
	mux           *Mux
	statusCode    int
	customContext interface{}
	Data          interface{} /* Left to the user */
}

// Count returns the number of elements captured
// by the pattern which matched the handler.
func (c *Context) Count() int {
	return len(c.submatches)
}

// IndexValue returns the captured parameter
// at the given index or an empty string if no
// such parameter exists.
func (c *Context) IndexValue(idx int) string {
	if idx < len(c.submatches) {
		return c.submatches[idx]
	}
	return ""
}

// ParseIndexValue uses the captured parameter
// at the given index and tries to parse it
// into the given argument. See ParseFormValue
// for examples as well as the supported types.
func (c *Context) ParseIndexValue(idx int, arg interface{}) bool {
	val := c.IndexValue(idx)
	return c.parseTypedValue(val, arg)
}

// ParamValue returns the named captured parameter
// with the given name or an empty string if it
// does not exist.
func (c *Context) ParamValue(name string) string {
	return c.params[name]
}

// ParseParamValue uses the named captured parameter
// with the given name and tries to parse it into
// the given argument. See ParseFormValue
// for examples as well as the supported types.
func (c *Context) ParseParamValue(name string, arg interface{}) bool {
	val := c.ParamValue(name)
	return c.parseTypedValue(val, arg)
}

// FormValue returns the result of performing
// FormValue on the incoming request and trims
// any whitespaces on both sides. See the
// documentation for net/http for more details.
func (c *Context) FormValue(name string) string {
	return strings.TrimSpace(c.R.FormValue(name))
}

// RequireFormValue works like FormValue, but raises
// a MissingParameter error if the value is not present
// or empty.
func (c *Context) RequireFormValue(name string) string {
	val := c.FormValue(name)
	if val == "" {
		errors.MissingParameter(name)
	}
	return val
}

// ParseFormValue tries to parse the named form value into the given
// arg e.g.
// var f float32
// ctx.ParseFormValue("quality", &f)
// var width uint
// ctx.ParseFormValue("width", &width)
// Supported types are: bool, u?int(8|16|32|64)? and float(32|64)
func (c *Context) ParseFormValue(name string, arg interface{}) bool {
	val := c.FormValue(name)
	return c.parseTypedValue(val, arg)
}

// RequireParseFormValue works like ParseFormValue but raises a
// MissingParameterError if the parameter is missing or an
// InvalidParameterTypeError if the parameter does not have the
// required type
func (c *Context) RequireParseFormValue(name string, arg interface{}) {
	val := c.RequireFormValue(name)
	if !c.parseTypedValue(val, arg) {
		t := reflect.TypeOf(arg)
		for t.Kind() == reflect.Ptr {
			t = t.Elem()
		}
		errors.InvalidParameterType(name, t.String())
	}
}

// StatusCode returns the response status code. If the headers
// haven't been written yet, it returns 0
func (c *Context) StatusCode() int {
	return c.statusCode
}

func (c *Context) funcName(depth int) string {
	funcName := "???"
	caller, _, _, ok := runtime.Caller(depth + 1)
	if ok {
		f := runtime.FuncForPC(caller)
		if f != nil {
			fullName := strings.Trim(f.Name(), ".")
			parts := strings.Split(fullName, ".")
			simpleName := parts[len(parts)-1]
			funcName = fmt.Sprintf("%s()", simpleName)
		}
	}
	return funcName
}

func (c *Context) parseTypedValue(val string, arg interface{}) bool {
	v := reflect.ValueOf(arg)
	for v.Type().Kind() == reflect.Ptr {
		v = v.Elem()
	}
	if !v.CanSet() {
		panic(fmt.Errorf("Invalid argument type passed to %s. Please pass %s instead of %s.",
			c.funcName(1), reflect.PtrTo(v.Type()), v.Type()))
	}
	switch v.Type().Kind() {
	case reflect.Bool:
		res := false
		if val != "" && val != "0" && strings.ToLower(val) != "false" {
			res = true
		}
		v.SetBool(res)
		return true
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		res, err := strconv.ParseInt(val, 0, 64)
		if err == nil {
			if v.OverflowInt(res) {
				if res > 0 {
					res = int64(math.Pow(2, float64(8*v.Type().Size()-1)) - 1)
				} else {
					res = -int64(math.Pow(2, float64(8*v.Type().Size()-1)))
				}
			}
			v.SetInt(res)
			return true
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		res, err := strconv.ParseUint(val, 0, 64)
		if err == nil {
			if v.OverflowUint(res) {
				res = uint64(math.Pow(2, float64(8*v.Type().Size())) - 1)
			}
			v.SetUint(res)
			return true
		}
	case reflect.Float32, reflect.Float64:
		res, err := strconv.ParseFloat(val, 64)
		if err == nil {
			v.SetFloat(res)
			return true
		}
	case reflect.String:
		v.SetString(val)
		return true
	default:
		panic(fmt.Errorf("Invalid arguent type passed to %s: %s. Please, see ParseFormValue() docs for a list of the supported types.",
			c.funcName(1), v.Type()))
	}
	return false
}

// Cache returns the default cache
// See gondola/cache for a further
// explanation
func (c *Context) Cache() *cache.Cache {
	if c.c == nil {
		c.c = cache.NewDefault()
	}
	return c.c
}

// Redirect sends an HTTP redirect to the client,
// using the provided redirect, which may be either
// absolute or relative. The permanent argument
// indicates if the redirect should be sent as a
// permanent or a temporary one.
func (c *Context) Redirect(redir string, permanent bool) {
	code := http.StatusFound
	if permanent {
		code = http.StatusMovedPermanently
	}
	http.Redirect(c, c.R, redir, code)
}

// Error replies to the request with the specified
// message and HTTP code.
func (c *Context) Error(error string, code int) {
	http.Error(c, error, code)
}

// SetCached is used internaly by cache layers.
// Don't call this method
func (c *Context) SetCached(b bool) {
	c.cached = b
}

// SetServedFromCache is used internally by cache layers.
// Don't call this method
func (c *Context) SetServedFromCache(b bool) {
	c.fromCache = b
}

// Cached() returns true if the request was
// cached by a cache layer
// (see gondola/cache/layer)
func (c *Context) Cached() bool {
	return c.cached
}

// ServedFromCache returns true if the request
// was served by a cache layer
// (see gondola/cache/layer)
func (c *Context) ServedFromCache() bool {
	return c.fromCache
}

// HandlerName returns the name of the handler which
// handled this context
func (c *Context) HandlerName() string {
	return c.handlerName
}

// Mux returns the Mux this Context originated from
func (c *Context) Mux() *Mux {
	return c.mux
}

// MustReverse calls MustReverse on the mux this context originated
// from. See the documentation on Mux for details.
func (c *Context) MustReverse(name string, args ...interface{}) string {
	return c.mux.MustReverse(name, args...)
}

// Reverse calls Reverse on the mux this context originated
// from. See the documentation on Mux for details.
func (c *Context) Reverse(name string, args ...interface{}) (string, error) {
	return c.mux.Reverse(name, args...)
}

// RedirectReverse calls Reverse to find the URL and then sends
// the redirect to the client. See the documentation on Mux.Reverse
// for further details.
func (c *Context) RedirectReverse(permanent bool, name string, args ...interface{}) error {
	rev, err := c.Reverse(name, args...)
	if err != nil {
		return err
	}
	c.Redirect(rev, permanent)
	return nil
}

// C returns the custom type context wrapped in
// an interface{}. Intended for use in templates
// e.g. {{ Ctx.C.MyCustomMethod }}
//
// For use in code it's better to use the function
// you previously defined as the context transformation for
// the mux e.g.
// function in your own code to avoid type assertions
// func Context(ctx *mux.Context) *mycontext {
//	return (*mycontext)(ctx)
// }
// ...
// mymux.SetContextTransform(Context)
func (c *Context) C() interface{} {
	if c.customContext == nil {
		if c.mux.contextTransform != nil {
			result := c.mux.contextTransform.Call([]reflect.Value{reflect.ValueOf(c)})
			c.customContext = result[0].Interface()
		} else {
			c.customContext = c
		}
	}
	return c.customContext
}

// Close closes any resources opened by the context
// (for now, the cache connection). It's automatically
// called by the mux, so you don't need to call it
// manually
func (c *Context) Close() {
	if c.c != nil {
		c.c.Close()
		c.c = nil
	}
}

// Intercept http.ResponseWriter calls to find response
// status code

func (c *Context) WriteHeader(code int) {
	c.ResponseWriter.WriteHeader(code)
}

func (c *Context) Write(data []byte) (int, error) {
	n, err := c.ResponseWriter.Write(data)
	if c.statusCode == 0 {
		c.statusCode = http.StatusOK
	}
	return n, err
}
<commit_msg>Add a few more conveniency methods<commit_after>package mux

import (
	"fmt"
	"gondola/cache"
	"gondola/errors"
	"math"
	"net/http"
	"reflect"
	"runtime"
	"strconv"
	"strings"
)

type ContextFinalizer func(*Context)

type Context struct {
	http.ResponseWriter
	R             *http.Request
	submatches    []string
	params        map[string]string
	c             *cache.Cache
	cached        bool
	fromCache     bool
	handlerName   string
	mux           *Mux
	statusCode    int
	customContext interface{}
	Data          interface{} /* Left to the user */
}

// Count returns the number of elements captured
// by the pattern which matched the handler.
func (c *Context) Count() int {
	return len(c.submatches)
}

// IndexValue returns the captured parameter
// at the given index or an empty string if no
// such parameter exists.
func (c *Context) IndexValue(idx int) string {
	if idx < len(c.submatches) {
		return c.submatches[idx]
	}
	return ""
}

// ParseIndexValue uses the captured parameter
// at the given index and tries to parse it
// into the given argument. See ParseFormValue
// for examples as well as the supported types.
func (c *Context) ParseIndexValue(idx int, arg interface{}) bool {
	val := c.IndexValue(idx)
	return c.parseTypedValue(val, arg)
}

// ParamValue returns the named captured parameter
// with the given name or an empty string if it
// does not exist.
func (c *Context) ParamValue(name string) string {
	return c.params[name]
}

// ParseParamValue uses the named captured parameter
// with the given name and tries to parse it into
// the given argument. See ParseFormValue
// for examples as well as the supported types.
func (c *Context) ParseParamValue(name string, arg interface{}) bool {
	val := c.ParamValue(name)
	return c.parseTypedValue(val, arg)
}

// FormValue returns the result of performing
// FormValue on the incoming request and trims
// any whitespaces on both sides. See the
// documentation for net/http for more details.
func (c *Context) FormValue(name string) string {
	return strings.TrimSpace(c.R.FormValue(name))
}

// RequireFormValue works like FormValue, but raises
// a MissingParameter error if the value is not present
// or empty.
func (c *Context) RequireFormValue(name string) string {
	val := c.FormValue(name)
	if val == "" {
		errors.MissingParameter(name)
	}
	return val
}

// ParseFormValue tries to parse the named form value into the given
// arg e.g.
// var f float32
// ctx.ParseFormValue("quality", &f)
// var width uint
// ctx.ParseFormValue("width", &width)
// Supported types are: bool, u?int(8|16|32|64)? and float(32|64)
func (c *Context) ParseFormValue(name string, arg interface{}) bool {
	val := c.FormValue(name)
	return c.parseTypedValue(val, arg)
}

// RequireParseFormValue works like ParseFormValue but raises a
// MissingParameterError if the parameter is missing or an
// InvalidParameterTypeError if the parameter does not have the
// required type
func (c *Context) RequireParseFormValue(name string, arg interface{}) {
	val := c.RequireFormValue(name)
	if !c.parseTypedValue(val, arg) {
		t := reflect.TypeOf(arg)
		for t.Kind() == reflect.Ptr {
			t = t.Elem()
		}
		errors.InvalidParameterType(name, t.String())
	}
}

// StatusCode returns the response status code. If the headers
// haven't been written yet, it returns 0
func (c *Context) StatusCode() int {
	return c.statusCode
}

func (c *Context) funcName(depth int) string {
	funcName := "???"
	caller, _, _, ok := runtime.Caller(depth + 1)
	if ok {
		f := runtime.FuncForPC(caller)
		if f != nil {
			fullName := strings.Trim(f.Name(), ".")
			parts := strings.Split(fullName, ".")
			simpleName := parts[len(parts)-1]
			funcName = fmt.Sprintf("%s()", simpleName)
		}
	}
	return funcName
}

func (c *Context) parseTypedValue(val string, arg interface{}) bool {
	v := reflect.ValueOf(arg)
	for v.Type().Kind() == reflect.Ptr {
		v = v.Elem()
	}
	if !v.CanSet() {
		panic(fmt.Errorf("Invalid argument type passed to %s. Please pass %s instead of %s.",
			c.funcName(1), reflect.PtrTo(v.Type()), v.Type()))
	}
	switch v.Type().Kind() {
	case reflect.Bool:
		res := false
		if val != "" && val != "0" && strings.ToLower(val) != "false" {
			res = true
		}
		v.SetBool(res)
		return true
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		res, err := strconv.ParseInt(val, 0, 64)
		if err == nil {
			if v.OverflowInt(res) {
				if res > 0 {
					res = int64(math.Pow(2, float64(8*v.Type().Size()-1)) - 1)
				} else {
					res = -int64(math.Pow(2, float64(8*v.Type().Size()-1)))
				}
			}
			v.SetInt(res)
			return true
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		res, err := strconv.ParseUint(val, 0, 64)
		if err == nil {
			if v.OverflowUint(res) {
				res = uint64(math.Pow(2, float64(8*v.Type().Size())) - 1)
			}
			v.SetUint(res)
			return true
		}
	case reflect.Float32, reflect.Float64:
		res, err := strconv.ParseFloat(val, 64)
		if err == nil {
			v.SetFloat(res)
			return true
		}
	case reflect.String:
		v.SetString(val)
		return true
	default:
		panic(fmt.Errorf("Invalid arguent type passed to %s: %s. Please, see ParseFormValue() docs for a list of the supported types.",
			c.funcName(1), v.Type()))
	}
	return false
}

// Cache returns the default cache
// See gondola/cache for a further
// explanation
func (c *Context) Cache() *cache.Cache {
	if c.c == nil {
		c.c = cache.NewDefault()
	}
	return c.c
}

// Redirect sends an HTTP redirect to the client,
// using the provided redirect, which may be either
// absolute or relative. The permanent argument
// indicates if the redirect should be sent as a
// permanent or a temporary one.
func (c *Context) Redirect(redir string, permanent bool) {
	code := http.StatusFound
	if permanent {
		code = http.StatusMovedPermanently
	}
	http.Redirect(c, c.R, redir, code)
}

// Error replies to the request with the specified
// message and HTTP code. If an error handler
// has been defined for the mux, it will be
// given the opportunity to intercept the
// error and provide its own response.
func (c *Context) Error(error string, code int) {
	http.Error(c, error, code)
}

// NotFound is equivalent to calling Error()
// with http.StatusNotFound.
func (c *Context) NotFound(error string) {
	c.Error(error, http.StatusNotFound)
}

// Forbidden is equivalent to calling Error()
// with http.StatusForbidden.
func (c *Context) Forbidden(error string) {
	c.Error(error, http.StatusForbidden)
}

// BadRequest is equivalent to calling Error()
// with http.StatusBadRequest.
func (c *Context) BadRequest(error string) {
	c.Error(error, http.StatusBadRequest)
}

// SetCached is used internaly by cache layers.
// Don't call this method
func (c *Context) SetCached(b bool) {
	c.cached = b
}

// SetServedFromCache is used internally by cache layers.
// Don't call this method
func (c *Context) SetServedFromCache(b bool) {
	c.fromCache = b
}

// Cached() returns true if the request was
// cached by a cache layer
// (see gondola/cache/layer)
func (c *Context) Cached() bool {
	return c.cached
}

// ServedFromCache returns true if the request
// was served by a cache layer
// (see gondola/cache/layer)
func (c *Context) ServedFromCache() bool {
	return c.fromCache
}

// HandlerName returns the name of the handler which
// handled this context
func (c *Context) HandlerName() string {
	return c.handlerName
}

// Mux returns the Mux this Context originated from
func (c *Context) Mux() *Mux {
	return c.mux
}

// MustReverse calls MustReverse on the mux this context originated
// from. See the documentation on Mux for details.
func (c *Context) MustReverse(name string, args ...interface{}) string {
	return c.mux.MustReverse(name, args...)
}

// Reverse calls Reverse on the mux this context originated
// from. See the documentation on Mux for details.
func (c *Context) Reverse(name string, args ...interface{}) (string, error) {
	return c.mux.Reverse(name, args...)
}

// RedirectReverse calls Reverse to find the URL and then sends
// the redirect to the client. See the documentation on Mux.Reverse
// for further details.
func (c *Context) RedirectReverse(permanent bool, name string, args ...interface{}) error {
	rev, err := c.Reverse(name, args...)
	if err != nil {
		return err
	}
	c.Redirect(rev, permanent)
	return nil
}

// C returns the custom type context wrapped in
// an interface{}. Intended for use in templates
// e.g. {{ Ctx.C.MyCustomMethod }}
//
// For use in code it's better to use the function
// you previously defined as the context transformation for
// the mux e.g.
// function in your own code to avoid type assertions
// func Context(ctx *mux.Context) *mycontext {
//	return (*mycontext)(ctx)
// }
// ...
// mymux.SetContextTransform(Context)
func (c *Context) C() interface{} {
	if c.customContext == nil {
		if c.mux.contextTransform != nil {
			result := c.mux.contextTransform.Call([]reflect.Value{reflect.ValueOf(c)})
			c.customContext = result[0].Interface()
		} else {
			c.customContext = c
		}
	}
	return c.customContext
}

// Close closes any resources opened by the context
// (for now, the cache connection). It's automatically
// called by the mux, so you don't need to call it
// manually
func (c *Context) Close() {
	if c.c != nil {
		c.c.Close()
		c.c = nil
	}
}

// Intercept http.ResponseWriter calls to find response
// status code

func (c *Context) WriteHeader(code int) {
	c.ResponseWriter.WriteHeader(code)
}

func (c *Context) Write(data []byte) (int, error) {
	n, err := c.ResponseWriter.Write(data)
	if c.statusCode == 0 {
		c.statusCode = http.StatusOK
	}
	return n, err
}
<|endoftext|>


<commit_before>package meter2

import (
	"log"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"

	tc "github.com/alxarch/go-timecodec"
	"github.com/go-redis/redis"
)

const DefaultKeyPrefix = "meter:"

type DB struct {
	Redis     redis.UniversalClient
	Registry  *Registry
	KeyPrefix string
}

func NewDB(r redis.UniversalClient) *DB {
	db := new(DB)
	db.Redis = r
	db.Registry = defaultRegistry
	db.KeyPrefix = DefaultKeyPrefix
	return db
}

func (e *Desc) MatchingQueries(q url.Values) url.Values {
	if e == nil || q == nil {
		return nil
	}
	m := make(map[string][]string, len(q))
	for key, values := range q {
		if e.HasLabel(key) {
			m[key] = values
		}
	}
	return m
}

func TimeSequence(start time.Time, end time.Time, unit time.Duration) []time.Time {
	if unit == 0 {
		return []time.Time{}
	}
	start = tc.Round(start, unit).In(start.Location())
	end = tc.Round(end, unit).In(end.Location())
	n := end.Sub(start) / unit

	results := make([]time.Time, 0, n)

	for s := start; end.Sub(s) >= 0; s = s.Add(unit) {
		results = append(results, s)
	}
	return results
}

func QueryPermutations(input url.Values) []map[string]string {
	vcount := []int{}
	keys := []string{}
	combinations := [][]int{}
	for k, v := range input {
		if c := len(v); c > 0 {
			keys = append(keys, k)
			vcount = append(vcount, c)
		}
	}
	var generate func([]int)
	generate = func(comb []int) {
		if i := len(comb); i == len(vcount) {
			combinations = append(combinations, comb)
			return
		} else {
			for j := 0; j < vcount[i]; j++ {
				next := make([]int, i+1)
				if i > 0 {
					copy(next[:i], comb)
				}
				next[i] = j
				generate(next)
			}
		}
	}
	generate([]int{})
	results := make([]map[string]string, 0, len(combinations))
	for _, comb := range combinations {
		result := make(map[string]string, len(comb))
		for i, j := range comb {
			key := keys[i]
			result[key] = input[key][j]
		}
		if len(result) > 0 {
			results = append(results, result)
		}
	}
	return results
}

const LabelSeparator = '\x1f'
const FieldTerminator = '\x1e'

func (db DB) Key(r Resolution, event string, t time.Time) (k string) {
	return string(db.AppendKey(nil, r, event, t))
}

func (db DB) AppendKey(data []byte, r Resolution, event string, t time.Time) []byte {
	if db.KeyPrefix != "" {
		data = append(data, db.KeyPrefix...)
		data = append(data, LabelSeparator)
	}
	data = append(data, r.Name()...)
	data = append(data, LabelSeparator)
	data = append(data, r.MarshalTime(t)...)
	data = append(data, LabelSeparator)
	data = append(data, event...)
	return data
}

const NilByte byte = 0

func AppendField(data []byte, labels, values []string) []byte {
	n := len(values)
	for i := 0; i < len(labels); i++ {
		label := labels[i]
		if i > 0 {
			data = append(data, LabelSeparator)
		}
		data = append(data, label...)
		data = append(data, LabelSeparator)
		if i < n {
			value := values[i]
			data = append(data, value...)
		} else {
			data = append(data, NilByte)
		}
	}
	data = append(data, FieldTerminator)
	return data
}

// TODO: Write metrics in parallel with fan out

func (db *DB) Gather(col Collector) error {
	ch := make(chan Metric)
	result := make(chan error)
	go func() {
		pipeline := db.Redis.Pipeline()
		defer pipeline.Close()
		data := []byte{}
		tm := time.Now()
		pipelineSize := 0
		for m := range ch {
			if m == nil {
				continue
			}
			n := m.Set(0)
			if n == 0 {
				// log.Println("Empty counter")
				continue
			}
			values := m.Values()
			desc := m.Describe()
			name := desc.Name()
			labels := desc.Labels()
			for _, res := range desc.Resolutions() {
				data = db.AppendKey(data[:0], res, name, tm)
				key := string(data)
				data = AppendField(data[:0], labels, values)
				// log.Println(res.Name(), key, string(data))
				pipeline.HIncrBy(key, string(data), n)
				if ttl := res.TTL(); ttl > 0 {
					pipeline.Expire(key, ttl)
				}
				pipelineSize++
			}
		}
		if pipelineSize == 0 {
			result <- nil
			return
		}
		if _, err := pipeline.Exec(); err != nil && err != redis.Nil {
			log.Println(err)
			result <- err
			return
		}
		result <- nil
	}()
	col.Collect(ch)
	close(ch)
	return <-result
}

type ScanResult struct {
	Name   string
	Time   time.Time
	Group  string
	Values LabelValues
	err    error
	count  int64
}

func AppendMatch(data []byte, s string) []byte {
	for i := 0; i < len(s); i++ {
		switch b := s[i]; b {
		case '*', '[', ']', '?', '^':
			data = append(data, '\\', b)
		default:
			data = append(data, b)
		}
	}
	return data
}
func AppendMatchField(data []byte, labels []string, group string, q map[string]string) []byte {
	for i := 0; i < len(labels); i++ {
		if i > 0 {
			data = append(data, LabelSeparator)
		}
		label := labels[i]
		data = AppendMatch(data, label)
		data = append(data, LabelSeparator)
		if label == group {
			data = append(data, '[', '^', NilByte, ']', '*')
			continue
		}
		if q != nil {
			if v, ok := q[label]; ok {
				data = AppendMatch(data, v)
				continue
			}
		}
		data = append(data, '*')

	}
	data = append(data, FieldTerminator)
	return data
}

type ScanQuery struct {
	Event      string
	Start, End time.Time
	Group      string
	Query      url.Values
	Resolution Resolution
}

func (q ScanQuery) QueryValues(d *Desc) []map[string]string {
	if d == nil {
		return nil
	}
	if q.Group != "" {
		if !d.HasLabel(q.Group) {
			return nil
		}
		delete(q.Query, q.Group)
	}

	queries := d.MatchingQueries(q.Query)
	return QueryPermutations(queries)
}

func (db *DB) Query(q ScanQuery) (Results, error) {
	results := Results{}
	ch := make(chan ScanResult, 1)
	go func() {
		for r := range ch {
			results = r.AppendResults(results)
		}
	}()
	err := db.ScanQuery(q, ch)
	close(ch)
	return results, err
}

func (db *DB) ScanQuery(q ScanQuery, results chan<- ScanResult) (err error) {
	event := db.Registry.Get(q.Event)
	if event == nil {
		return ErrUnregisteredEvent
	}
	desc := event.Describe()
	if desc == nil {
		return ErrNilDesc
	}
	if q.Group != "" && !desc.HasLabel(q.Group) {
		return ErrInvalidEventLabel
	}
	queries := q.QueryValues(desc)
	if queries == nil {
		return nil
	}
	if len(queries) == 0 {
		queries = append(queries, map[string]string{})
	}
	ts := q.Resolution.TimeSequence(q.Start, q.End)
	if len(ts) == 0 {
		return nil
	}
	wg := &sync.WaitGroup{}
	result := ScanResult{
		Name: q.Event,
	}
	for _, values := range queries {
		result.Values = values
		m := event.WithLabels(values)
		if m == nil {
			continue
		}
		desc = m.Describe()
		if desc == nil {
			continue
		}

		if e := desc.Error(); e != nil {
			result.err = e
			results <- result
			continue
		}

		result.Name = desc.Name()
		data := AppendMatchField(nil, desc.Labels(), q.Group, values)
		match := string(data)
		// Let redis client pool size determine parallel requests
		for _, tm := range ts {
			result.Time = tm
			data = db.AppendKey(data[:0], q.Resolution, desc.Name(), tm)
			key := string(data)
			wg.Add(1)
			go func(r ScanResult, key string) {
				db.Scan(key, match, q.Group, r, results)
				wg.Done()
			}(result, key)
		}
	}
	wg.Wait()

	return
}

func scanField(val []byte, field, group string) []byte {
	i := strings.Index(field, group)
	if i == -1 {
		return val
	}
	i += len(group)
	for j := i; j < len(field); j++ {
		if field[j] == LabelSeparator || field[j] == FieldTerminator {
			return append(val, field[i:j]...)
		}
	}
	return val
}

const sLabelSeparator = "\x1f"

func (db *DB) Scan(key, match, group string, r ScanResult, results chan<- ScanResult) (err error) {
	var fields []string
	scan := db.Redis.HScan(key, 0, match, -1).Iterator()
	for scan.Next() {
		fields = append(fields, scan.Val())
	}
	if err = scan.Err(); err != nil {
		return
	}
	if len(fields) == 0 {
		return
	}
	var reply []interface{}
	reply, err = db.Redis.HMGet(key, fields...).Result()
	if err != nil {
		if err == redis.Nil {
			err = nil
		}
		return
	}
	var grouped map[string]int64
	var val []byte
	if group != "" {
		group = r.Group + sLabelSeparator
		val = []byte(group)
		grouped = make(map[string]int64, len(fields))
	}
	// Just in case
	r.count = 0
	for i, x := range reply {
		if a, ok := x.(string); ok {
			if n, e := strconv.ParseInt(a, 10, 64); e == nil {
				if group != "" {
					val = scanField(val[:0], fields[i], group)
					grouped[string(val)] += n
				} else {
					r.count += n
				}
			}
		}
	}
	if group != "" {
		for g, count := range grouped {
			r.Group = g
			r.count = count
			results <- r
		}
		return
	}
	results <- r

	return

}

func (r ScanResult) AppendResults(results Results) Results {
	p := DataPoint{r.Time.Unix(), r.count}
	if results != nil {
		if i := results.Find(r.Name, r.Group, r.Values); i != -1 {
			results[i].Data = append(results[i].Data, p)
			return results
		}
	}
	return append(results, Result{
		Event:  r.Name,
		Labels: r.Values,
		Group:  r.Group,
		Data:   DataPoints{p},
	})

}
<commit_msg>Reuse field for all resolutions<commit_after>package meter2

import (
	"log"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"

	tc "github.com/alxarch/go-timecodec"
	"github.com/go-redis/redis"
)

const DefaultKeyPrefix = "meter:"

type DB struct {
	Redis     redis.UniversalClient
	Registry  *Registry
	KeyPrefix string
}

func NewDB(r redis.UniversalClient) *DB {
	db := new(DB)
	db.Redis = r
	db.Registry = defaultRegistry
	db.KeyPrefix = DefaultKeyPrefix
	return db
}

func (e *Desc) MatchingQueries(q url.Values) url.Values {
	if e == nil || q == nil {
		return nil
	}
	m := make(map[string][]string, len(q))
	for key, values := range q {
		if e.HasLabel(key) {
			m[key] = values
		}
	}
	return m
}

func TimeSequence(start time.Time, end time.Time, unit time.Duration) []time.Time {
	if unit == 0 {
		return []time.Time{}
	}
	start = tc.Round(start, unit).In(start.Location())
	end = tc.Round(end, unit).In(end.Location())
	n := end.Sub(start) / unit

	results := make([]time.Time, 0, n)

	for s := start; end.Sub(s) >= 0; s = s.Add(unit) {
		results = append(results, s)
	}
	return results
}

func QueryPermutations(input url.Values) []map[string]string {
	vcount := []int{}
	keys := []string{}
	combinations := [][]int{}
	for k, v := range input {
		if c := len(v); c > 0 {
			keys = append(keys, k)
			vcount = append(vcount, c)
		}
	}
	var generate func([]int)
	generate = func(comb []int) {
		if i := len(comb); i == len(vcount) {
			combinations = append(combinations, comb)
			return
		} else {
			for j := 0; j < vcount[i]; j++ {
				next := make([]int, i+1)
				if i > 0 {
					copy(next[:i], comb)
				}
				next[i] = j
				generate(next)
			}
		}
	}
	generate([]int{})
	results := make([]map[string]string, 0, len(combinations))
	for _, comb := range combinations {
		result := make(map[string]string, len(comb))
		for i, j := range comb {
			key := keys[i]
			result[key] = input[key][j]
		}
		if len(result) > 0 {
			results = append(results, result)
		}
	}
	return results
}

const LabelSeparator = '\x1f'
const FieldTerminator = '\x1e'

func (db DB) Key(r Resolution, event string, t time.Time) (k string) {
	return string(db.AppendKey(nil, r, event, t))
}

func (db DB) AppendKey(data []byte, r Resolution, event string, t time.Time) []byte {
	if db.KeyPrefix != "" {
		data = append(data, db.KeyPrefix...)
		data = append(data, LabelSeparator)
	}
	data = append(data, r.Name()...)
	data = append(data, LabelSeparator)
	data = append(data, r.MarshalTime(t)...)
	data = append(data, LabelSeparator)
	data = append(data, event...)
	return data
}

const NilByte byte = 0

func AppendField(data []byte, labels, values []string) []byte {
	n := len(values)
	for i := 0; i < len(labels); i++ {
		label := labels[i]
		if i > 0 {
			data = append(data, LabelSeparator)
		}
		data = append(data, label...)
		data = append(data, LabelSeparator)
		if i < n {
			value := values[i]
			data = append(data, value...)
		} else {
			data = append(data, NilByte)
		}
	}
	data = append(data, FieldTerminator)
	return data
}

// TODO: Write metrics in parallel with fan out

func (db *DB) Gather(col Collector) error {
	ch := make(chan Metric)
	result := make(chan error)
	go func() {
		pipeline := db.Redis.Pipeline()
		defer pipeline.Close()
		data := []byte{}
		tm := time.Now()
		pipelineSize := 0
		for m := range ch {
			if m == nil {
				continue
			}
			n := m.Set(0)
			if n == 0 {
				continue
			}
			values := m.Values()
			desc := m.Describe()
			name := desc.Name()
			labels := desc.Labels()
			data = AppendField(data[:0], labels, values)
			field := string(data)
			for _, res := range desc.Resolutions() {
				data = db.AppendKey(data[:0], res, name, tm)
				key := string(data)
				pipeline.HIncrBy(key, field, n)
				if ttl := res.TTL(); ttl > 0 {
					pipeline.Expire(key, ttl)
				}
				pipelineSize++
			}
		}
		if pipelineSize == 0 {
			result <- nil
			return
		}
		if _, err := pipeline.Exec(); err != nil && err != redis.Nil {
			log.Println(err)
			result <- err
			return
		}
		result <- nil
	}()
	col.Collect(ch)
	close(ch)
	return <-result
}

type ScanResult struct {
	Name   string
	Time   time.Time
	Group  string
	Values LabelValues
	err    error
	count  int64
}

func AppendMatch(data []byte, s string) []byte {
	for i := 0; i < len(s); i++ {
		switch b := s[i]; b {
		case '*', '[', ']', '?', '^':
			data = append(data, '\\', b)
		default:
			data = append(data, b)
		}
	}
	return data
}
func AppendMatchField(data []byte, labels []string, group string, q map[string]string) []byte {
	for i := 0; i < len(labels); i++ {
		if i > 0 {
			data = append(data, LabelSeparator)
		}
		label := labels[i]
		data = AppendMatch(data, label)
		data = append(data, LabelSeparator)
		if label == group {
			data = append(data, '[', '^', NilByte, ']', '*')
			continue
		}
		if q != nil {
			if v, ok := q[label]; ok {
				data = AppendMatch(data, v)
				continue
			}
		}
		data = append(data, '*')

	}
	data = append(data, FieldTerminator)
	return data
}

type ScanQuery struct {
	Event      string
	Start, End time.Time
	Group      string
	Query      url.Values
	Resolution Resolution
}

func (q ScanQuery) QueryValues(d *Desc) []map[string]string {
	if d == nil {
		return nil
	}
	if q.Group != "" {
		if !d.HasLabel(q.Group) {
			return nil
		}
		delete(q.Query, q.Group)
	}

	queries := d.MatchingQueries(q.Query)
	return QueryPermutations(queries)
}

func (db *DB) Query(q ScanQuery) (Results, error) {
	results := Results{}
	ch := make(chan ScanResult, 1)
	go func() {
		for r := range ch {
			results = r.AppendResults(results)
		}
	}()
	err := db.ScanQuery(q, ch)
	close(ch)
	return results, err
}

func (db *DB) ScanQuery(q ScanQuery, results chan<- ScanResult) (err error) {
	event := db.Registry.Get(q.Event)
	if event == nil {
		return ErrUnregisteredEvent
	}
	desc := event.Describe()
	if desc == nil {
		return ErrNilDesc
	}
	if q.Group != "" && !desc.HasLabel(q.Group) {
		return ErrInvalidEventLabel
	}
	queries := q.QueryValues(desc)
	if queries == nil {
		return nil
	}
	if len(queries) == 0 {
		queries = append(queries, map[string]string{})
	}
	ts := q.Resolution.TimeSequence(q.Start, q.End)
	if len(ts) == 0 {
		return nil
	}
	wg := &sync.WaitGroup{}
	result := ScanResult{
		Name: q.Event,
	}
	for _, values := range queries {
		result.Values = values
		m := event.WithLabels(values)
		if m == nil {
			continue
		}
		desc = m.Describe()
		if desc == nil {
			continue
		}

		if e := desc.Error(); e != nil {
			result.err = e
			results <- result
			continue
		}

		result.Name = desc.Name()
		data := AppendMatchField(nil, desc.Labels(), q.Group, values)
		match := string(data)
		// Let redis client pool size determine parallel requests
		for _, tm := range ts {
			result.Time = tm
			data = db.AppendKey(data[:0], q.Resolution, desc.Name(), tm)
			key := string(data)
			wg.Add(1)
			go func(r ScanResult, key string) {
				db.Scan(key, match, q.Group, r, results)
				wg.Done()
			}(result, key)
		}
	}
	wg.Wait()

	return
}

func scanField(val []byte, field, group string) []byte {
	i := strings.Index(field, group)
	if i == -1 {
		return val
	}
	i += len(group)
	for j := i; j < len(field); j++ {
		if field[j] == LabelSeparator || field[j] == FieldTerminator {
			return append(val, field[i:j]...)
		}
	}
	return val
}

const sLabelSeparator = "\x1f"

func (db *DB) Scan(key, match, group string, r ScanResult, results chan<- ScanResult) (err error) {
	var fields []string
	scan := db.Redis.HScan(key, 0, match, -1).Iterator()
	for scan.Next() {
		fields = append(fields, scan.Val())
	}
	if err = scan.Err(); err != nil {
		return
	}
	if len(fields) == 0 {
		return
	}
	var reply []interface{}
	reply, err = db.Redis.HMGet(key, fields...).Result()
	if err != nil {
		if err == redis.Nil {
			err = nil
		}
		return
	}
	var grouped map[string]int64
	var val []byte
	if group != "" {
		group = r.Group + sLabelSeparator
		val = []byte(group)
		grouped = make(map[string]int64, len(fields))
	}
	// Just in case
	r.count = 0
	for i, x := range reply {
		if a, ok := x.(string); ok {
			if n, e := strconv.ParseInt(a, 10, 64); e == nil {
				if group != "" {
					val = scanField(val[:0], fields[i], group)
					grouped[string(val)] += n
				} else {
					r.count += n
				}
			}
		}
	}
	if group != "" {
		for g, count := range grouped {
			r.Group = g
			r.count = count
			results <- r
		}
		return
	}
	results <- r

	return

}

func (r ScanResult) AppendResults(results Results) Results {
	p := DataPoint{r.Time.Unix(), r.count}
	if results != nil {
		if i := results.Find(r.Name, r.Group, r.Values); i != -1 {
			results[i].Data = append(results[i].Data, p)
			return results
		}
	}
	return append(results, Result{
		Event:  r.Name,
		Labels: r.Values,
		Group:  r.Group,
		Data:   DataPoints{p},
	})

}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testowner

import (
	"bufio"
	"bytes"
	"io/ioutil"
	"os"
	"testing"
	"time"
)

func TestNormalize(t *testing.T) {
	tests := map[string]string{
		"A":                                    "a",
		"Perf [Performance]":                   "perf",
		"[k8s.io] test [performance] stuff":    "test stuff",
		"[k8s.io] blah {Kubernetes e2e suite}": "blah",
	}
	for input, output := range tests {
		result := normalize(input)
		if result != output {
			t.Errorf("normalize(%s) != %s (got %s)", input, output, result)
		}
	}
}

func TestOwnerList(t *testing.T) {
	list := NewOwnerList(map[string]*OwnerInfo{"Perf [performance]": {
		User: "me",
		SIG:  "group",
	}})
	owner := list.TestOwner("perf [flaky]")
	if owner != "me" {
		t.Error("Unexpected return value ", owner)
	}
	sig := list.TestSIG("perf [flaky]")
	if sig != "group" {
		t.Error("Unexpected sig: ", sig)
	}
	owner = list.TestOwner("Unknown test")
	if owner != "" {
		t.Error("Unexpected return value ", owner)
	}
	sig = list.TestSIG("Unknown test")
	if sig != "" {
		t.Error("Unexpected sig: ", sig)
	}
}

func TestOwnerGlob(t *testing.T) {
	list := NewOwnerList(map[string]*OwnerInfo{"blah * [performance] test *": {
		User: "me",
		SIG:  "group",
	}})
	owner := list.TestOwner("blah 200 test foo")
	if owner != "me" {
		t.Error("Unexpected return value ", owner)
	}
	sig := list.TestSIG("blah 200 test foo")
	if sig != "group" {
		t.Error("Unexpected sig: ", sig)
	}
	owner = list.TestOwner("Unknown test")
	if owner != "" {
		t.Error("Unexpected return value ", owner)
	}
	sig = list.TestSIG("Unknown test")
	if sig != "" {
		t.Error("Unexpected sig: ", sig)
	}
}

func TestOwnerListRandom(t *testing.T) {
	list := NewOwnerList(map[string]*OwnerInfo{"testname": {
		User: "a/b/c/d",
	}})
	counts := map[string]int{"a": 0, "b": 0, "c": 0, "d": 0}
	for i := 0; i < 1000; i++ {
		counts[list.TestOwner("testname")]++
	}
	for name, count := range counts {
		if count <= 200 {
			t.Errorf("Too few assigments to %s: only %d, expected > 200", name, count)
		}
	}
}

func TestOwnerListFromCsv(t *testing.T) {
	r := bytes.NewReader([]byte(",,,header nonsense,\n" +
		",owner,suggested owner,name,sig\n" +
		",foo,other,Test name,Node\n" +
		", bar,foo,other test, Windows\n"))
	list, err := NewOwnerListFromCsv(r)
	if err != nil {
		t.Error(err)
	}
	if owner := list.TestOwner("test name"); owner != "foo" {
		t.Error("unexpected return value ", owner)
	}
	if sig := list.TestSIG("test name"); sig != "Node" {
		t.Error("unexpected sig value ", sig)
	}
	if owner := list.TestOwner("other test"); owner != "bar" {
		t.Error("unexpected return value ", owner)
	}
	if sig := list.TestSIG("other test"); sig != "Windows" {
		t.Error("unexpected sig value ", sig)
	}
}

func TestReloadingOwnerList(t *testing.T) {
	cases := []struct {
		name   string
		csv    string
		lookup string
		owner  string
		sig    string
		err    bool
	}{
		{
			name:   "owner and sig",
			csv:    "owner,name,sig\nfoo,flake,Scheduling\n",
			lookup: "flake",
			owner:  "foo",
			sig:    "Scheduling",
		},
		{
			name:   "missing sig returns BadCsv",
			csv:    "owner,name,sig\nfoo,flake\n",
			lookup: "flake",
			err:    true,
		},
	}
	tempfile, err := ioutil.TempFile(os.TempDir(), "ownertest")
	if err != nil {
		t.Error(err)
	}
	defer os.Remove(tempfile.Name())
	defer tempfile.Close()
	writer := bufio.NewWriter(tempfile)

	for _, tc := range cases {
		// Assuming millisecond resolution on our FS, this sleep
		// ensures the mtime will change with the next write.
		time.Sleep(5 * time.Millisecond)
		// Clear file and reset writing offset
		tempfile.Truncate(0)
		tempfile.Seek(0, os.SEEK_SET)
		writer.Reset(tempfile)
		_, err = writer.WriteString(tc.csv)
		if err != nil {
			t.Error(err)
		}
		err = writer.Flush()
		if err != nil {
			t.Error(err)
		}
		list, err := NewReloadingOwnerList(tempfile.Name())
		if err != nil && !tc.err {
			t.Errorf("%s: unexpected error: %v", tc.name, err)
		}
		if tc.err {
			if err == nil {
				t.Errorf("%s: expected an error", tc.name)
			}
			_, ok := err.(BadCsv)
			if !ok {
				t.Errorf("%s: error type is not BadCsv: %v", tc.name, err)
			}
			if list == nil {
				t.Errorf("%s: did not return a list during BadCsv", tc.name)
			}
		}
		if owner := list.TestOwner(tc.lookup); owner != tc.owner {
			t.Errorf("%s: bad owner %s != %s", tc.name, owner, tc.owner)
		}
		if sig := list.TestSIG(tc.lookup); sig != tc.sig {
			t.Errorf("%s: bad sig %s != %s", tc.name, sig, tc.sig)
		}
	}
}
<commit_msg>remove a unit test that flakes based on rand<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testowner

import (
	"bufio"
	"bytes"
	"io/ioutil"
	"os"
	"testing"
	"time"
)

func TestNormalize(t *testing.T) {
	tests := map[string]string{
		"A":                                    "a",
		"Perf [Performance]":                   "perf",
		"[k8s.io] test [performance] stuff":    "test stuff",
		"[k8s.io] blah {Kubernetes e2e suite}": "blah",
	}
	for input, output := range tests {
		result := normalize(input)
		if result != output {
			t.Errorf("normalize(%s) != %s (got %s)", input, output, result)
		}
	}
}

func TestOwnerList(t *testing.T) {
	list := NewOwnerList(map[string]*OwnerInfo{"Perf [performance]": {
		User: "me",
		SIG:  "group",
	}})
	owner := list.TestOwner("perf [flaky]")
	if owner != "me" {
		t.Error("Unexpected return value ", owner)
	}
	sig := list.TestSIG("perf [flaky]")
	if sig != "group" {
		t.Error("Unexpected sig: ", sig)
	}
	owner = list.TestOwner("Unknown test")
	if owner != "" {
		t.Error("Unexpected return value ", owner)
	}
	sig = list.TestSIG("Unknown test")
	if sig != "" {
		t.Error("Unexpected sig: ", sig)
	}
}

func TestOwnerGlob(t *testing.T) {
	list := NewOwnerList(map[string]*OwnerInfo{"blah * [performance] test *": {
		User: "me",
		SIG:  "group",
	}})
	owner := list.TestOwner("blah 200 test foo")
	if owner != "me" {
		t.Error("Unexpected return value ", owner)
	}
	sig := list.TestSIG("blah 200 test foo")
	if sig != "group" {
		t.Error("Unexpected sig: ", sig)
	}
	owner = list.TestOwner("Unknown test")
	if owner != "" {
		t.Error("Unexpected return value ", owner)
	}
	sig = list.TestSIG("Unknown test")
	if sig != "" {
		t.Error("Unexpected sig: ", sig)
	}
}

func TestOwnerListFromCsv(t *testing.T) {
	r := bytes.NewReader([]byte(",,,header nonsense,\n" +
		",owner,suggested owner,name,sig\n" +
		",foo,other,Test name,Node\n" +
		", bar,foo,other test, Windows\n"))
	list, err := NewOwnerListFromCsv(r)
	if err != nil {
		t.Error(err)
	}
	if owner := list.TestOwner("test name"); owner != "foo" {
		t.Error("unexpected return value ", owner)
	}
	if sig := list.TestSIG("test name"); sig != "Node" {
		t.Error("unexpected sig value ", sig)
	}
	if owner := list.TestOwner("other test"); owner != "bar" {
		t.Error("unexpected return value ", owner)
	}
	if sig := list.TestSIG("other test"); sig != "Windows" {
		t.Error("unexpected sig value ", sig)
	}
}

func TestReloadingOwnerList(t *testing.T) {
	cases := []struct {
		name   string
		csv    string
		lookup string
		owner  string
		sig    string
		err    bool
	}{
		{
			name:   "owner and sig",
			csv:    "owner,name,sig\nfoo,flake,Scheduling\n",
			lookup: "flake",
			owner:  "foo",
			sig:    "Scheduling",
		},
		{
			name:   "missing sig returns BadCsv",
			csv:    "owner,name,sig\nfoo,flake\n",
			lookup: "flake",
			err:    true,
		},
	}
	tempfile, err := ioutil.TempFile(os.TempDir(), "ownertest")
	if err != nil {
		t.Error(err)
	}
	defer os.Remove(tempfile.Name())
	defer tempfile.Close()
	writer := bufio.NewWriter(tempfile)

	for _, tc := range cases {
		// Assuming millisecond resolution on our FS, this sleep
		// ensures the mtime will change with the next write.
		time.Sleep(5 * time.Millisecond)
		// Clear file and reset writing offset
		tempfile.Truncate(0)
		tempfile.Seek(0, os.SEEK_SET)
		writer.Reset(tempfile)
		_, err = writer.WriteString(tc.csv)
		if err != nil {
			t.Error(err)
		}
		err = writer.Flush()
		if err != nil {
			t.Error(err)
		}
		list, err := NewReloadingOwnerList(tempfile.Name())
		if err != nil && !tc.err {
			t.Errorf("%s: unexpected error: %v", tc.name, err)
		}
		if tc.err {
			if err == nil {
				t.Errorf("%s: expected an error", tc.name)
			}
			_, ok := err.(BadCsv)
			if !ok {
				t.Errorf("%s: error type is not BadCsv: %v", tc.name, err)
			}
			if list == nil {
				t.Errorf("%s: did not return a list during BadCsv", tc.name)
			}
		}
		if owner := list.TestOwner(tc.lookup); owner != tc.owner {
			t.Errorf("%s: bad owner %s != %s", tc.name, owner, tc.owner)
		}
		if sig := list.TestSIG(tc.lookup); sig != tc.sig {
			t.Errorf("%s: bad sig %s != %s", tc.name, sig, tc.sig)
		}
	}
}
<|endoftext|>


<commit_before>package detect

import (
	"errors"
	"path"

	"../fileutil"
)

/*
	detect - guess the project type from files present
*/

type Project struct {
	Id       string
	Commands Commands
	Detect   func(string) bool
}

type Commands map[string]string

var ProjectTypes = []*Project{
	&Project{"clojure/leiningen", Commands{"build": "lein uberjar", "run": "lein run", "test": "lein test"},
		clojureLeiningen},
	&Project{"docker/fig", Commands{"build": "fig build", "run": "fig up"}, dockerFig},
	&Project{"docker/default", Commands{"build": "docker build ."}, dockerDefault},
	&Project{"executable", Commands{"run": "{file}"}, executableDefault},
	&Project{"go/default", Commands{"build": "go build {file}", "run": "go build $(basename {file}) && ./$(basename {file} .go)"},
		goDefault},
	&Project{"java/maven", Commands{"build": "mvn compile", "test": "mvn compile test"}, javaMaven},
	&Project{"javascript/npm", Commands{"build": "npm install", "test": "npm test"}, javascriptNpm},
	&Project{"javascript/meteor", Commands{"run": "meteor"}, javascriptMeteor},
	&Project{"javascript/default", Commands{"run": "node {file}"}, javascriptDefault},
	&Project{"python/django", Commands{"build": "python manage.py syncdb", "run": "python manage.py runserver",
		"test": "python manage.py test"}, pythonDjango},
	&Project{"python/default", Commands{"run": "python {file}"}, pythonDefault},
	&Project{"ruby/rails", Commands{"run": "rails server", "test": "bundle exec rake test"}, rubyRails},
	&Project{"ruby/rake", Commands{"run": "rake", "test": "rake test"}, rubyRake},
	&Project{"ruby/default", Commands{"run": "ruby {file}"}, rubyDefault},
	&Project{"make", Commands{"run": "make", "test": "make test"}, makeDefault},
	&Project{"procfile", Commands{}, procfileDefault},
}

func Detect(file string) (*Project, error) {
	for _, project := range ProjectTypes {
		if project.Detect(file) {
			return project, nil
		}
	}

	return nil, errors.New("no project matches")
}

func DetectAll(file string) []*Project {
	projects := make([]*Project, 0, len(ProjectTypes))

	for _, project := range ProjectTypes {
		if project.Detect(file) {
			n := len(projects)
			projects = projects[0 : n+1]
			projects[n] = project
		}
	}

	return projects
}

func GetById(id string) *Project {
	for _, project := range ProjectTypes {
		if project.Id == id {
			return project
		}
	}
	return nil
}

func matchingFileOrDir(file string, pattern string) bool {
	if fileutil.IsFile(file) {
		_, f := path.Split(file)
		isMatch, _ := path.Match(pattern, f)
		return isMatch
	} else {
		return fileutil.MatchExists(path.Join(path.Dir(file), pattern))
	}
}

func hasFile(fileOrDir string, file string) bool {
	return fileutil.IsFile(fileutil.Join(fileOrDir, file))
}

func clojureLeiningen(file string) bool {
	return hasFile(file, "project.clj")
}

func dockerFig(file string) bool {
	return hasFile(file, "fig.yml")
}

func dockerDefault(file string) bool {
	return hasFile(file, "Dockerfile")
}

func executableDefault(file string) bool {
	return fileutil.IsExecutable(file)
}

func goDefault(file string) bool {
	return matchingFileOrDir(file, "*.go")
}

func javaMaven(file string) bool {
	return hasFile(file, "pom.xml")
}

func javascriptNpm(file string) bool {
	return hasFile(file, "package.json")
}

func javascriptMeteor(file string) bool {
	return hasFile(file, ".meteor/.id")
}

func javascriptDefault(file string) bool {
	return matchingFileOrDir(file, "*.js")
}

func makeDefault(file string) bool {
	return hasFile(file, "Makefile")
}

func procfileDefault(file string) bool {
	return hasFile(file, "Procfile")
}

func pythonDjango(file string) bool {
	return hasFile(file, "manage.py")
}

func pythonDefault(file string) bool {
	return matchingFileOrDir(file, "*.py")
}

func rubyRails(file string) bool {
	return hasFile(file, "bin/rails")
}

func rubyRake(file string) bool {
	return hasFile(file, "Rakefile")
}

func rubyDefault(file string) bool {
	return matchingFileOrDir(file, "*.rb")
}
<commit_msg>add rake db:migrate as build step for rails.<commit_after>package detect

import (
	"errors"
	"path"

	"../fileutil"
)

/*
	detect - guess the project type from files present
*/

type Project struct {
	Id       string
	Commands Commands
	Detect   func(string) bool
}

type Commands map[string]string

var ProjectTypes = []*Project{
	&Project{"clojure/leiningen", Commands{"build": "lein uberjar", "run": "lein run", "test": "lein test"},
		clojureLeiningen},
	&Project{"docker/fig", Commands{"build": "fig build", "run": "fig up"}, dockerFig},
	&Project{"docker/default", Commands{"build": "docker build ."}, dockerDefault},
	&Project{"executable", Commands{"run": "{file}"}, executableDefault},
	&Project{"go/default", Commands{"build": "go build {file}", "run": "go build $(basename {file}) && ./$(basename {file} .go)"},
		goDefault},
	&Project{"java/maven", Commands{"build": "mvn compile", "test": "mvn compile test"}, javaMaven},
	&Project{"javascript/npm", Commands{"build": "npm install", "test": "npm test"}, javascriptNpm},
	&Project{"javascript/meteor", Commands{"run": "meteor"}, javascriptMeteor},
	&Project{"javascript/default", Commands{"run": "node {file}"}, javascriptDefault},
	&Project{"python/django", Commands{"build": "python manage.py syncdb", "run": "python manage.py runserver",
		"test": "python manage.py test"}, pythonDjango},
	&Project{"python/default", Commands{"run": "python {file}"}, pythonDefault},
	&Project{"ruby/rails", Commands{"build": "bundle exec rake db:migrate", "run": "rails server",
		"test": "bundle exec rake test"}, rubyRails},
	&Project{"ruby/rake", Commands{"run": "rake", "test": "rake test"}, rubyRake},
	&Project{"ruby/default", Commands{"run": "ruby {file}"}, rubyDefault},
	&Project{"make", Commands{"run": "make", "test": "make test"}, makeDefault},
	&Project{"procfile", Commands{}, procfileDefault},
}

func Detect(file string) (*Project, error) {
	for _, project := range ProjectTypes {
		if project.Detect(file) {
			return project, nil
		}
	}

	return nil, errors.New("no project matches")
}

func DetectAll(file string) []*Project {
	projects := make([]*Project, 0, len(ProjectTypes))

	for _, project := range ProjectTypes {
		if project.Detect(file) {
			n := len(projects)
			projects = projects[0 : n+1]
			projects[n] = project
		}
	}

	return projects
}

func GetById(id string) *Project {
	for _, project := range ProjectTypes {
		if project.Id == id {
			return project
		}
	}
	return nil
}

func matchingFileOrDir(file string, pattern string) bool {
	if fileutil.IsFile(file) {
		_, f := path.Split(file)
		isMatch, _ := path.Match(pattern, f)
		return isMatch
	} else {
		return fileutil.MatchExists(path.Join(path.Dir(file), pattern))
	}
}

func hasFile(fileOrDir string, file string) bool {
	return fileutil.IsFile(fileutil.Join(fileOrDir, file))
}

func clojureLeiningen(file string) bool {
	return hasFile(file, "project.clj")
}

func dockerFig(file string) bool {
	return hasFile(file, "fig.yml")
}

func dockerDefault(file string) bool {
	return hasFile(file, "Dockerfile")
}

func executableDefault(file string) bool {
	return fileutil.IsExecutable(file)
}

func goDefault(file string) bool {
	return matchingFileOrDir(file, "*.go")
}

func javaMaven(file string) bool {
	return hasFile(file, "pom.xml")
}

func javascriptNpm(file string) bool {
	return hasFile(file, "package.json")
}

func javascriptMeteor(file string) bool {
	return hasFile(file, ".meteor/.id")
}

func javascriptDefault(file string) bool {
	return matchingFileOrDir(file, "*.js")
}

func makeDefault(file string) bool {
	return hasFile(file, "Makefile")
}

func procfileDefault(file string) bool {
	return hasFile(file, "Procfile")
}

func pythonDjango(file string) bool {
	return hasFile(file, "manage.py")
}

func pythonDefault(file string) bool {
	return matchingFileOrDir(file, "*.py")
}

func rubyRails(file string) bool {
	return hasFile(file, "bin/rails")
}

func rubyRake(file string) bool {
	return hasFile(file, "Rakefile")
}

func rubyDefault(file string) bool {
	return matchingFileOrDir(file, "*.rb")
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package core

import (
	"crypto/ecdsa"
	"crypto/tls"
	"fmt"
	"net/http"
	"runtime"
	"sync/atomic"
	"time"

	"github.com/TheThingsNetwork/go-account-lib/cache"
	"github.com/TheThingsNetwork/go-account-lib/claims"
	"github.com/TheThingsNetwork/go-account-lib/tokenkey"
	pb_discovery "github.com/TheThingsNetwork/ttn/api/discovery"
	pb_noc "github.com/TheThingsNetwork/ttn/api/noc"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/logging"
	"github.com/TheThingsNetwork/ttn/utils/security"
	"github.com/apex/log"
	"github.com/dgrijalva/jwt-go"
	"github.com/mwitkow/go-grpc-middleware"
	"github.com/spf13/viper"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/grpclog"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/peer"
)

type ComponentInterface interface {
	RegisterRPC(s *grpc.Server)
	Init(c *Component) error
	ValidateNetworkContext(ctx context.Context) (*pb_discovery.Announcement, error)
	ValidateTTNAuthContext(ctx context.Context) (*claims.Claims, error)
}

type ManagementInterface interface {
	RegisterManager(s *grpc.Server)
}

// NewComponent creates a new Component
func NewComponent(ctx log.Interface, serviceName string, announcedAddress string) (*Component, error) {
	go func() {
		memstats := new(runtime.MemStats)
		for range time.Tick(time.Minute) {
			runtime.ReadMemStats(memstats)
			ctx.WithFields(log.Fields{
				"Goroutines": runtime.NumGoroutine(),
				"Memory":     float64(memstats.Alloc) / 1000000,
			}).Debugf("Stats")
		}
	}()

	grpclog.SetLogger(logging.NewGRPCLogger(ctx))

	component := &Component{
		Ctx: ctx,
		Identity: &pb_discovery.Announcement{
			Id:             viper.GetString("id"),
			Description:    viper.GetString("description"),
			ServiceName:    serviceName,
			ServiceVersion: fmt.Sprintf("%s-%s (%s)", viper.GetString("version"), viper.GetString("gitCommit"), viper.GetString("buildDate")),
			NetAddress:     announcedAddress,
		},
		AccessToken: viper.GetString("auth-token"),
		TokenKeyProvider: tokenkey.HTTPProvider(
			viper.GetStringMapString("auth-servers"),
			cache.WriteTroughCacheWithFormat(viper.GetString("key-dir"), "auth-%s.pub"),
		),
	}

	if serviceName != "discovery" {
		var err error
		component.Discovery, err = pb_discovery.NewClient(
			viper.GetString("discovery-server"),
			component.Identity,
			func() string {
				token, _ := component.BuildJWT()
				return token
			},
		)
		if err != nil {
			return nil, err
		}
	}

	if priv, err := security.LoadKeypair(viper.GetString("key-dir")); err == nil {
		component.privateKey = priv

		pubPEM, _ := security.PublicPEM(priv)
		component.Identity.PublicKey = string(pubPEM)

		privPEM, _ := security.PrivatePEM(priv)

		if viper.GetBool("tls") {
			cert, err := security.LoadCert(viper.GetString("key-dir"))
			if err != nil {
				return nil, err
			}
			component.Identity.Certificate = string(cert)

			cer, err := tls.X509KeyPair(cert, privPEM)
			if err != nil {
				return nil, err
			}
			component.tlsConfig = &tls.Config{Certificates: []tls.Certificate{cer}}
		}
	}

	if healthPort := viper.GetInt("health-port"); healthPort > 0 {
		http.HandleFunc("/healthz", func(w http.ResponseWriter, req *http.Request) {
			switch component.GetStatus() {
			case StatusHealthy:
				w.WriteHeader(200)
				w.Write([]byte("Status is HEALTHY"))
				return
			case StatusUnhealthy:
				w.WriteHeader(503)
				w.Write([]byte("Status is UNHEALTHY"))
				return
			}
		})
		go http.ListenAndServe(fmt.Sprintf(":%d", healthPort), nil)
	}

	return component, nil
}

// Status indicates the health status of this component
type Status int

const (
	// StatusHealthy indicates a healthy component
	StatusHealthy Status = iota
	// StatusUnhealthy indicates an unhealthy component
	StatusUnhealthy
)

// Component contains the common attributes for all TTN components
type Component struct {
	Identity         *pb_discovery.Announcement
	Discovery        pb_discovery.Client
	Monitor          pb_noc.MonitorClient
	Ctx              log.Interface
	AccessToken      string
	privateKey       *ecdsa.PrivateKey
	tlsConfig        *tls.Config
	TokenKeyProvider tokenkey.Provider
	status           int64
}

// GetStatus gets the health status of the component
func (c *Component) GetStatus() Status {
	return Status(atomic.LoadInt64(&c.status))
}

// SetStatus sets the health status of the component
func (c *Component) SetStatus(status Status) {
	atomic.StoreInt64(&c.status, int64(status))
}

// Discover is used to discover another component
func (c *Component) Discover(serviceName, id string) (*pb_discovery.Announcement, error) {
	res, err := c.Discovery.Get(serviceName, id)
	if err != nil {
		return nil, errors.Wrapf(errors.FromGRPCError(err), "Failed to discover %s/%s", serviceName, id)
	}
	return res, nil
}

// Announce the component to TTN discovery
func (c *Component) Announce() error {
	if c.Identity.Id == "" {
		return errors.NewErrInvalidArgument("Component ID", "can not be empty")
	}
	err := c.Discovery.Announce(c.AccessToken)
	if err != nil {
		return errors.Wrapf(errors.FromGRPCError(err), "Failed to announce this component to TTN discovery: %s", err.Error())
	}
	c.Ctx.Info("ttn: Announced to TTN discovery")

	return nil
}

// UpdateTokenKey updates the OAuth Bearer token key
func (c *Component) UpdateTokenKey() error {
	if c.TokenKeyProvider == nil {
		return errors.NewErrInternal("No public key provider configured for token validation")
	}

	// Set up Auth Server Token Validation
	err := c.TokenKeyProvider.Update()
	if err != nil {
		c.Ctx.Warnf("ttn: Failed to refresh public keys for token validation: %s", err.Error())
	} else {
		c.Ctx.Info("ttn: Got public keys for token validation")
	}

	return nil

}

// ValidateNetworkContext validates the context of a network request (router-broker, broker-handler, etc)
func (c *Component) ValidateNetworkContext(ctx context.Context) (component *pb_discovery.Announcement, err error) {
	defer func() {
		if err != nil {
			time.Sleep(time.Second)
		}
	}()

	md, ok := metadata.FromContext(ctx)
	if !ok {
		err = errors.NewErrInternal("Could not get metadata from context")
		return
	}
	var id, serviceName, token string
	if ids, ok := md["id"]; ok && len(ids) == 1 {
		id = ids[0]
	}
	if id == "" {
		err = errors.NewErrInvalidArgument("Metadata", "id missing")
		return
	}
	if serviceNames, ok := md["service-name"]; ok && len(serviceNames) == 1 {
		serviceName = serviceNames[0]
	}
	if serviceName == "" {
		err = errors.NewErrInvalidArgument("Metadata", "service-name missing")
		return
	}
	if tokens, ok := md["token"]; ok && len(tokens) == 1 {
		token = tokens[0]
	}

	var announcement *pb_discovery.Announcement
	announcement, err = c.Discover(serviceName, id)
	if err != nil {
		return
	}

	if announcement.PublicKey == "" {
		return announcement, nil
	}

	if token == "" {
		err = errors.NewErrInvalidArgument("Metadata", "token missing")
		return
	}

	var claims *jwt.StandardClaims
	claims, err = security.ValidateJWT(token, []byte(announcement.PublicKey))
	if err != nil {
		return
	}
	if claims.Issuer != id {
		err = errors.NewErrInvalidArgument("Metadata", "token was issued by different component id")
		return
	}

	return announcement, nil
}

// ValidateTTNAuthContext gets a token from the context and validates it
func (c *Component) ValidateTTNAuthContext(ctx context.Context) (*claims.Claims, error) {
	md, ok := metadata.FromContext(ctx)
	if !ok {
		return nil, errors.NewErrInternal("Could not get metadata from context")
	}
	token, ok := md["token"]
	if !ok || len(token) < 1 {
		return nil, errors.NewErrInvalidArgument("Metadata", "token missing")
	}

	if c.TokenKeyProvider == nil {
		return nil, errors.NewErrInternal("No token provider configured")
	}

	if token[0] == "" {
		return nil, errors.NewErrInvalidArgument("Metadata", "token is empty")
	}

	claims, err := claims.FromToken(c.TokenKeyProvider, token[0])
	if err != nil {
		return nil, err
	}

	return claims, nil
}

func (c *Component) ServerOptions() []grpc.ServerOption {
	unary := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
		var peerAddr string
		peer, ok := peer.FromContext(ctx)
		if ok {
			peerAddr = peer.Addr.String()
		}
		var peerID string
		meta, ok := metadata.FromContext(ctx)
		if ok {
			id, ok := meta["id"]
			if ok && len(id) > 0 {
				peerID = id[0]
			}
		}
		logCtx := c.Ctx.WithFields(log.Fields{
			"CallerID": peerID,
			"CallerIP": peerAddr,
			"Method":   info.FullMethod,
		})
		t := time.Now()
		iface, err := handler(ctx, req)
		logCtx = logCtx.WithField("Duration", time.Now().Sub(t))
		if err != nil {
			err := errors.FromGRPCError(err)
			logCtx.WithField("error", err.Error()).Warn("Could not handle Request")
		} else {
			logCtx.Info("Handled request")
		}
		return iface, err
	}

	stream := func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
		var peerAddr string
		peer, ok := peer.FromContext(stream.Context())
		if ok {
			peerAddr = peer.Addr.String()
		}
		var peerID string
		meta, ok := metadata.FromContext(stream.Context())
		if ok {
			id, ok := meta["id"]
			if ok && len(id) > 0 {
				peerID = id[0]
			}
		}
		c.Ctx.WithFields(log.Fields{
			"CallerID": peerID,
			"CallerIP": peerAddr,
			"Method":   info.FullMethod,
		}).Info("Start stream")
		return handler(srv, stream)
	}

	opts := []grpc.ServerOption{
		grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unary)),
		grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(stream)),
	}

	if c.tlsConfig != nil {
		opts = append(opts, grpc.Creds(credentials.NewTLS(c.tlsConfig)))
	}

	return opts
}

// BuildJWT builds a short-lived JSON Web Token for this component
func (c *Component) BuildJWT() (string, error) {
	if c.privateKey != nil {
		privPEM, err := security.PrivatePEM(c.privateKey)
		if err != nil {
			return "", err
		}
		return security.BuildJWT(c.Identity.Id, 20*time.Second, privPEM)
	}
	return "", nil
}

// GetContext returns a context for outgoing RPC request. If token is "", this function will generate a short lived token from the component
func (c *Component) GetContext(token string) context.Context {
	var serviceName, id, netAddress string
	if c.Identity != nil {
		serviceName = c.Identity.ServiceName
		id = c.Identity.Id
		if token == "" {
			token, _ = c.BuildJWT()
		}
		netAddress = c.Identity.NetAddress
	}
	md := metadata.Pairs(
		"service-name", serviceName,
		"id", id,
		"token", token,
		"net-address", netAddress,
	)
	ctx := metadata.NewContext(context.Background(), md)
	return ctx
}
<commit_msg>Disable gRPC Tracing<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package core

import (
	"crypto/ecdsa"
	"crypto/tls"
	"fmt"
	"net/http"
	"runtime"
	"sync/atomic"
	"time"

	"github.com/TheThingsNetwork/go-account-lib/cache"
	"github.com/TheThingsNetwork/go-account-lib/claims"
	"github.com/TheThingsNetwork/go-account-lib/tokenkey"
	pb_discovery "github.com/TheThingsNetwork/ttn/api/discovery"
	pb_noc "github.com/TheThingsNetwork/ttn/api/noc"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/logging"
	"github.com/TheThingsNetwork/ttn/utils/security"
	"github.com/apex/log"
	"github.com/dgrijalva/jwt-go"
	"github.com/mwitkow/go-grpc-middleware"
	"github.com/spf13/viper"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/grpclog"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/peer"
)

type ComponentInterface interface {
	RegisterRPC(s *grpc.Server)
	Init(c *Component) error
	ValidateNetworkContext(ctx context.Context) (*pb_discovery.Announcement, error)
	ValidateTTNAuthContext(ctx context.Context) (*claims.Claims, error)
}

type ManagementInterface interface {
	RegisterManager(s *grpc.Server)
}

// NewComponent creates a new Component
func NewComponent(ctx log.Interface, serviceName string, announcedAddress string) (*Component, error) {
	go func() {
		memstats := new(runtime.MemStats)
		for range time.Tick(time.Minute) {
			runtime.ReadMemStats(memstats)
			ctx.WithFields(log.Fields{
				"Goroutines": runtime.NumGoroutine(),
				"Memory":     float64(memstats.Alloc) / 1000000,
			}).Debugf("Stats")
		}
	}()

	// Disable gRPC tracing
	// SEE: https://github.com/grpc/grpc-go/issues/695
	grpc.EnableTracing = false

	grpclog.SetLogger(logging.NewGRPCLogger(ctx))

	component := &Component{
		Ctx: ctx,
		Identity: &pb_discovery.Announcement{
			Id:             viper.GetString("id"),
			Description:    viper.GetString("description"),
			ServiceName:    serviceName,
			ServiceVersion: fmt.Sprintf("%s-%s (%s)", viper.GetString("version"), viper.GetString("gitCommit"), viper.GetString("buildDate")),
			NetAddress:     announcedAddress,
		},
		AccessToken: viper.GetString("auth-token"),
		TokenKeyProvider: tokenkey.HTTPProvider(
			viper.GetStringMapString("auth-servers"),
			cache.WriteTroughCacheWithFormat(viper.GetString("key-dir"), "auth-%s.pub"),
		),
	}

	if serviceName != "discovery" {
		var err error
		component.Discovery, err = pb_discovery.NewClient(
			viper.GetString("discovery-server"),
			component.Identity,
			func() string {
				token, _ := component.BuildJWT()
				return token
			},
		)
		if err != nil {
			return nil, err
		}
	}

	if priv, err := security.LoadKeypair(viper.GetString("key-dir")); err == nil {
		component.privateKey = priv

		pubPEM, _ := security.PublicPEM(priv)
		component.Identity.PublicKey = string(pubPEM)

		privPEM, _ := security.PrivatePEM(priv)

		if viper.GetBool("tls") {
			cert, err := security.LoadCert(viper.GetString("key-dir"))
			if err != nil {
				return nil, err
			}
			component.Identity.Certificate = string(cert)

			cer, err := tls.X509KeyPair(cert, privPEM)
			if err != nil {
				return nil, err
			}
			component.tlsConfig = &tls.Config{Certificates: []tls.Certificate{cer}}
		}
	}

	if healthPort := viper.GetInt("health-port"); healthPort > 0 {
		http.HandleFunc("/healthz", func(w http.ResponseWriter, req *http.Request) {
			switch component.GetStatus() {
			case StatusHealthy:
				w.WriteHeader(200)
				w.Write([]byte("Status is HEALTHY"))
				return
			case StatusUnhealthy:
				w.WriteHeader(503)
				w.Write([]byte("Status is UNHEALTHY"))
				return
			}
		})
		go http.ListenAndServe(fmt.Sprintf(":%d", healthPort), nil)
	}

	return component, nil
}

// Status indicates the health status of this component
type Status int

const (
	// StatusHealthy indicates a healthy component
	StatusHealthy Status = iota
	// StatusUnhealthy indicates an unhealthy component
	StatusUnhealthy
)

// Component contains the common attributes for all TTN components
type Component struct {
	Identity         *pb_discovery.Announcement
	Discovery        pb_discovery.Client
	Monitor          pb_noc.MonitorClient
	Ctx              log.Interface
	AccessToken      string
	privateKey       *ecdsa.PrivateKey
	tlsConfig        *tls.Config
	TokenKeyProvider tokenkey.Provider
	status           int64
}

// GetStatus gets the health status of the component
func (c *Component) GetStatus() Status {
	return Status(atomic.LoadInt64(&c.status))
}

// SetStatus sets the health status of the component
func (c *Component) SetStatus(status Status) {
	atomic.StoreInt64(&c.status, int64(status))
}

// Discover is used to discover another component
func (c *Component) Discover(serviceName, id string) (*pb_discovery.Announcement, error) {
	res, err := c.Discovery.Get(serviceName, id)
	if err != nil {
		return nil, errors.Wrapf(errors.FromGRPCError(err), "Failed to discover %s/%s", serviceName, id)
	}
	return res, nil
}

// Announce the component to TTN discovery
func (c *Component) Announce() error {
	if c.Identity.Id == "" {
		return errors.NewErrInvalidArgument("Component ID", "can not be empty")
	}
	err := c.Discovery.Announce(c.AccessToken)
	if err != nil {
		return errors.Wrapf(errors.FromGRPCError(err), "Failed to announce this component to TTN discovery: %s", err.Error())
	}
	c.Ctx.Info("ttn: Announced to TTN discovery")

	return nil
}

// UpdateTokenKey updates the OAuth Bearer token key
func (c *Component) UpdateTokenKey() error {
	if c.TokenKeyProvider == nil {
		return errors.NewErrInternal("No public key provider configured for token validation")
	}

	// Set up Auth Server Token Validation
	err := c.TokenKeyProvider.Update()
	if err != nil {
		c.Ctx.Warnf("ttn: Failed to refresh public keys for token validation: %s", err.Error())
	} else {
		c.Ctx.Info("ttn: Got public keys for token validation")
	}

	return nil

}

// ValidateNetworkContext validates the context of a network request (router-broker, broker-handler, etc)
func (c *Component) ValidateNetworkContext(ctx context.Context) (component *pb_discovery.Announcement, err error) {
	defer func() {
		if err != nil {
			time.Sleep(time.Second)
		}
	}()

	md, ok := metadata.FromContext(ctx)
	if !ok {
		err = errors.NewErrInternal("Could not get metadata from context")
		return
	}
	var id, serviceName, token string
	if ids, ok := md["id"]; ok && len(ids) == 1 {
		id = ids[0]
	}
	if id == "" {
		err = errors.NewErrInvalidArgument("Metadata", "id missing")
		return
	}
	if serviceNames, ok := md["service-name"]; ok && len(serviceNames) == 1 {
		serviceName = serviceNames[0]
	}
	if serviceName == "" {
		err = errors.NewErrInvalidArgument("Metadata", "service-name missing")
		return
	}
	if tokens, ok := md["token"]; ok && len(tokens) == 1 {
		token = tokens[0]
	}

	var announcement *pb_discovery.Announcement
	announcement, err = c.Discover(serviceName, id)
	if err != nil {
		return
	}

	if announcement.PublicKey == "" {
		return announcement, nil
	}

	if token == "" {
		err = errors.NewErrInvalidArgument("Metadata", "token missing")
		return
	}

	var claims *jwt.StandardClaims
	claims, err = security.ValidateJWT(token, []byte(announcement.PublicKey))
	if err != nil {
		return
	}
	if claims.Issuer != id {
		err = errors.NewErrInvalidArgument("Metadata", "token was issued by different component id")
		return
	}

	return announcement, nil
}

// ValidateTTNAuthContext gets a token from the context and validates it
func (c *Component) ValidateTTNAuthContext(ctx context.Context) (*claims.Claims, error) {
	md, ok := metadata.FromContext(ctx)
	if !ok {
		return nil, errors.NewErrInternal("Could not get metadata from context")
	}
	token, ok := md["token"]
	if !ok || len(token) < 1 {
		return nil, errors.NewErrInvalidArgument("Metadata", "token missing")
	}

	if c.TokenKeyProvider == nil {
		return nil, errors.NewErrInternal("No token provider configured")
	}

	if token[0] == "" {
		return nil, errors.NewErrInvalidArgument("Metadata", "token is empty")
	}

	claims, err := claims.FromToken(c.TokenKeyProvider, token[0])
	if err != nil {
		return nil, err
	}

	return claims, nil
}

func (c *Component) ServerOptions() []grpc.ServerOption {
	unary := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
		var peerAddr string
		peer, ok := peer.FromContext(ctx)
		if ok {
			peerAddr = peer.Addr.String()
		}
		var peerID string
		meta, ok := metadata.FromContext(ctx)
		if ok {
			id, ok := meta["id"]
			if ok && len(id) > 0 {
				peerID = id[0]
			}
		}
		logCtx := c.Ctx.WithFields(log.Fields{
			"CallerID": peerID,
			"CallerIP": peerAddr,
			"Method":   info.FullMethod,
		})
		t := time.Now()
		iface, err := handler(ctx, req)
		logCtx = logCtx.WithField("Duration", time.Now().Sub(t))
		if err != nil {
			err := errors.FromGRPCError(err)
			logCtx.WithField("error", err.Error()).Warn("Could not handle Request")
		} else {
			logCtx.Info("Handled request")
		}
		return iface, err
	}

	stream := func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
		var peerAddr string
		peer, ok := peer.FromContext(stream.Context())
		if ok {
			peerAddr = peer.Addr.String()
		}
		var peerID string
		meta, ok := metadata.FromContext(stream.Context())
		if ok {
			id, ok := meta["id"]
			if ok && len(id) > 0 {
				peerID = id[0]
			}
		}
		c.Ctx.WithFields(log.Fields{
			"CallerID": peerID,
			"CallerIP": peerAddr,
			"Method":   info.FullMethod,
		}).Info("Start stream")
		return handler(srv, stream)
	}

	opts := []grpc.ServerOption{
		grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unary)),
		grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(stream)),
	}

	if c.tlsConfig != nil {
		opts = append(opts, grpc.Creds(credentials.NewTLS(c.tlsConfig)))
	}

	return opts
}

// BuildJWT builds a short-lived JSON Web Token for this component
func (c *Component) BuildJWT() (string, error) {
	if c.privateKey != nil {
		privPEM, err := security.PrivatePEM(c.privateKey)
		if err != nil {
			return "", err
		}
		return security.BuildJWT(c.Identity.Id, 20*time.Second, privPEM)
	}
	return "", nil
}

// GetContext returns a context for outgoing RPC request. If token is "", this function will generate a short lived token from the component
func (c *Component) GetContext(token string) context.Context {
	var serviceName, id, netAddress string
	if c.Identity != nil {
		serviceName = c.Identity.ServiceName
		id = c.Identity.Id
		if token == "" {
			token, _ = c.BuildJWT()
		}
		netAddress = c.Identity.NetAddress
	}
	md := metadata.Pairs(
		"service-name", serviceName,
		"id", id,
		"token", token,
		"net-address", netAddress,
	)
	ctx := metadata.NewContext(context.Background(), md)
	return ctx
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v2alpha1

import (
	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/runtime/schema"
	versionedwatch "k8s.io/kubernetes/pkg/watch/versioned"
)

// GroupName is the group name use in this package
const GroupName = "batch"

// SchemeGroupVersion is group version used to register these objects
var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v2alpha1"}

// Resource takes an unqualified resource and returns a Group qualified GroupResource
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}

var (
	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes, addDefaultingFuncs, addConversionFuncs)
	AddToScheme   = SchemeBuilder.AddToScheme
)

// Adds the list of known types to api.Scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&Job{},
		&JobList{},
		&JobTemplate{},
		&CronJob{},
		&CronJobList{},
		&v1.ListOptions{},
		&v1.DeleteOptions{},
	)
	scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("ScheduledJob"), &CronJob{})
	scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("ScheduledJobList"), &CronJobList{})
	versionedwatch.AddToGroupVersion(scheme, SchemeGroupVersion)
	return nil
}
<commit_msg>Register GetOptions in batch/v2alpha1 group<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v2alpha1

import (
	"k8s.io/kubernetes/pkg/api/v1"
	metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/runtime/schema"
	versionedwatch "k8s.io/kubernetes/pkg/watch/versioned"
)

// GroupName is the group name use in this package
const GroupName = "batch"

// SchemeGroupVersion is group version used to register these objects
var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v2alpha1"}

// Resource takes an unqualified resource and returns a Group qualified GroupResource
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}

var (
	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes, addDefaultingFuncs, addConversionFuncs)
	AddToScheme   = SchemeBuilder.AddToScheme
)

// Adds the list of known types to api.Scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&Job{},
		&JobList{},
		&JobTemplate{},
		&CronJob{},
		&CronJobList{},
		&v1.ListOptions{},
		&v1.DeleteOptions{},
		&metav1.ExportOptions{},
		&metav1.GetOptions{},
	)
	scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("ScheduledJob"), &CronJob{})
	scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("ScheduledJobList"), &CronJobList{})
	versionedwatch.AddToGroupVersion(scheme, SchemeGroupVersion)
	return nil
}
<|endoftext|>


<commit_before>package handler

import (
	"context"
	"encoding/json"
	"time"

	"cloud.google.com/go/pubsub"
	"github.com/checkr/flagr/pkg/config"
	"github.com/checkr/flagr/swagger_gen/models"
	"google.golang.org/api/option"

	"github.com/sirupsen/logrus"
)

type pubsubRecorder struct {
	enabled  bool
	producer *pubsub.Client
	topic    *pubsub.Topic
}

// NewPubsubRecorder creates a new Pubsub recorder
var NewPubsubRecorder = func() DataRecorder {
	client, err := pubsub.NewClient(
		context.Background(),
		config.Config.RecorderPubsubProjectID,
		option.WithCredentialsFile(config.Config.RecorderPubsubKeyFile),
	)
	if err != nil {
		logrus.WithField("pubsub_error", err).Fatal("error getting pubsub client")
	}

	return &pubsubRecorder{
		producer: client,
		topic:    client.Topic(config.Config.RecorderPubsubTopicName),
		enabled:  config.Config.RecorderEnabled,
	}
}

func (p *pubsubRecorder) AsyncRecord(r *models.EvalResult) {
	if !p.enabled {
		return
	}

	pr := &pubsubEvalResult{
		EvalResult: r,
	}

	payload, err := pr.Payload()
	if err != nil {
		logrus.WithField("pubsub_error", err).Error("error marshaling")
	}

	messageFrame := pubsubMessageFrame{
		Payload:   string(payload),
		Encrypted: false,
	}

	_, err = messageFrame.encode()
	if err != nil {
		logrus.WithField("pubsub_error", err).Error("error marshaling")
	}

	ctx := context.Background()
	res := p.topic.Publish(ctx, &pubsub.Message{Data: payload})
	if config.Config.RecorderPubsubVerbose {
		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
		defer cancel()
		id, err := res.Get(ctx)
		if err != nil {
			logrus.WithFields(logrus.Fields{"pubsub_error": err, "id": id}).Error("error pushing to pubsub")
		}
	}
}

type pubsubEvalResult struct {
	*models.EvalResult
}

type pubsubMessageFrame struct {
	Payload   string `json:"payload"`
	Encrypted bool   `json:"encrypted"`
}

func (pmf *pubsubMessageFrame) encode() ([]byte, error) {
	return json.MarshalIndent(pmf, "", "  ")
}

// Payload marshals the EvalResult
func (r *pubsubEvalResult) Payload() ([]byte, error) {
	return r.EvalResult.MarshalBinary()
}
<commit_msg>use encoded frame instead of payload<commit_after>package handler

import (
	"context"
	"encoding/json"
	"time"

	"cloud.google.com/go/pubsub"
	"github.com/checkr/flagr/pkg/config"
	"github.com/checkr/flagr/swagger_gen/models"
	"google.golang.org/api/option"

	"github.com/sirupsen/logrus"
)

type pubsubRecorder struct {
	enabled  bool
	producer *pubsub.Client
	topic    *pubsub.Topic
}

// NewPubsubRecorder creates a new Pubsub recorder
var NewPubsubRecorder = func() DataRecorder {
	client, err := pubsub.NewClient(
		context.Background(),
		config.Config.RecorderPubsubProjectID,
		option.WithCredentialsFile(config.Config.RecorderPubsubKeyFile),
	)
	if err != nil {
		logrus.WithField("pubsub_error", err).Fatal("error getting pubsub client")
	}

	return &pubsubRecorder{
		producer: client,
		topic:    client.Topic(config.Config.RecorderPubsubTopicName),
		enabled:  config.Config.RecorderEnabled,
	}
}

func (p *pubsubRecorder) AsyncRecord(r *models.EvalResult) {
	if !p.enabled {
		return
	}

	pr := &pubsubEvalResult{
		EvalResult: r,
	}

	payload, err := pr.Payload()
	if err != nil {
		logrus.WithField("pubsub_error", err).Error("error marshaling")
	}

	messageFrame := pubsubMessageFrame{
		Payload:   string(payload),
		Encrypted: false,
	}

	message, err := messageFrame.encode()
	if err != nil {
		logrus.WithField("pubsub_error", err).Error("error marshaling")
	}

	ctx := context.Background()
	res := p.topic.Publish(ctx, &pubsub.Message{Data: []byte(message)})
	if config.Config.RecorderPubsubVerbose {
		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
		defer cancel()
		id, err := res.Get(ctx)
		if err != nil {
			logrus.WithFields(logrus.Fields{"pubsub_error": err, "id": id}).Error("error pushing to pubsub")
		}
	}
}

type pubsubEvalResult struct {
	*models.EvalResult
}

type pubsubMessageFrame struct {
	Payload   string `json:"payload"`
	Encrypted bool   `json:"encrypted"`
}

func (pmf *pubsubMessageFrame) encode() ([]byte, error) {
	return json.MarshalIndent(pmf, "", "  ")
}

// Payload marshals the EvalResult
func (r *pubsubEvalResult) Payload() ([]byte, error) {
	return r.EvalResult.MarshalBinary()
}
<|endoftext|>


<commit_before>// Copyright 2015-2016, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package storage provides everything from MySQL, Redis, BoltDB, file, etc functions.

DB Schema Go Generation

bitbucket.org/jatone/genieql
sql query and code generation program. its purpose is to generate as much of the
boilerplate code for interacting with database/sql as possible. without putting
any runtime dependencies into your codebase. it only supports postgresql
currently. adding additional support is very straight forward, just implement
the Dialect interface. see the postgresql implementation as the example.

github.com/go-reform/reform
A better ORM for Go, based on non-empty interfaces and code generation.
https://gopkg.in/reform.v1

DB Migrations

github.com/rubenv/sql-migrate 500 Stars
SQL Schema migration tool for Go. Based on gorp and goose.
Is better because the API for using it within your code (FOSDEM2016).

https://bitbucket.org/liamstask/goose
goose is a database migration tool. You can manage your database's evolution by
creating incremental SQL or Go scripts.
Cons: https://www.reddit.com/r/golang/comments/2dlbz5/database_migration_handling_in_go/

github.com/mattes/migrate 744 Stars
A migration helper written in Go. Use it in your existing Golang code or run
commands via the CLI.

*/
package storage
<commit_msg>storage: Update documentation<commit_after>// Copyright 2015-2016, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package storage provides everything from MySQL, Redis, BoltDB, file etc
// functions.
//
// DB Schema Go Generation
//
// Will be written on our own as all code generation tools have some serious
// architectural flaw. See package codegen.
//
// bitbucket.org/jatone/genieql
// sql query and code generation program. its purpose is to generate as much of the
// boilerplate code for interacting with database/sql as possible. without putting
// any runtime dependencies into your codebase. it only supports postgresql
// currently. adding additional support is very straight forward, just implement
// the Dialect interface. see the postgresql implementation as the example.
//
// github.com/go-reform/reform
// A better ORM for Go, based on non-empty interfaces and code generation.
// https://gopkg.in/reform.v1
package storage
<|endoftext|>