<commit_before>package env

import (
	"encoding"
	"errors"
	"fmt"
	"os"
	"reflect"
	"strconv"
	"strings"

	"github.com/caarlos0/env/parsers"
)

// nolint: gochecknoglobals
var (
	// ErrNotAStructPtr is returned if you pass something that is not a pointer to a
	// Struct to Parse
	ErrNotAStructPtr = errors.New("expected a pointer to a Struct")
	// ErrUnsupportedType if the struct field type is not supported by env
	ErrUnsupportedType = errors.New("type is not supported")
	// ErrUnsupportedSliceType if the slice element type is not supported by env
	ErrUnsupportedSliceType = errors.New("unsupported slice type")
	// OnEnvVarSet is an optional convenience callback, such as for logging purposes.
	// If not nil, it's called after successfully setting the given field from the given value.
	OnEnvVarSet func(reflect.StructField, string)

	defaultBuiltInParsers = map[reflect.Kind]ParserFunc{
		reflect.Bool: func(v string) (interface{}, error) {
			return strconv.ParseBool(v)
		},
		reflect.String: func(v string) (interface{}, error) {
			return v, nil
		},
		reflect.Int: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 32)
			return int(i), err
		},
		reflect.Int16: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 16)
			return int16(i), err
		},
		reflect.Int32: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 32)
			return int32(i), err
		},
		reflect.Int64: func(v string) (interface{}, error) {
			return strconv.ParseInt(v, 10, 64)
		},
		reflect.Int8: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 8)
			return int8(i), err
		},
		reflect.Uint: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 32)
			return uint(i), err
		},
		reflect.Uint16: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 16)
			return uint16(i), err
		},
		reflect.Uint32: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 32)
			return uint32(i), err
		},
		reflect.Uint64: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 64)
			return uint64(i), err
		},
		reflect.Uint8: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 8)
			return uint8(i), err
		},
		reflect.Float64: func(v string) (interface{}, error) {
			return strconv.ParseFloat(v, 64)
		},
		reflect.Float32: func(v string) (interface{}, error) {
			f, err := strconv.ParseFloat(v, 32)
			return float32(f), err
		},
	}
)

func defaultCustomParsers() CustomParsers {
	return CustomParsers{
		parsers.URLType:      parsers.URLFunc,
		parsers.DurationType: parsers.DurationFunc,
	}
}

// CustomParsers is a friendly name for the type that `ParseWithFuncs()` accepts
type CustomParsers map[reflect.Type]ParserFunc

// ParserFunc defines the signature of a function that can be used within `CustomParsers`
type ParserFunc func(v string) (interface{}, error)

// Parse parses a struct containing `env` tags and loads its values from
// environment variables.
func Parse(v interface{}) error {
	return ParseWithFuncs(v, defaultCustomParsers())
}

// ParseWithFuncs is the same as `Parse` except it also allows the user to pass
// in custom parsers.
func ParseWithFuncs(v interface{}, funcMap CustomParsers) error {
	ptrRef := reflect.ValueOf(v)
	if ptrRef.Kind() != reflect.Ptr {
		return ErrNotAStructPtr
	}
	ref := ptrRef.Elem()
	if ref.Kind() != reflect.Struct {
		return ErrNotAStructPtr
	}
	return doParse(ref, funcMap)
}

func doParse(ref reflect.Value, funcMap CustomParsers) error {
	refType := ref.Type()
	var errorList []string

	for i := 0; i < refType.NumField(); i++ {
		refField := ref.Field(i)
		if reflect.Ptr == refField.Kind() && !refField.IsNil() && refField.CanSet() {
			err := Parse(refField.Interface())
			if nil != err {
				return err
			}
			continue
		}
		refTypeField := refType.Field(i)
		value, err := get(refTypeField)
		if err != nil {
			errorList = append(errorList, err.Error())
			continue
		}
		if value == "" {
			if reflect.Struct == refField.Kind() {
				err := doParse(refField, funcMap)
				if nil != err {
					errorList = append(errorList, err.Error())
				}
			}
			continue
		}
		if err := set(refField, refTypeField, value, funcMap); err != nil {
			errorList = append(errorList, err.Error())
			continue
		}
		if OnEnvVarSet != nil {
			OnEnvVarSet(refTypeField, value)
		}
	}
	if len(errorList) == 0 {
		return nil
	}
	return errors.New(strings.Join(errorList, ". "))
}

func get(field reflect.StructField) (string, error) {
	var (
		val string
		err error
	)

	key, opts := parseKeyForOption(field.Tag.Get("env"))

	defaultValue := field.Tag.Get("envDefault")
	val = getOr(key, defaultValue)

	expandVar := field.Tag.Get("envExpand")
	if strings.ToLower(expandVar) == "true" {
		val = os.ExpandEnv(val)
	}

	if len(opts) > 0 {
		for _, opt := range opts {
			// The only option supported is "required".
			switch opt {
			case "":
				break
			case "required":
				val, err = getRequired(key)
			default:
				err = fmt.Errorf("env tag option %q not supported", opt)
			}
		}
	}

	return val, err
}

// split the env tag's key into the expected key and desired option, if any.
func parseKeyForOption(key string) (string, []string) {
	opts := strings.Split(key, ",")
	return opts[0], opts[1:]
}

func getRequired(key string) (string, error) {
	if value, ok := os.LookupEnv(key); ok {
		return value, nil
	}
	return "", fmt.Errorf("required environment variable %q is not set", key)
}

func getOr(key, defaultValue string) string {
	value, ok := os.LookupEnv(key)
	if ok {
		return value
	}
	return defaultValue
}

func set(field reflect.Value, refType reflect.StructField, value string, funcMap CustomParsers) error {
	parserFunc, ok := funcMap[refType.Type]
	if ok {
		val, err := parserFunc(value)
		if err != nil {
			return fmt.Errorf("custom parser error: %v", err)
		}
		field.Set(reflect.ValueOf(val))
		return nil
	}

	parserFunc, ok = defaultBuiltInParsers[field.Kind()]
	if ok {
		val, err := parserFunc(value)
		if err != nil {
			return fmt.Errorf("parser error: %v", err)
		}
		field.Set(reflect.ValueOf(val))
		return nil
	}

	if field.Kind() == reflect.Slice {
		return handleSlice(field, value, refType.Tag.Get("envSeparator"), funcMap)
	}

	return handleTextUnmarshaler(field, value)
}

func handleSlice(field reflect.Value, value, separator string, funcMap CustomParsers) error {
	if separator == "" {
		separator = ","
	}
	parts := strings.Split(value, separator)
	result := reflect.MakeSlice(field.Type(), 0, len(parts))

	parserFunc, ok := funcMap[field.Type().Elem()]
	if !ok {
		parserFunc, ok = defaultBuiltInParsers[field.Type().Elem().Kind()]
		if !ok {
			return fmt.Errorf("no parser for slice of %s", field.Type().Elem().Kind().String())
		}
	}

	for _, part := range parts {
		r, err := parserFunc(part)
		if err != nil {
			return fmt.Errorf("parser error: %v", err)
		}
		result = reflect.Append(result, reflect.ValueOf(r))
	}

	field.Set(result)
	return nil
}

func handleTextUnmarshaler(field reflect.Value, value string) error {
	if reflect.Ptr == field.Kind() {
		if field.IsNil() {
			field.Set(reflect.New(field.Type().Elem()))
		}
	} else if field.CanAddr() {
		field = field.Addr()
	}

	tm, ok := field.Interface().(encoding.TextUnmarshaler)
	if !ok {
		return ErrUnsupportedType
	}

	return tm.UnmarshalText([]byte(value))
}
<commit_msg>fix: custom parsers<commit_after>package env

import (
	"encoding"
	"errors"
	"fmt"
	"os"
	"reflect"
	"strconv"
	"strings"

	"github.com/caarlos0/env/parsers"
)

// nolint: gochecknoglobals
var (
	// ErrNotAStructPtr is returned if you pass something that is not a pointer to a
	// Struct to Parse
	ErrNotAStructPtr = errors.New("expected a pointer to a Struct")
	// ErrUnsupportedType if the struct field type is not supported by env
	ErrUnsupportedType = errors.New("type is not supported")
	// ErrUnsupportedSliceType if the slice element type is not supported by env
	ErrUnsupportedSliceType = errors.New("unsupported slice type")
	// OnEnvVarSet is an optional convenience callback, such as for logging purposes.
	// If not nil, it's called after successfully setting the given field from the given value.
	OnEnvVarSet func(reflect.StructField, string)

	defaultBuiltInParsers = map[reflect.Kind]ParserFunc{
		reflect.Bool: func(v string) (interface{}, error) {
			return strconv.ParseBool(v)
		},
		reflect.String: func(v string) (interface{}, error) {
			return v, nil
		},
		reflect.Int: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 32)
			return int(i), err
		},
		reflect.Int16: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 16)
			return int16(i), err
		},
		reflect.Int32: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 32)
			return int32(i), err
		},
		reflect.Int64: func(v string) (interface{}, error) {
			return strconv.ParseInt(v, 10, 64)
		},
		reflect.Int8: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 8)
			return int8(i), err
		},
		reflect.Uint: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 32)
			return uint(i), err
		},
		reflect.Uint16: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 16)
			return uint16(i), err
		},
		reflect.Uint32: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 32)
			return uint32(i), err
		},
		reflect.Uint64: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 64)
			return uint64(i), err
		},
		reflect.Uint8: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 8)
			return uint8(i), err
		},
		reflect.Float64: func(v string) (interface{}, error) {
			return strconv.ParseFloat(v, 64)
		},
		reflect.Float32: func(v string) (interface{}, error) {
			f, err := strconv.ParseFloat(v, 32)
			return float32(f), err
		},
	}
)

func defaultCustomParsers() CustomParsers {
	return CustomParsers{
		parsers.URLType:      parsers.URLFunc,
		parsers.DurationType: parsers.DurationFunc,
	}
}

// CustomParsers is a friendly name for the type that `ParseWithFuncs()` accepts
type CustomParsers map[reflect.Type]ParserFunc

// ParserFunc defines the signature of a function that can be used within `CustomParsers`
type ParserFunc func(v string) (interface{}, error)

// Parse parses a struct containing `env` tags and loads its values from
// environment variables.
func Parse(v interface{}) error {
	return ParseWithFuncs(v, CustomParsers{})
}

// ParseWithFuncs is the same as `Parse` except it also allows the user to pass
// in custom parsers.
func ParseWithFuncs(v interface{}, funcMap CustomParsers) error {
	ptrRef := reflect.ValueOf(v)
	if ptrRef.Kind() != reflect.Ptr {
		return ErrNotAStructPtr
	}
	ref := ptrRef.Elem()
	if ref.Kind() != reflect.Struct {
		return ErrNotAStructPtr
	}
	var parsers = defaultCustomParsers()
	for k, v := range funcMap {
		parsers[k] = v
	}
	return doParse(ref, parsers)
}

func doParse(ref reflect.Value, funcMap CustomParsers) error {
	refType := ref.Type()
	var errorList []string

	for i := 0; i < refType.NumField(); i++ {
		refField := ref.Field(i)
		if reflect.Ptr == refField.Kind() && !refField.IsNil() && refField.CanSet() {
			err := Parse(refField.Interface())
			if nil != err {
				return err
			}
			continue
		}
		refTypeField := refType.Field(i)
		value, err := get(refTypeField)
		if err != nil {
			errorList = append(errorList, err.Error())
			continue
		}
		if value == "" {
			if reflect.Struct == refField.Kind() {
				err := doParse(refField, funcMap)
				if nil != err {
					errorList = append(errorList, err.Error())
				}
			}
			continue
		}
		if err := set(refField, refTypeField, value, funcMap); err != nil {
			errorList = append(errorList, err.Error())
			continue
		}
		if OnEnvVarSet != nil {
			OnEnvVarSet(refTypeField, value)
		}
	}
	if len(errorList) == 0 {
		return nil
	}
	return errors.New(strings.Join(errorList, ". "))
}

func get(field reflect.StructField) (string, error) {
	var (
		val string
		err error
	)

	key, opts := parseKeyForOption(field.Tag.Get("env"))

	defaultValue := field.Tag.Get("envDefault")
	val = getOr(key, defaultValue)

	expandVar := field.Tag.Get("envExpand")
	if strings.ToLower(expandVar) == "true" {
		val = os.ExpandEnv(val)
	}

	if len(opts) > 0 {
		for _, opt := range opts {
			// The only option supported is "required".
			switch opt {
			case "":
				break
			case "required":
				val, err = getRequired(key)
			default:
				err = fmt.Errorf("env tag option %q not supported", opt)
			}
		}
	}

	return val, err
}

// split the env tag's key into the expected key and desired option, if any.
func parseKeyForOption(key string) (string, []string) {
	opts := strings.Split(key, ",")
	return opts[0], opts[1:]
}

func getRequired(key string) (string, error) {
	if value, ok := os.LookupEnv(key); ok {
		return value, nil
	}
	return "", fmt.Errorf("required environment variable %q is not set", key)
}

func getOr(key, defaultValue string) string {
	value, ok := os.LookupEnv(key)
	if ok {
		return value
	}
	return defaultValue
}

func set(field reflect.Value, refType reflect.StructField, value string, funcMap CustomParsers) error {
	parserFunc, ok := funcMap[refType.Type]
	if ok {
		val, err := parserFunc(value)
		if err != nil {
			return fmt.Errorf("custom parser error: %v", err)
		}
		field.Set(reflect.ValueOf(val))
		return nil
	}

	parserFunc, ok = defaultBuiltInParsers[field.Kind()]
	if ok {
		val, err := parserFunc(value)
		if err != nil {
			return fmt.Errorf("parser error: %v", err)
		}
		field.Set(reflect.ValueOf(val))
		return nil
	}

	if field.Kind() == reflect.Slice {
		return handleSlice(field, value, refType.Tag.Get("envSeparator"), funcMap)
	}

	return handleTextUnmarshaler(field, value)
}

func handleSlice(field reflect.Value, value, separator string, funcMap CustomParsers) error {
	if separator == "" {
		separator = ","
	}
	parts := strings.Split(value, separator)
	result := reflect.MakeSlice(field.Type(), 0, len(parts))

	parserFunc, ok := funcMap[field.Type().Elem()]
	if !ok {
		parserFunc, ok = defaultBuiltInParsers[field.Type().Elem().Kind()]
		if !ok {
			return fmt.Errorf("no parser for slice of %s", field.Type().Elem().Kind().String())
		}
	}

	for _, part := range parts {
		r, err := parserFunc(part)
		if err != nil {
			return fmt.Errorf("parser error: %v", err)
		}
		result = reflect.Append(result, reflect.ValueOf(r))
	}

	field.Set(result)
	return nil
}

func handleTextUnmarshaler(field reflect.Value, value string) error {
	if reflect.Ptr == field.Kind() {
		if field.IsNil() {
			field.Set(reflect.New(field.Type().Elem()))
		}
	} else if field.CanAddr() {
		field = field.Addr()
	}

	tm, ok := field.Interface().(encoding.TextUnmarshaler)
	if !ok {
		return ErrUnsupportedType
	}

	return tm.UnmarshalText([]byte(value))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"

	"github.com/couchbaselabs/cbfs/client"
	"github.com/couchbaselabs/cbfs/tools"
	"github.com/dustin/httputil"
)

type findType int

const (
	findTypeAny = findType(iota)
	findTypeFile
	findTypeDir
)

var findFlags = flag.NewFlagSet("find", flag.ExitOnError)
var findTemplate = findFlags.String("t", "", "Display template")
var findTemplateFile = findFlags.String("T", "", "Display template filename")
var findDashName = findFlags.String("name", "", "Glob name to match")

var findDashType findType

const defaultFindTemplate = `{{.Name}}
`

func (t findType) String() string {
	switch t {
	case findTypeAny:
		return ""
	case findTypeFile:
		return "f"
	case findTypeDir:
		return "d"
	}
	panic("unreachable")
}

func (t *findType) Set(s string) error {
	switch s {
	case "":
		*t = findTypeAny
	case "f":
		*t = findTypeFile
	case "d":
		*t = findTypeDir
	default:
		return fmt.Errorf("must be 'f' or 'd'")
	}
	return nil
}

func init() {
	findFlags.Var(&findDashType, "type", "Type to match (f or d)")
}

type dirAndFileMatcher struct {
	m map[string]struct{}
}

func newDirAndFileMatcher() dirAndFileMatcher {
	return dirAndFileMatcher{map[string]struct{}{}}
}

type findMatch struct {
	path  string
	isDir bool
}

func (d dirAndFileMatcher) match(name string, isdir bool) bool {
	switch findDashType {
	case findTypeAny:
	case findTypeFile:
		if isdir {
			return false
		}
	case findTypeDir:
		if !isdir {
			return false
		}
	}
	matched, err := filepath.Match(*findDashName, name)
	if err != nil {
		log.Fatalf("Error globbing: %v", err)
	}
	return matched
}

func (d dirAndFileMatcher) matches(name string) []findMatch {
	if *findDashName == "" {
		return []findMatch{{name, false}}
	}
	var matches []findMatch

	dir := filepath.Dir(name)
	for dir != "." {
		if _, seen := d.m[dir]; !seen {
			matched := d.match(filepath.Base(dir), true)
			d.m[dir] = struct{}{}
			if matched {
				matches = append(matches, findMatch{dir, true})
			}
		}
		dir = filepath.Dir(dir)
	}
	// Reverse these so the traversal order makes sense
	for i := 0; i < len(matches)/2; i++ {
		j := len(matches) - i - 1
		matches[i], matches[j] = matches[j], matches[i]
	}

	if d.match(filepath.Base(name), false) {
		matches = append(matches, findMatch{name, false})
	}
	return matches
}

func findCommand(u string, args []string) {
	src := findFlags.Arg(0)
	for src[len(src)-1] == '/' {
		src = src[:len(src)-1]
	}

	tmpl := cbfstool.GetTemplate(*findTemplate, *findTemplateFile,
		defaultFindTemplate)

	httputil.InitHTTPTracker(false)

	client, err := cbfsclient.New(u)
	cbfstool.MaybeFatal(err, "Can't build a client: %v", err)

	things, err := client.ListDepth(src, 4096)
	cbfstool.MaybeFatal(err, "Can't list things: %v", err)

	matcher := newDirAndFileMatcher()
	for fn, inf := range things.Files {
		fn = fn[len(src)+1:]
		for _, match := range matcher.matches(fn) {
			if err := tmpl.Execute(os.Stdout, struct {
				Name  string
				IsDir bool
				Meta  cbfsclient.FileMeta
			}{match.path, match.isDir, inf}); err != nil {
				log.Fatalf("Error executing template: %v", err)
			}
		}
	}
}
<commit_msg>Minor find refactoring<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"

	"github.com/couchbaselabs/cbfs/client"
	"github.com/couchbaselabs/cbfs/tools"
	"github.com/dustin/httputil"
)

type findType int

const (
	findTypeAny = findType(iota)
	findTypeFile
	findTypeDir
)

var findFlags = flag.NewFlagSet("find", flag.ExitOnError)
var findTemplate = findFlags.String("t", "", "Display template")
var findTemplateFile = findFlags.String("T", "", "Display template filename")
var findDashName = findFlags.String("name", "", "Glob name to match")

var findDashType findType

const defaultFindTemplate = `{{.Name}}
`

func (t findType) String() string {
	switch t {
	case findTypeAny:
		return ""
	case findTypeFile:
		return "f"
	case findTypeDir:
		return "d"
	}
	panic("unreachable")
}

func (t *findType) Set(s string) error {
	switch s {
	case "":
		*t = findTypeAny
	case "f":
		*t = findTypeFile
	case "d":
		*t = findTypeDir
	default:
		return fmt.Errorf("must be 'f' or 'd'")
	}
	return nil
}

func init() {
	findFlags.Var(&findDashType, "type", "Type to match (f or d)")
}

type dirAndFileMatcher struct {
	m map[string]struct{}
}

func newDirAndFileMatcher() dirAndFileMatcher {
	return dirAndFileMatcher{map[string]struct{}{}}
}

type findMatch struct {
	path  string
	isDir bool
}

func (d dirAndFileMatcher) match(name string, isdir bool) bool {
	switch findDashType {
	case findTypeAny:
	case findTypeFile:
		if isdir {
			return false
		}
	case findTypeDir:
		if !isdir {
			return false
		}
	}
	matched, err := filepath.Match(*findDashName, name)
	if err != nil {
		log.Fatalf("Error globbing: %v", err)
	}
	return matched
}

func (d dirAndFileMatcher) matches(name string) []findMatch {
	if *findDashName == "" {
		return []findMatch{{name, false}}
	}
	var matches []findMatch

	dir := filepath.Dir(name)
	for dir != "." {
		if _, seen := d.m[dir]; !seen {
			d.m[dir] = struct{}{}
			if d.match(filepath.Base(dir), true) {
				matches = append(matches, findMatch{dir, true})
			}
		}
		dir = filepath.Dir(dir)
	}
	// Reverse these so the traversal order makes sense
	for i := 0; i < len(matches)/2; i++ {
		j := len(matches) - i - 1
		matches[i], matches[j] = matches[j], matches[i]
	}

	if d.match(filepath.Base(name), false) {
		matches = append(matches, findMatch{name, false})
	}
	return matches
}

func findCommand(u string, args []string) {
	src := findFlags.Arg(0)
	for src[len(src)-1] == '/' {
		src = src[:len(src)-1]
	}

	tmpl := cbfstool.GetTemplate(*findTemplate, *findTemplateFile,
		defaultFindTemplate)

	httputil.InitHTTPTracker(false)

	client, err := cbfsclient.New(u)
	cbfstool.MaybeFatal(err, "Can't build a client: %v", err)

	things, err := client.ListDepth(src, 4096)
	cbfstool.MaybeFatal(err, "Can't list things: %v", err)

	matcher := newDirAndFileMatcher()
	for fn, inf := range things.Files {
		fn = fn[len(src)+1:]
		for _, match := range matcher.matches(fn) {
			if err := tmpl.Execute(os.Stdout, struct {
				Name  string
				IsDir bool
				Meta  cbfsclient.FileMeta
			}{match.path, match.isDir, inf}); err != nil {
				log.Fatalf("Error executing template: %v", err)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Take well-formed json from a sensu check result and create an elasticsearch document to be used to
// generate user specific dashboards or highly contextual alerts.
//
// LICENSE:
//   Copyright 2015 Yieldbot. <devops@yieldbot.com>
//   Released under the MIT License; see LICENSE
//   for details.

package main

import (
	"flag"
	"fmt"
	"github.com/yieldbot/ybsensues/Godeps/_workspace/src/github.com/yieldbot/ybsensu/handler"
	"github.com/yieldbot/ybsensues/Godeps/_workspace/src/github.com/yieldbot/ybsensu/util"
	"github.com/yieldbot/ybsensues/lib"
	"github.com/yieldbot/ybsensues/Godeps/_workspace/src/github.com/olivere/elastic"
	"time"
)

func main() {

	// set commandline flags
	esIndexPtr := flag.String("index", lib.StatusEsIndex, "the elasticsearch index to use")
	esHostPtr := flag.String("host", lib.DefaultEsHost, "the elasticsearch host")
	esPortPtr := flag.String("port", lib.DefaultEsPort, "the elasticsearch port")

	flag.Parse()
	esIndex := *esIndexPtr
	esType := lib.DefaultEsType
	esHost := *esHostPtr
	esPort := *esPortPtr

	sensuEvent := new(handler.SensuEvent)

	sensuEnv := handler.SetSensuEnv()
	sensuEvent = sensuEvent.AcquireSensuEvent()

	// Create a client
	client, err := elastic.NewClient(
		elastic.SetURL("http://" + esHost + ":" + esPort),
	)
	if err != nil {
		util.EHndlr(err)
	}

	// Check to see if the index exists and if not create it
	if client.IndexExists(esIndex) == nil { // need to test to make sure this does what I want
		_, err = client.CreateIndex(esIndex).Do()
		if err != nil {
			util.EHndlr(err)
		}
	}

	// Create an Elasticsearch document. The document type will define the mapping used for the document.
	doc := make(map[string]interface{})
	var docID string
	docID = handler.EventName(sensuEvent.Client.Name, sensuEvent.Check.Name)
	doc["monitored_instance"] = sensuEvent.AcquireMonitoredInstance()
	doc["sensu_client"] = sensuEvent.Client.Name
	doc["incident_timestamp"] = time.Unix(sensuEvent.Check.Issued, 0).Format(time.RFC3339)
	doc["check_name"] = handler.CreateCheckName(sensuEvent.Check.Name)
	doc["check_state"] = handler.DefineStatus(sensuEvent.Check.Status)
	doc["sensuEnv"] = handler.DefineSensuEnv(sensuEnv.Sensu.Environment)
	doc["tags"] = sensuEvent.Check.Tags
	doc["instance_address"] = sensuEvent.Client.Address
	doc["check_state_duration"] = handler.DefineCheckStateDuration()

	// Add a document to the Elasticsearch index
	_, err = client.Index().
		Index(esIndex).
		Type(esType).
		Id(docID).
		BodyJson(doc).
		Do()
	if err != nil {
		util.EHndlr(err)
	}

	// Log a successful document push to stdout. I don't add the id here as some id's are fixed but
	// the user has the ability to autogenerate an id if they don't want to provide one.
	fmt.Printf("Record added to ES\n")
}
<commit_msg>format issue<commit_after>// Take well-formed json from a sensu check result and create an elasticsearch document to be used to
// generate user specific dashboards or highly contextual alerts.
//
// LICENSE:
//   Copyright 2015 Yieldbot. <devops@yieldbot.com>
//   Released under the MIT License; see LICENSE
//   for details.

package main

import (
	"flag"
	"fmt"
	"github.com/yieldbot/ybsensues/Godeps/_workspace/src/github.com/olivere/elastic"
	"github.com/yieldbot/ybsensues/Godeps/_workspace/src/github.com/yieldbot/ybsensu/handler"
	"github.com/yieldbot/ybsensues/Godeps/_workspace/src/github.com/yieldbot/ybsensu/util"
	"github.com/yieldbot/ybsensues/lib"
	"time"
)

func main() {

	// set commandline flags
	esIndexPtr := flag.String("index", lib.StatusEsIndex, "the elasticsearch index to use")
	esHostPtr := flag.String("host", lib.DefaultEsHost, "the elasticsearch host")
	esPortPtr := flag.String("port", lib.DefaultEsPort, "the elasticsearch port")

	flag.Parse()
	esIndex := *esIndexPtr
	esType := lib.DefaultEsType
	esHost := *esHostPtr
	esPort := *esPortPtr

	sensuEvent := new(handler.SensuEvent)

	sensuEnv := handler.SetSensuEnv()
	sensuEvent = sensuEvent.AcquireSensuEvent()

	// Create a client
	client, err := elastic.NewClient(
		elastic.SetURL("http://" + esHost + ":" + esPort),
	)
	if err != nil {
		util.EHndlr(err)
	}

	// Check to see if the index exists and if not create it
	if client.IndexExists(esIndex) == nil { // need to test to make sure this does what I want
		_, err = client.CreateIndex(esIndex).Do()
		if err != nil {
			util.EHndlr(err)
		}
	}

	// Create an Elasticsearch document. The document type will define the mapping used for the document.
	doc := make(map[string]interface{})
	var docID string
	docID = handler.EventName(sensuEvent.Client.Name, sensuEvent.Check.Name)
	doc["monitored_instance"] = sensuEvent.AcquireMonitoredInstance()
	doc["sensu_client"] = sensuEvent.Client.Name
	doc["incident_timestamp"] = time.Unix(sensuEvent.Check.Issued, 0).Format(time.RFC3339)
	doc["check_name"] = handler.CreateCheckName(sensuEvent.Check.Name)
	doc["check_state"] = handler.DefineStatus(sensuEvent.Check.Status)
	doc["sensuEnv"] = handler.DefineSensuEnv(sensuEnv.Sensu.Environment)
	doc["tags"] = sensuEvent.Check.Tags
	doc["instance_address"] = sensuEvent.Client.Address
	doc["check_state_duration"] = handler.DefineCheckStateDuration()

	// Add a document to the Elasticsearch index
	_, err = client.Index().
		Index(esIndex).
		Type(esType).
		Id(docID).
		BodyJson(doc).
		Do()
	if err != nil {
		util.EHndlr(err)
	}

	// Log a successful document push to stdout. I don't add the id here as some id's are fixed but
	// the user has the ability to autogenerate an id if they don't want to provide one.
	fmt.Printf("Record added to ES\n")
}
<|endoftext|>


<commit_before>package handlers

import (
	r "reflect"
	"testing"

	"github.com/anonx/sunplate/internal/reflect"
	"github.com/anonx/sunplate/log"
)

func TestProcessPackage(t *testing.T) {
	ps := packages{}
	ps.processPackage("github.com/anonx/sunplate/commands/generate/handlers/testdata/controllers")
}

func TestParentPackage(t *testing.T) {
	p := parent{}
	s := p.Package()
	if s != "" {
		// E.g. if we are using it for generation of:
		//	uniquePkgName.Application.Index.
		// In case the Application is local (i.e. its import is empty) we need:
		//	Application.Index.
		// I.e. the method must return empty string.
		t.Errorf("Packages with empty imports must have no names.")
	}
	p = parent{
		ID:     1,
		Import: "net/http",
		Name:   "Request",
	}
	s = p.Package(".XXX")
	if s != "c1.XXX" {
		t.Errorf(`Incorrect package name. Expected "c1.XXX", got "%s".`, s)
	}
}

func TestControllerIgnoredArgs(t *testing.T) {
	c := controller{}
	a := ps["github.com/anonx/sunplate/internal/programs/generate/handlers/testdata/controllers"]["App"].Actions[1]
	exp := ", _, _"
	if r := c.IgnoredArgs(&a); r != exp {
		t.Errorf(`Incorrect IgnoreArgs result. Expected "%s", got "%s".`, exp, r)
	}
}

func assertDeepEqualController(c1, c2 *controller) {
	if c1 == nil || c2 == nil {
		if c1 != c2 {
			log.Error.Panicf(
				"One of the controllers is equal to nil while another is not: %#v != %#v.", c1, c2,
			)
		}
		return
	}
	if c1.File != c2.File {
		log.Error.Panicf("Controllers are from different files: %s != %s.", c1.File, c2.File)
	}
	if !r.DeepEqual(c1.Comments, c2.Comments) {
		log.Error.Panicf("Controllers have different comments: %#v != %#v.", c1.Comments, c2.Comments)
	}
	if !r.DeepEqual(c1.Parents, c2.Parents) {
		log.Error.Panicf("Controllers have different parent controllers: %#v != %#v.", c1.Parents, c2.Parents)
	}
	if err := reflect.AssertEqualFuncs(c1.Actions, c2.Actions); err != nil {
		log.Error.Panic(err)
	}
	if err := reflect.AssertEqualFunc(c1.After, c2.After); err != nil {
		log.Error.Panic(err)
	}
	if err := reflect.AssertEqualFunc(c1.Before, c2.Before); err != nil {
		log.Error.Panic(err)
	}
	if err := reflect.AssertEqualFunc(c1.Finally, c2.Finally); err != nil {
		log.Error.Panic(err)
	}
}

func assertDeepEqualControllers(cs1, cs2 controllers) {
	if len(cs1) != len(cs2) {
		log.Error.Panicf(
			"controllers maps %#v and %#v have different length: %d != %d",
			cs1, cs2, len(cs1), len(cs2),
		)
	}
	for i := range cs1 {
		c1 := cs1[i]
		c2 := cs2[i]
		assertDeepEqualController(&c1, &c2)
	}
}

func assertDeepEqualPkgs(ps1, ps2 packages) {
	if len(ps1) != len(ps2) {
		log.Error.Panicf(
			"packages maps %#v and %#v have different length: %d != %d",
			ps1, ps2, len(ps1), len(ps2),
		)
	}
	for i := range ps1 {
		assertDeepEqualControllers(ps1[i], ps2[i])
	}
}

var ps = packages{
	"github.com/anonx/sunplate/internal/programs/generate/handlers/testdata/controllers": controllers{
		"Controller": controller{
			After: &reflect.Func{
				Comments: []string{"// After is a magic method that is executed after every request."},
				File:     "init.go",
				Name:     "After",
				Params: []reflect.Arg{
					{
						Name: "name",
						Type: &reflect.Type{
							Name: "string",
						},
					},
				},
				Recv: &reflect.Arg{
					Name: "c",
					Type: &reflect.Type{
						Name: "Controller",
						Star: true,
					},
				},
				Results: []reflect.Arg{
					{
						Type: &reflect.Type{
							Name:    "Handler",
							Package: "h",
						},
					},
				},
			},
			Before: &reflect.Func{
				Comments: []string{"// Before is a magic method that is executed before every request."},
				File:     "init.go",
				Name:     "Before",
				Params: []reflect.Arg{
					{
						Name: "uid",
						Type: &reflect.Type{
							Name: "string",
						},
					},
				},
				Recv: &reflect.Arg{
					Name: "c",
					Type: &reflect.Type{
						Name: "Controller",
						Star: true,
					},
				},
				Results: []reflect.Arg{
					{
						Type: &reflect.Type{
							Name:    "Handler",
							Package: "h",
						},
					},
				},
			},
			Finally: &reflect.Func{
				Comments: []string{"// Finally is a magic method that is executed after every request."},
				File:     "app.go",
				Name:     "Finally",
				Params: []reflect.Arg{
					{
						Name: "w",
						Type: &reflect.Type{
							Package: "http",
							Name:    "ResponseWriter",
						},
					},
					{
						Name: "r",
						Type: &reflect.Type{
							Package: "http",
							Name:    "Request",
							Star:    true,
						},
					},
				},
				Recv: &reflect.Arg{
					Name: "c",
					Type: &reflect.Type{
						Name: "Controller",
						Star: true,
					},
				},
				Results: []reflect.Arg{
					{
						Type: &reflect.Type{
							Name: "bool",
						},
					},
				},
			},
			Initially: &reflect.Func{
				Comments: []string{"// Initially is a magic method that is executed before every request."},
				File:     "app.go",
				Name:     "Initially",
				Params: []reflect.Arg{
					{
						Name: "w",
						Type: &reflect.Type{
							Package: "http",
							Name:    "ResponseWriter",
						},
					},
					{
						Name: "r",
						Type: &reflect.Type{
							Package: "http",
							Name:    "Request",
							Star:    true,
						},
					},
				},
				Recv: &reflect.Arg{
					Name: "c",
					Type: &reflect.Type{
						Name: "Controller",
						Star: true,
					},
				},
				Results: []reflect.Arg{
					{
						Type: &reflect.Type{
							Name: "bool",
						},
					},
				},
			},

			Comments: []string{
				"// Controller is a struct that should be embedded into every controller",
				"// of your app to make methods provided by middleware controllers available.",
			},
			File: "init.go",
			Parents: []parent{
				{
					Import: "github.com/anonx/sunplate/internal/programs/generate/handlers/testdata/controllers/subpackage",
					Name:   "Controller",
				},
			},
		},
		"App": controller{
			Actions: []reflect.Func{
				{
					Comments: []string{"// Index is a sample action."},
					File:     "init.go",
					Name:     "Index",
					Params: []reflect.Arg{
						{
							Name: "page",
							Type: &reflect.Type{
								Name: "int",
							},
						},
					},
					Recv: &reflect.Arg{
						Name: "c",
						Type: &reflect.Type{
							Name: "App",
							Star: true,
						},
					},
					Results: []reflect.Arg{
						{
							Type: &reflect.Type{
								Name:    "Handler",
								Package: "h",
							},
						},
					},
				},
				{
					Comments: []string{"// HelloWorld is a sample action."},
					File:     "app.go",
					Name:     "HelloWorld",
					Params: []reflect.Arg{
						{
							Name: "page",
							Type: &reflect.Type{
								Name: "int",
							},
						},
					},
					Recv: &reflect.Arg{
						Name: "c",
						Type: &reflect.Type{
							Name: "App",
						},
					},
					Results: []reflect.Arg{
						{
							Type: &reflect.Type{
								Name:    "Handler",
								Package: "http",
							},
						},
						{
							Type: &reflect.Type{
								Name: "bool",
							},
						},
						{
							Type: &reflect.Type{
								Name: "error",
							},
						},
					},
				},
			},
			After:   &reflect.Func{},
			Before:  &reflect.Func{},
			Finally: &reflect.Func{},

			Comments: []string{
				"// App is a sample controller.",
			},
			File: "app.go",
			Parents: []parent{
				{
					Import: "github.com/anonx/sunplate/internal/programs/generate/handlers/testdata/controllers",
					Name:   "Controller",
				},
			},
		},
	},
	"github.com/anonx/sunplate/internal/programs/generate/handlers/testdata/controllers/subpackage": controllers{
		"Controller": controller{
			Actions: []reflect.Func{
				{
					Comments: []string{"// Index is a sample action."},
					File:     "app.go",
					Name:     "Index",
					Params: []reflect.Arg{
						{
							Name: "page",
							Type: &reflect.Type{
								Name: "int",
							},
						},
					},
					Recv: &reflect.Arg{
						Name: "c",
						Type: &reflect.Type{
							Name: "App",
							Star: true,
						},
					},
					Results: []reflect.Arg{
						{
							Type: &reflect.Type{
								Name:    "Handler",
								Package: "http",
							},
						},
					},
				},
			},
			After: &reflect.Func{
				Comments: []string{
					"// After is a magic function that is executed after any request.",
				},
				File:   "app.go",
				Name:   "After",
				Params: []reflect.Arg{},
				Recv: &reflect.Arg{
					Name: "c",
					Type: &reflect.Type{
						Name: "Controller",
						Star: true,
					},
				},
				Results: []reflect.Arg{
					{
						Type: &reflect.Type{
							Name:    "Handler",
							Package: "http",
						},
					},
				},
			},
			Before: &reflect.Func{
				Comments: []string{
					"// Before is a magic function that is executed before any request.",
				},
				File:   "app.go",
				Name:   "Before",
				Params: []reflect.Arg{},
				Recv: &reflect.Arg{
					Name: "c",
					Type: &reflect.Type{
						Name: "Controller",
						Star: true,
					},
				},
				Results: []reflect.Arg{
					{
						Type: &reflect.Type{
							Name:    "Handler",
							Package: "http",
						},
					},
				},
			},
			Finally: &reflect.Func{
				Comments: []string{
					"// Finally is a magic function that is executed after any request",
					"// no matter what.",
				},
				File: "app.go",
				Name: "Finally",
				Params: []reflect.Arg{
					{
						Name: "w",
						Type: &reflect.Type{
							Package: "http",
							Name:    "ResponseWriter",
						},
					},
					{
						Name: "r",
						Type: &reflect.Type{
							Package: "http",
							Name:    "Request",
							Star:    true,
						},
					},
				},
				Recv: &reflect.Arg{
					Name: "c",
					Type: &reflect.Type{
						Name: "Controller",
						Star: true,
					},
				},
				Results: []reflect.Arg{
					{
						Type: &reflect.Type{
							Name: "bool",
						},
					},
				},
			},

			Comments: []string{
				"// Controller is some controller.",
			},
			File:    "app.go",
			Parents: []parent{},
		},
	},
}
<commit_msg>Fixing broken tests of 'generate handlers'<commit_after>package handlers

import (
	r "reflect"
	"testing"

	"github.com/anonx/sunplate/internal/reflect"
	"github.com/anonx/sunplate/log"
)

func TestProcessPackage(t *testing.T) {
	ps := packages{}
	ps.processPackage("github.com/anonx/sunplate/commands/generate/handlers/testdata/controllers")
}

func TestParentPackage(t *testing.T) {
	p := parent{}
	s := p.Package()
	if s != "" {
		// E.g. if we are using it for generation of:
		//	uniquePkgName.Application.Index.
		// In case the Application is local (i.e. its import is empty) we need:
		//	Application.Index.
		// I.e. the method must return empty string.
		t.Errorf("Packages with empty imports must have no names.")
	}
	p = parent{
		ID:     1,
		Import: "net/http",
		Name:   "Request",
	}
	s = p.Package(".XXX")
	if s != "c1.XXX" {
		t.Errorf(`Incorrect package name. Expected "c1.XXX", got "%s".`, s)
	}
}

func TestControllerIgnoredArgs(t *testing.T) {
	c := controller{}
	a := ps["github.com/anonx/sunplate/internal/programs/generate/handlers/testdata/controllers"].data["App"].Actions[1]
	exp := ", _, _"
	if r := c.IgnoredArgs(&a); r != exp {
		t.Errorf(`Incorrect IgnoreArgs result. Expected "%s", got "%s".`, exp, r)
	}
}

func assertDeepEqualController(c1, c2 *controller) {
	if c1 == nil || c2 == nil {
		if c1 != c2 {
			log.Error.Panicf(
				"One of the controllers is equal to nil while another is not: %#v != %#v.", c1, c2,
			)
		}
		return
	}
	if c1.File != c2.File {
		log.Error.Panicf("Controllers are from different files: %s != %s.", c1.File, c2.File)
	}
	if !r.DeepEqual(c1.Comments, c2.Comments) {
		log.Error.Panicf("Controllers have different comments: %#v != %#v.", c1.Comments, c2.Comments)
	}
	if !r.DeepEqual(c1.Parents, c2.Parents) {
		log.Error.Panicf("Controllers have different parent controllers: %#v != %#v.", c1.Parents, c2.Parents)
	}
	if err := reflect.AssertEqualFuncs(c1.Actions, c2.Actions); err != nil {
		log.Error.Panic(err)
	}
	if err := reflect.AssertEqualFunc(c1.After, c2.After); err != nil {
		log.Error.Panic(err)
	}
	if err := reflect.AssertEqualFunc(c1.Before, c2.Before); err != nil {
		log.Error.Panic(err)
	}
	if err := reflect.AssertEqualFunc(c1.Finally, c2.Finally); err != nil {
		log.Error.Panic(err)
	}
}

func assertDeepEqualControllers(cs1, cs2 controllers) {
	if len(cs1.data) != len(cs2.data) {
		log.Error.Panicf(
			"controllers maps %#v and %#v have different length: %d != %d",
			cs1.data, cs2.data, len(cs1.data), len(cs2.data),
		)
	}
	for i := range cs1.data {
		c1 := cs1.data[i]
		c2 := cs2.data[i]
		assertDeepEqualController(&c1, &c2)
	}
}

func assertDeepEqualPkgs(ps1, ps2 packages) {
	if len(ps1) != len(ps2) {
		log.Error.Panicf(
			"packages maps %#v and %#v have different length: %d != %d",
			ps1, ps2, len(ps1), len(ps2),
		)
	}
	for i := range ps1 {
		assertDeepEqualControllers(ps1[i], ps2[i])
	}
}

var ps = packages{
	"github.com/anonx/sunplate/internal/programs/generate/handlers/testdata/controllers": controllers{
		data: map[string]controller{
			"Controller": controller{
				After: &reflect.Func{
					Comments: []string{"// After is a magic method that is executed after every request."},
					File:     "init.go",
					Name:     "After",
					Params: []reflect.Arg{
						{
							Name: "name",
							Type: &reflect.Type{
								Name: "string",
							},
						},
					},
					Recv: &reflect.Arg{
						Name: "c",
						Type: &reflect.Type{
							Name: "Controller",
							Star: true,
						},
					},
					Results: []reflect.Arg{
						{
							Type: &reflect.Type{
								Name:    "Handler",
								Package: "h",
							},
						},
					},
				},
				Before: &reflect.Func{
					Comments: []string{"// Before is a magic method that is executed before every request."},
					File:     "init.go",
					Name:     "Before",
					Params: []reflect.Arg{
						{
							Name: "uid",
							Type: &reflect.Type{
								Name: "string",
							},
						},
					},
					Recv: &reflect.Arg{
						Name: "c",
						Type: &reflect.Type{
							Name: "Controller",
							Star: true,
						},
					},
					Results: []reflect.Arg{
						{
							Type: &reflect.Type{
								Name:    "Handler",
								Package: "h",
							},
						},
					},
				},
				Finally: &reflect.Func{
					Comments: []string{"// Finally is a magic method that is executed after every request."},
					File:     "app.go",
					Name:     "Finally",
					Params: []reflect.Arg{
						{
							Name: "w",
							Type: &reflect.Type{
								Package: "http",
								Name:    "ResponseWriter",
							},
						},
						{
							Name: "r",
							Type: &reflect.Type{
								Package: "http",
								Name:    "Request",
								Star:    true,
							},
						},
					},
					Recv: &reflect.Arg{
						Name: "c",
						Type: &reflect.Type{
							Name: "Controller",
							Star: true,
						},
					},
					Results: []reflect.Arg{
						{
							Type: &reflect.Type{
								Name: "bool",
							},
						},
					},
				},
				Initially: &reflect.Func{
					Comments: []string{"// Initially is a magic method that is executed before every request."},
					File:     "app.go",
					Name:     "Initially",
					Params: []reflect.Arg{
						{
							Name: "w",
							Type: &reflect.Type{
								Package: "http",
								Name:    "ResponseWriter",
							},
						},
						{
							Name: "r",
							Type: &reflect.Type{
								Package: "http",
								Name:    "Request",
								Star:    true,
							},
						},
					},
					Recv: &reflect.Arg{
						Name: "c",
						Type: &reflect.Type{
							Name: "Controller",
							Star: true,
						},
					},
					Results: []reflect.Arg{
						{
							Type: &reflect.Type{
								Name: "bool",
							},
						},
					},
				},

				Comments: []string{
					"// Controller is a struct that should be embedded into every controller",
					"// of your app to make methods provided by middleware controllers available.",
				},
				File: "init.go",
				Parents: []parent{
					{
						Import: "github.com/anonx/sunplate/internal/programs/generate/handlers/testdata/controllers/subpackage",
						Name:   "Controller",
					},
				},
			},
			"App": controller{
				Actions: []reflect.Func{
					{
						Comments: []string{"// Index is a sample action."},
						File:     "init.go",
						Name:     "Index",
						Params: []reflect.Arg{
							{
								Name: "page",
								Type: &reflect.Type{
									Name: "int",
								},
							},
						},
						Recv: &reflect.Arg{
							Name: "c",
							Type: &reflect.Type{
								Name: "App",
								Star: true,
							},
						},
						Results: []reflect.Arg{
							{
								Type: &reflect.Type{
									Name:    "Handler",
									Package: "h",
								},
							},
						},
					},
					{
						Comments: []string{"// HelloWorld is a sample action."},
						File:     "app.go",
						Name:     "HelloWorld",
						Params: []reflect.Arg{
							{
								Name: "page",
								Type: &reflect.Type{
									Name: "int",
								},
							},
						},
						Recv: &reflect.Arg{
							Name: "c",
							Type: &reflect.Type{
								Name: "App",
							},
						},
						Results: []reflect.Arg{
							{
								Type: &reflect.Type{
									Name:    "Handler",
									Package: "http",
								},
							},
							{
								Type: &reflect.Type{
									Name: "bool",
								},
							},
							{
								Type: &reflect.Type{
									Name: "error",
								},
							},
						},
					},
				},
				After:   &reflect.Func{},
				Before:  &reflect.Func{},
				Finally: &reflect.Func{},

				Comments: []string{
					"// App is a sample controller.",
				},
				File: "app.go",
				Parents: []parent{
					{
						Import: "github.com/anonx/sunplate/internal/programs/generate/handlers/testdata/controllers",
						Name:   "Controller",
					},
				},
			},
		},
	},
	"github.com/anonx/sunplate/internal/programs/generate/handlers/testdata/controllers/subpackage": controllers{
		data: map[string]controller{
			"Controller": controller{
				Actions: []reflect.Func{
					{
						Comments: []string{"// Index is a sample action."},
						File:     "app.go",
						Name:     "Index",
						Params: []reflect.Arg{
							{
								Name: "page",
								Type: &reflect.Type{
									Name: "int",
								},
							},
						},
						Recv: &reflect.Arg{
							Name: "c",
							Type: &reflect.Type{
								Name: "App",
								Star: true,
							},
						},
						Results: []reflect.Arg{
							{
								Type: &reflect.Type{
									Name:    "Handler",
									Package: "http",
								},
							},
						},
					},
				},
				After: &reflect.Func{
					Comments: []string{
						"// After is a magic function that is executed after any request.",
					},
					File:   "app.go",
					Name:   "After",
					Params: []reflect.Arg{},
					Recv: &reflect.Arg{
						Name: "c",
						Type: &reflect.Type{
							Name: "Controller",
							Star: true,
						},
					},
					Results: []reflect.Arg{
						{
							Type: &reflect.Type{
								Name:    "Handler",
								Package: "http",
							},
						},
					},
				},
				Before: &reflect.Func{
					Comments: []string{
						"// Before is a magic function that is executed before any request.",
					},
					File:   "app.go",
					Name:   "Before",
					Params: []reflect.Arg{},
					Recv: &reflect.Arg{
						Name: "c",
						Type: &reflect.Type{
							Name: "Controller",
							Star: true,
						},
					},
					Results: []reflect.Arg{
						{
							Type: &reflect.Type{
								Name:    "Handler",
								Package: "http",
							},
						},
					},
				},
				Finally: &reflect.Func{
					Comments: []string{
						"// Finally is a magic function that is executed after any request",
						"// no matter what.",
					},
					File: "app.go",
					Name: "Finally",
					Params: []reflect.Arg{
						{
							Name: "w",
							Type: &reflect.Type{
								Package: "http",
								Name:    "ResponseWriter",
							},
						},
						{
							Name: "r",
							Type: &reflect.Type{
								Package: "http",
								Name:    "Request",
								Star:    true,
							},
						},
					},
					Recv: &reflect.Arg{
						Name: "c",
						Type: &reflect.Type{
							Name: "Controller",
							Star: true,
						},
					},
					Results: []reflect.Arg{
						{
							Type: &reflect.Type{
								Name: "bool",
							},
						},
					},
				},

				Comments: []string{
					"// Controller is some controller.",
				},
				File:    "app.go",
				Parents: []parent{},
			},
		},
	},
}
<|endoftext|>


<commit_before>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build freebsd
// +build 386 amd64

package runtime

import (
	"runtime/internal/atomic"
	"unsafe"
)

const (
	_VDSO_TH_ALGO_X86_TSC  = 1
	_VDSO_TH_ALGO_X86_HPET = 2
)

const (
	_HPET_DEV_MAP_MAX  = 10
	_HPET_MAIN_COUNTER = 0xf0 /* Main counter register */
)

var (
	hpetDevMap  [_HPET_DEV_MAP_MAX]uintptr
	hpetDevPath = [_HPET_DEV_MAP_MAX][11]byte{
		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '0', 0},
		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '1', 0},
		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '2', 0},
		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '3', 0},
		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '4', 0},
		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '5', 0},
		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '6', 0},
		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '7', 0},
		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '8', 0},
		{'/', 'd', 'e', 'v', '/', 'h', 'p', 'e', 't', '9', 0},
	}
)

//go:nosplit
func (th *vdsoTimehands) getTSCTimecounter() uint32 {
	tsc := cputicks()
	if th.x86_shift > 0 {
		tsc >>= th.x86_shift
	}
	return uint32(tsc)
}

//go:nosplit
func (th *vdsoTimehands) getHPETTimecounter() (uint32, bool) {
	idx := int(th.x86_hpet_idx)
	if idx >= len(hpetDevMap) {
		return 0, false
	}

	p := atomic.Loaduintptr(&hpetDevMap[idx])
	if p == 0 {
		fd := open(&hpetDevPath[idx][0], 0 /* O_RDONLY */, 0)
		if fd < 0 {
			atomic.Casuintptr(&hpetDevMap[idx], 0, ^uintptr(0))
			return 0, false
		}

		addr, mmapErr := mmap(nil, physPageSize, _PROT_READ, _MAP_SHARED, fd, 0)
		closefd(fd)
		newP := uintptr(addr)
		if mmapErr != 0 {
			newP = ^uintptr(0)
		}
		if !atomic.Casuintptr(&hpetDevMap[idx], 0, newP) && mmapErr == 0 {
			munmap(addr, physPageSize)
		}
		p = atomic.Loaduintptr(&hpetDevMap[idx])
	}
	if p == ^uintptr(0) {
		return 0, false
	}
	return *(*uint32)(unsafe.Pointer(p + _HPET_MAIN_COUNTER)), true
}

//go:nosplit
func (th *vdsoTimehands) getTimecounter() (uint32, bool) {
	switch th.algo {
	case _VDSO_TH_ALGO_X86_TSC:
		return th.getTSCTimecounter(), true
	case _VDSO_TH_ALGO_X86_HPET:
		return th.getHPETTimecounter()
	default:
		return 0, false
	}
}
<commit_msg>runtime: fast clock_gettime on FreeBSD, always call getHPETTimecounter on systemstack<commit_after>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build freebsd
// +build 386 amd64

package runtime

import (
	"runtime/internal/atomic"
	"unsafe"
)

const (
	_VDSO_TH_ALGO_X86_TSC  = 1
	_VDSO_TH_ALGO_X86_HPET = 2
)

const (
	_HPET_DEV_MAP_MAX  = 10
	_HPET_MAIN_COUNTER = 0xf0 /* Main counter register */

	hpetDevPath = "/dev/hpetX\x00"
)

var hpetDevMap [_HPET_DEV_MAP_MAX]uintptr

//go:nosplit
func (th *vdsoTimehands) getTSCTimecounter() uint32 {
	tsc := cputicks()
	if th.x86_shift > 0 {
		tsc >>= th.x86_shift
	}
	return uint32(tsc)
}

//go:systemstack
func (th *vdsoTimehands) getHPETTimecounter() (uint32, bool) {
	const digits = "0123456789"

	idx := int(th.x86_hpet_idx)
	if idx >= len(hpetDevMap) {
		return 0, false
	}

	p := atomic.Loaduintptr(&hpetDevMap[idx])
	if p == 0 {
		var devPath [len(hpetDevPath)]byte
		copy(devPath[:], hpetDevPath)
		devPath[9] = digits[idx]

		fd := open(&devPath[0], 0 /* O_RDONLY */, 0)
		if fd < 0 {
			atomic.Casuintptr(&hpetDevMap[idx], 0, ^uintptr(0))
			return 0, false
		}

		addr, mmapErr := mmap(nil, physPageSize, _PROT_READ, _MAP_SHARED, fd, 0)
		closefd(fd)
		newP := uintptr(addr)
		if mmapErr != 0 {
			newP = ^uintptr(0)
		}
		if !atomic.Casuintptr(&hpetDevMap[idx], 0, newP) && mmapErr == 0 {
			munmap(addr, physPageSize)
		}
		p = atomic.Loaduintptr(&hpetDevMap[idx])
	}
	if p == ^uintptr(0) {
		return 0, false
	}
	return *(*uint32)(unsafe.Pointer(p + _HPET_MAIN_COUNTER)), true
}

//go:nosplit
func (th *vdsoTimehands) getTimecounter() (uint32, bool) {
	switch th.algo {
	case _VDSO_TH_ALGO_X86_TSC:
		return th.getTSCTimecounter(), true
	case _VDSO_TH_ALGO_X86_HPET:
		var (
			tc uint32
			ok bool
		)
		systemstack(func() {
			tc, ok = th.getHPETTimecounter()
		})
		return tc, ok
	default:
		return 0, false
	}
}
<|endoftext|>


<commit_before>package scraper

import (
	"io"
	"net/url"

	"github.com/PuerkitoBio/goquery"
	"github.com/uber-go/zap"
)

func (s *Scraper) fixFileReferences(buf io.Reader) (string, error) {
	g, err := goquery.NewDocumentFromReader(buf)
	if err != nil {
		return "", err
	}

	g.Find("a").Each(func(_ int, selection *goquery.Selection) {
		s.fixQuerySelection("href", selection)
	})

	g.Find("img").Each(func(_ int, selection *goquery.Selection) {
		s.fixQuerySelection("src", selection)
	})

	g.Find("script").Each(func(_ int, selection *goquery.Selection) {
		s.fixQuerySelection("src", selection)
	})

	return g.Html()
}

func (s *Scraper) fixQuerySelection(attribute string, selection *goquery.Selection) {
	src, ok := selection.Attr(attribute)
	if !ok {
		return
	}

	ur, err := url.Parse(src)
	if err != nil {
		return
	}
	if ur.Host != s.URL.Host {
		return
	}

	refRes := s.URL.ResolveReference(ur)
	refRes.Scheme = "" // remove http/https
	refRes.Host = ""   // remove host
	refStr := refRes.String()

	if refStr == "" {
		refStr = "/" // website root
	} else if refStr[0] == '/' {
		refStr = refStr[1:]
	}
	if refStr[len(refStr)-1] == '/' {
		refStr += "index.html"
	}

	s.log.Debug("HTML Element fixed", zap.Stringer("URL", refRes), zap.String("Fixed", refStr))
	selection.SetAttr(attribute, refStr)
}
<commit_msg>Fixed link handling<commit_after>package scraper

import (
	"io"
	"net/url"

	"github.com/PuerkitoBio/goquery"
	"github.com/uber-go/zap"
)

func (s *Scraper) fixFileReferences(buf io.Reader) (string, error) {
	g, err := goquery.NewDocumentFromReader(buf)
	if err != nil {
		return "", err
	}

	g.Find("a").Each(func(_ int, selection *goquery.Selection) {
		s.fixQuerySelection("href", selection)
	})

	g.Find("link").Each(func(_ int, selection *goquery.Selection) {
		s.fixQuerySelection("href", selection)
	})

	g.Find("img").Each(func(_ int, selection *goquery.Selection) {
		s.fixQuerySelection("src", selection)
	})

	g.Find("script").Each(func(_ int, selection *goquery.Selection) {
		s.fixQuerySelection("src", selection)
	})

	return g.Html()
}

func (s *Scraper) fixQuerySelection(attribute string, selection *goquery.Selection) {
	src, ok := selection.Attr(attribute)
	if !ok {
		return
	}

	ur, err := url.Parse(src)
	if err != nil {
		return
	}
	if ur.Host != s.URL.Host {
		return
	}

	refRes := s.URL.ResolveReference(ur)
	refRes.Scheme = "" // remove http/https
	refRes.Host = ""   // remove host
	refStr := refRes.String()

	if refStr == "" {
		refStr = "/" // website root
	} else if len(refStr) > 1 && refStr[0] == '/' {
		refStr = refStr[1:]
	}
	if refStr[len(refStr)-1] == '/' {
		refStr += "index.html"
	}

	s.log.Debug("HTML Element fixed", zap.Stringer("URL", refRes), zap.String("Fixed", refStr))
	selection.SetAttr(attribute, refStr)
}
<|endoftext|>


<commit_before>package server

import (
	"context"
	"fmt"
	"path"
	"time"

	log "github.com/sirupsen/logrus"
	"k8s.io/kubernetes/pkg/api"
	labelspkg "k8s.io/kubernetes/pkg/labels"
	kube_watch "k8s.io/kubernetes/pkg/watch"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	col "github.com/pachyderm/pachyderm/src/server/pkg/collection"
	"github.com/pachyderm/pachyderm/src/server/pkg/dlock"
	"github.com/pachyderm/pachyderm/src/server/pkg/util"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"
	ppsserver "github.com/pachyderm/pachyderm/src/server/pps"
)

const (
	masterLockPath = "_master_lock"
)

var (
	failures = map[string]bool{
		"InvalidImageName": true,
		"ErrImagePull":     true,
	}
)

// The master process is responsible for creating/deleting workers as
// pipelines are created/removed.
func (a *apiServer) master() {
	masterLock := dlock.NewDLock(a.etcdClient, path.Join(a.etcdPrefix, masterLockPath))
	backoff.RetryNotify(func() error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		ctx, err := masterLock.Lock(ctx)
		if err != nil {
			return err
		}
		defer masterLock.Unlock(ctx)

		log.Infof("Launching PPS master process")

		pipelineWatcher, err := a.pipelines.ReadOnly(ctx).WatchWithPrev()
		if err != nil {
			return fmt.Errorf("error creating watch: %+v", err)
		}
		defer pipelineWatcher.Close()

		kubePipelineWatch, err := a.kubeClient.Pods(a.namespace).Watch(api.ListOptions{
			LabelSelector: labelspkg.SelectorFromSet(map[string]string{
				"component": "worker",
			}),
			Watch: true,
		})
		if err != nil {
			return err
		}
		defer kubePipelineWatch.Stop()

		for {
			select {
			case event := <-pipelineWatcher.Watch():
				if event.Err != nil {
					return fmt.Errorf("event err: %+v", event.Err)
				}
				switch event.Type {
				case watch.EventPut:
					var pipelineName string
					var pipelineInfo pps.PipelineInfo
					if err := event.Unmarshal(&pipelineName, &pipelineInfo); err != nil {
						return err
					}

					if pipelineInfo.Salt == "" {
						if _, err := col.NewSTM(ctx, a.etcdClient, func(stm col.STM) error {
							pipelines := a.pipelines.ReadWrite(stm)
							newPipelineInfo := new(pps.PipelineInfo)
							if err := pipelines.Get(pipelineInfo.Pipeline.Name, newPipelineInfo); err != nil {
								return fmt.Errorf("error getting pipeline %s: %+v", pipelineName, err)
							}
							if newPipelineInfo.Salt == "" {
								newPipelineInfo.Salt = uuid.NewWithoutDashes()
							}
							pipelines.Put(pipelineInfo.Pipeline.Name, newPipelineInfo)
							return nil
						}); err != nil {
							return err
						}
					}

					var prevPipelineInfo pps.PipelineInfo
					if event.PrevKey != nil {
						if err := event.UnmarshalPrev(&pipelineName, &prevPipelineInfo); err != nil {
							return err
						}
					}

					// If the pipeline has been stopped, delete workers
					if pipelineStateToStopped(pipelineInfo.State) {
						log.Infof("master: deleting workers for pipeline %s", pipelineInfo.Pipeline.Name)
						if err := a.deleteWorkersForPipeline(&pipelineInfo); err != nil {
							return err
						}
					}

					// If the pipeline has been restarted, create workers
					if !pipelineStateToStopped(pipelineInfo.State) && event.PrevKey != nil && pipelineStateToStopped(prevPipelineInfo.State) {
						if err := a.upsertWorkersForPipeline(&pipelineInfo); err != nil {
							if err := a.setPipelineFailure(ctx, pipelineInfo.Pipeline.Name, fmt.Sprintf("failed to create workers: %s", err.Error())); err != nil {
								return err
							}
							continue
						}
					}

					// If the pipeline has been updated, create new workers
					if pipelineInfo.Version > prevPipelineInfo.Version {
						log.Infof("master: creating/updating workers for pipeline %s", pipelineInfo.Pipeline.Name)
						if event.PrevKey != nil {
							if err := a.deleteWorkersForPipeline(&prevPipelineInfo); err != nil {
								return err
							}
						}
						if err := a.upsertWorkersForPipeline(&pipelineInfo); err != nil {
							if err := a.setPipelineFailure(ctx, pipelineInfo.Pipeline.Name, fmt.Sprintf("failed to create workers: %s", err.Error())); err != nil {
								return err
							}
							continue
						}
					}
				case watch.EventDelete:
					var pipelineName string
					var pipelineInfo pps.PipelineInfo
					if err := event.UnmarshalPrev(&pipelineName, &pipelineInfo); err != nil {
						return err
					}
					if err := a.deleteWorkersForPipeline(&pipelineInfo); err != nil {
						return err
					}
				}
			case event := <-kubePipelineWatch.ResultChan():
				if event.Type == kube_watch.Error {
					kubePipelineWatch, err = a.kubeClient.Pods(a.namespace).Watch(api.ListOptions{
						LabelSelector: labelspkg.SelectorFromSet(map[string]string{
							"component": "worker",
						}),
						Watch: true,
					})
					if err != nil {
						return err
					}
					defer kubePipelineWatch.Stop()
				}
				pod, ok := event.Object.(*api.Pod)
				if !ok {
					continue
				}
				if pod.Status.Phase == api.PodFailed {
					log.Errorf("pod failed because: %s", pod.Status.Message)
				}
				for _, status := range pod.Status.ContainerStatuses {
					if status.Name == "user" && status.State.Waiting != nil && failures[status.State.Waiting.Reason] {
						if err := a.setPipelineFailure(ctx, pod.ObjectMeta.Annotations["pipelineName"], status.State.Waiting.Message); err != nil {
							return err
						}
					}
				}
			}
		}
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		log.Errorf("master: error running the master process: %v; retrying in %v", err, d)
		return nil
	})
}

func (a *apiServer) setPipelineFailure(ctx context.Context, pipelineName string, reason string) error {
	// Set pipeline state to failure
	_, err := col.NewSTM(ctx, a.etcdClient, func(stm col.STM) error {
		pipelines := a.pipelines.ReadWrite(stm)
		pipelineInfo := new(pps.PipelineInfo)
		if err := pipelines.Get(pipelineName, pipelineInfo); err != nil {
			return err
		}
		pipelineInfo.State = pps.PipelineState_PIPELINE_FAILURE
		pipelineInfo.Reason = reason
		pipelines.Put(pipelineName, pipelineInfo)
		return nil
	})
	return err
}

func (a *apiServer) upsertWorkersForPipeline(pipelineInfo *pps.PipelineInfo) error {
	var errCount int
	return backoff.RetryNotify(func() error {
		parallelism, err := ppsserver.GetExpectedNumWorkers(a.kubeClient, pipelineInfo.ParallelismSpec)
		if err != nil {
			return err
		}
		var resources *api.ResourceList
		if pipelineInfo.ResourceSpec != nil {
			resources, err = util.GetResourceListFromPipeline(pipelineInfo)
			if err != nil {
				return err
			}
		}

		// Retrieve the current state of the RC.  If the RC is scaled down,
		// we want to ensure that it remains scaled down.
		rc := a.kubeClient.ReplicationControllers(a.namespace)
		workerRc, err := rc.Get(ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version))
		if err == nil {
			if (workerRc.Spec.Template.Spec.Containers[0].Resources.Requests == nil) && workerRc.Spec.Replicas == 1 {
				parallelism = 1
				resources = nil
			}
		}

		options := a.getWorkerOptions(
			pipelineInfo.Pipeline.Name,
			ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version),
			int32(parallelism),
			resources,
			pipelineInfo.Transform,
			pipelineInfo.CacheSize)
		// Set the pipeline name env
		options.workerEnv = append(options.workerEnv, api.EnvVar{
			Name:  client.PPSPipelineNameEnv,
			Value: pipelineInfo.Pipeline.Name,
		})
		return a.createWorkerRc(options)
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		errCount++
		if errCount >= 3 {
			return err
		}
		log.Errorf("error creating workers for pipeline %v: %v; retrying in %v", pipelineInfo.Pipeline.Name, err, d)
		return nil
	})
}

func (a *apiServer) deleteWorkersForPipeline(pipelineInfo *pps.PipelineInfo) error {
	rcName := ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
	if err := a.kubeClient.Services(a.namespace).Delete(rcName); err != nil {
		if !isNotFoundErr(err) {
			return err
		}
	}
	falseVal := false
	deleteOptions := &api.DeleteOptions{
		OrphanDependents: &falseVal,
	}
	if err := a.kubeClient.ReplicationControllers(a.namespace).Delete(rcName, deleteOptions); err != nil {
		if !isNotFoundErr(err) {
			return err
		}
	}
	return nil
}
<commit_msg>Fix a weird bug with k8s watches.<commit_after>package server

import (
	"context"
	"fmt"
	"path"
	"time"

	log "github.com/sirupsen/logrus"
	"k8s.io/kubernetes/pkg/api"
	labelspkg "k8s.io/kubernetes/pkg/labels"
	kube_watch "k8s.io/kubernetes/pkg/watch"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	col "github.com/pachyderm/pachyderm/src/server/pkg/collection"
	"github.com/pachyderm/pachyderm/src/server/pkg/dlock"
	"github.com/pachyderm/pachyderm/src/server/pkg/util"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"
	ppsserver "github.com/pachyderm/pachyderm/src/server/pps"
)

const (
	masterLockPath = "_master_lock"
)

var (
	failures = map[string]bool{
		"InvalidImageName": true,
		"ErrImagePull":     true,
	}
)

// The master process is responsible for creating/deleting workers as
// pipelines are created/removed.
func (a *apiServer) master() {
	masterLock := dlock.NewDLock(a.etcdClient, path.Join(a.etcdPrefix, masterLockPath))
	backoff.RetryNotify(func() error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		ctx, err := masterLock.Lock(ctx)
		if err != nil {
			return err
		}
		defer masterLock.Unlock(ctx)

		log.Infof("Launching PPS master process")

		pipelineWatcher, err := a.pipelines.ReadOnly(ctx).WatchWithPrev()
		if err != nil {
			return fmt.Errorf("error creating watch: %+v", err)
		}
		defer pipelineWatcher.Close()

		kubePipelineWatch, err := a.kubeClient.Pods(a.namespace).Watch(api.ListOptions{
			LabelSelector: labelspkg.SelectorFromSet(map[string]string{
				"component": "worker",
			}),
			Watch: true,
		})
		if err != nil {
			return err
		}
		defer kubePipelineWatch.Stop()

		for {
			select {
			case event := <-pipelineWatcher.Watch():
				if event.Err != nil {
					return fmt.Errorf("event err: %+v", event.Err)
				}
				switch event.Type {
				case watch.EventPut:
					var pipelineName string
					var pipelineInfo pps.PipelineInfo
					if err := event.Unmarshal(&pipelineName, &pipelineInfo); err != nil {
						return err
					}

					if pipelineInfo.Salt == "" {
						if _, err := col.NewSTM(ctx, a.etcdClient, func(stm col.STM) error {
							pipelines := a.pipelines.ReadWrite(stm)
							newPipelineInfo := new(pps.PipelineInfo)
							if err := pipelines.Get(pipelineInfo.Pipeline.Name, newPipelineInfo); err != nil {
								return fmt.Errorf("error getting pipeline %s: %+v", pipelineName, err)
							}
							if newPipelineInfo.Salt == "" {
								newPipelineInfo.Salt = uuid.NewWithoutDashes()
							}
							pipelines.Put(pipelineInfo.Pipeline.Name, newPipelineInfo)
							return nil
						}); err != nil {
							return err
						}
					}

					var prevPipelineInfo pps.PipelineInfo
					if event.PrevKey != nil {
						if err := event.UnmarshalPrev(&pipelineName, &prevPipelineInfo); err != nil {
							return err
						}
					}

					// If the pipeline has been stopped, delete workers
					if pipelineStateToStopped(pipelineInfo.State) {
						log.Infof("master: deleting workers for pipeline %s", pipelineInfo.Pipeline.Name)
						if err := a.deleteWorkersForPipeline(&pipelineInfo); err != nil {
							return err
						}
					}

					// If the pipeline has been restarted, create workers
					if !pipelineStateToStopped(pipelineInfo.State) && event.PrevKey != nil && pipelineStateToStopped(prevPipelineInfo.State) {
						if err := a.upsertWorkersForPipeline(&pipelineInfo); err != nil {
							if err := a.setPipelineFailure(ctx, pipelineInfo.Pipeline.Name, fmt.Sprintf("failed to create workers: %s", err.Error())); err != nil {
								return err
							}
							continue
						}
					}

					// If the pipeline has been updated, create new workers
					if pipelineInfo.Version > prevPipelineInfo.Version {
						log.Infof("master: creating/updating workers for pipeline %s", pipelineInfo.Pipeline.Name)
						if event.PrevKey != nil {
							if err := a.deleteWorkersForPipeline(&prevPipelineInfo); err != nil {
								return err
							}
						}
						if err := a.upsertWorkersForPipeline(&pipelineInfo); err != nil {
							if err := a.setPipelineFailure(ctx, pipelineInfo.Pipeline.Name, fmt.Sprintf("failed to create workers: %s", err.Error())); err != nil {
								return err
							}
							continue
						}
					}
				case watch.EventDelete:
					var pipelineName string
					var pipelineInfo pps.PipelineInfo
					if err := event.UnmarshalPrev(&pipelineName, &pipelineInfo); err != nil {
						return err
					}
					if err := a.deleteWorkersForPipeline(&pipelineInfo); err != nil {
						return err
					}
				}
			case event := <-kubePipelineWatch.ResultChan():
				// if we get an error we restart the watch, k8s watches seem to
				// sometimes get stuck in a loop returning events with Type =
				// "" we treat these as errors since otherwise we get an
				// endless stream of them and can't do anything.
				if event.Type == kube_watch.Error || event.Type == "" {
					kubePipelineWatch.Stop()
					kubePipelineWatch, err = a.kubeClient.Pods(a.namespace).Watch(api.ListOptions{
						LabelSelector: labelspkg.SelectorFromSet(map[string]string{
							"component": "worker",
						}),
						Watch: true,
					})
					if err != nil {
						return err
					}
					defer kubePipelineWatch.Stop()
				}
				pod, ok := event.Object.(*api.Pod)
				if !ok {
					continue
				}
				if pod.Status.Phase == api.PodFailed {
					log.Errorf("pod failed because: %s", pod.Status.Message)
				}
				for _, status := range pod.Status.ContainerStatuses {
					if status.Name == "user" && status.State.Waiting != nil && failures[status.State.Waiting.Reason] {
						if err := a.setPipelineFailure(ctx, pod.ObjectMeta.Annotations["pipelineName"], status.State.Waiting.Message); err != nil {
							return err
						}
					}
				}
			}
		}
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		log.Errorf("master: error running the master process: %v; retrying in %v", err, d)
		return nil
	})
}

func (a *apiServer) setPipelineFailure(ctx context.Context, pipelineName string, reason string) error {
	// Set pipeline state to failure
	_, err := col.NewSTM(ctx, a.etcdClient, func(stm col.STM) error {
		pipelines := a.pipelines.ReadWrite(stm)
		pipelineInfo := new(pps.PipelineInfo)
		if err := pipelines.Get(pipelineName, pipelineInfo); err != nil {
			return err
		}
		pipelineInfo.State = pps.PipelineState_PIPELINE_FAILURE
		pipelineInfo.Reason = reason
		pipelines.Put(pipelineName, pipelineInfo)
		return nil
	})
	return err
}

func (a *apiServer) upsertWorkersForPipeline(pipelineInfo *pps.PipelineInfo) error {
	var errCount int
	return backoff.RetryNotify(func() error {
		parallelism, err := ppsserver.GetExpectedNumWorkers(a.kubeClient, pipelineInfo.ParallelismSpec)
		if err != nil {
			return err
		}
		var resources *api.ResourceList
		if pipelineInfo.ResourceSpec != nil {
			resources, err = util.GetResourceListFromPipeline(pipelineInfo)
			if err != nil {
				return err
			}
		}

		// Retrieve the current state of the RC.  If the RC is scaled down,
		// we want to ensure that it remains scaled down.
		rc := a.kubeClient.ReplicationControllers(a.namespace)
		workerRc, err := rc.Get(ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version))
		if err == nil {
			if (workerRc.Spec.Template.Spec.Containers[0].Resources.Requests == nil) && workerRc.Spec.Replicas == 1 {
				parallelism = 1
				resources = nil
			}
		}

		options := a.getWorkerOptions(
			pipelineInfo.Pipeline.Name,
			ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version),
			int32(parallelism),
			resources,
			pipelineInfo.Transform,
			pipelineInfo.CacheSize)
		// Set the pipeline name env
		options.workerEnv = append(options.workerEnv, api.EnvVar{
			Name:  client.PPSPipelineNameEnv,
			Value: pipelineInfo.Pipeline.Name,
		})
		return a.createWorkerRc(options)
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		errCount++
		if errCount >= 3 {
			return err
		}
		log.Errorf("error creating workers for pipeline %v: %v; retrying in %v", pipelineInfo.Pipeline.Name, err, d)
		return nil
	})
}

func (a *apiServer) deleteWorkersForPipeline(pipelineInfo *pps.PipelineInfo) error {
	rcName := ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
	if err := a.kubeClient.Services(a.namespace).Delete(rcName); err != nil {
		if !isNotFoundErr(err) {
			return err
		}
	}
	falseVal := false
	deleteOptions := &api.DeleteOptions{
		OrphanDependents: &falseVal,
	}
	if err := a.kubeClient.ReplicationControllers(a.namespace).Delete(rcName, deleteOptions); err != nil {
		if !isNotFoundErr(err) {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
 Generate input test scripts to verify index behaviour.

 Current functionality:
 - random width (1 to maxIdxColumn), random type schemas
 - prng values for all columns
 - "is", "ls", "if" commands on unique indexes

 todo: The abbrev (int, bint, sint...) handling is weird and
 repeated in the voltIdxTypes table as well as the Abbrev()
 functions. Oddly, NValues don't store any value, they only
 exist as types. Probably they should store their string
 representations.
*/
package main

import (
	"fmt"
	"rand"
	"strings"
)

var (
	// Types known to test tool that are valid index column types
	voltIdxTypes = [...]string{
		"int",
		"bint",
		"sint",
		"tint",
		"dec",
		"str4",
		"str128"}

	// Maximum number of indexed columns
	maxIdxColumns = 50
)

// Interface all values implement
type NValue interface {
	Create() string
	Abbrev() string
}

// INTEGER
type IntValue struct{}

func (nv IntValue) Create() string {
	val := rand.Int() * sign()
	return fmt.Sprint(val)
}

func (nv IntValue) Abbrev() string {
	return "int"
}

// BIGINT
type BintValue struct{}

func (nv BintValue) Create() string {
	val := rand.Int63()
	val = val * int64(sign())
	return fmt.Sprint(val)
}

func (nv BintValue) Abbrev() string {
	return "bint"
}

// SMALLINT
type SintValue struct{}

func (nv SintValue) Create() string {
	val := rand.Intn(0xFFFF) * sign()
	return fmt.Sprint(val)
}

func (nv SintValue) Abbrev() string {
	return "sint"
}

// TINYINT
type TintValue struct{}

func (nv TintValue) Create() string {
	val := rand.Intn(0xFF) * sign()
	return fmt.Sprint(val)
}

func (nv TintValue) Abbrev() string {
	return "tint"
}

// DECIMAL
type DecValue struct{}

func (nv DecValue) Create() string {
	vals := make([]string, 3)
	lhs := rand.Int63()
	lhs = lhs * int64(sign())
	vals[0] = fmt.Sprint(lhs)
	vals[1] = "."
	vals[2] = fmt.Sprint(rand.Int())
	return strings.Join(vals, "")
}

func (nv DecValue) Abbrev() string {
	return "dec"
}

// VARCHAR(4|128)
type StrValue struct {
	size int
}

func (nv StrValue) Create() string {
	if nv.size == 128 {
		substrs := [...]string{"ning", "izzy", "ariel", "nick",
			"mazur", "ryan", "hugg", "yankeesfan", "volt", "runs",
			"with", "scissors", "blue", "awesome", "weak", "sauce",
			"chicken", "strength", "vikram", "bobbi", "jarr", "bungee",
			"banjo", "arrow", "trinity", "coffee", "pvc"}
		cnt := (rand.Int() % 10) + 1
		vals := make([]string, cnt)

		for i := 0; i < cnt; i++ {
			vals[i] = substrs[rand.Int()%len(substrs)]
		}
		return strings.Join(vals, "")
	} else if nv.size == 4 {
		substrs := [...]string{"a", "b", "c", "d"}
		cnt := 4
		vals := make([]string, cnt)

		for i := 0; i < cnt; i++ {
			vals[i] = substrs[rand.Int()%len(substrs)]
		}
		return strings.Join(vals, "")
	}
	panic("Invalid string size.")
}

func (nv StrValue) Abbrev() string {
	return fmt.Sprintf("%s%d", "str", nv.size)
}

// Utility Functions

func nvalueFactory(abbrev string) NValue {
	switch abbrev {
	case "int":
		return &IntValue{}
	case "bint":
		return &BintValue{}
	case "sint":
		return &SintValue{}
	case "tint":
		return &TintValue{}
	case "dec":
		return &DecValue{}
	case "str4":
		return &StrValue{4}
	case "str128":
		return &StrValue{128}
	}
	panic(abbrev)
}

func printSliceAsList(slice []string) {
	for i := 0; i < len(slice); i++ {
		fmt.Printf(slice[i])
		if i == len(slice)-1 {
			fmt.Printf("\n")
		} else {
			fmt.Printf(",")
		}
	}
}

func createSchema() ([]NValue, []string) {
	schema := make([]NValue, (rand.Int()%maxIdxColumns)+1)
	abbrev := make([]string, len(schema))
	for ii := 0; ii < len(schema); ii++ {
		schemaType := voltIdxTypes[rand.Int()%len(voltIdxTypes)]
		schema[ii] = nvalueFactory(schemaType)
		abbrev[ii] = schemaType
	}
	return schema, abbrev
}

func createTuple(schema []NValue) []string {
	tuple := make([]string, len(schema))
	for ii := 0; ii < len(schema); ii++ {
		tuple[ii] = schema[ii].Create()
	}
	return tuple
}

// Create a reasonable serialization given a schema a tuple
func tupleKey(schema []NValue, tuple []string) string {
	parts := make([]string, len(schema) * 2)
	pi := 0
	for si := 0; si < len(schema); si++ {
		parts[pi], parts[pi+1] = schema[si].Abbrev(), tuple[si]
		pi = pi + 2
	}
	return strings.Join(parts, ":")
}

func sign() int {
	sign := rand.Int() % 2
	if sign == 0 {
		return 1
	}
	return -1
}

/*
 * Commands known to the test harness:
 *
 * is : insert success
 * if : insert failure
 * ls : lookup success
 * lf : lookup failure
 * us : update success
 * uf : update failure
 */

func generateUniqueGenericTree(testrun int) {
	// map where the keys are strings and the values are slices of strings
	tuples := make(map[string][]string)
	schema, abbrev := createSchema()

	// print the test introduction
	fmt.Printf("begin TestUniqueGenericTree_%d UniqueGenericTree ", testrun)
	printSliceAsList(abbrev)

	// create tuples. push them into a map to uniqify them.
	// (this is a unique index test)
	for cmd := 0; cmd < 10; cmd++ {
		tuple := createTuple(schema)
		tuplekey := tupleKey(schema, tuple)
		tuples[tuplekey] = tuple
	}

	// is commands
	for _, v := range tuples {
		fmt.Printf("is ")
		printSliceAsList(v)
	}

	// ls commands
	for _, v := range tuples {
		fmt.Printf("ls ")
		printSliceAsList(v)
	}

	// if commands (reinserting existing keys should fail)
	for _, v := range tuples {
		fmt.Printf("if ")
		printSliceAsList(v)
	}

	// print the test conclusion
	fmt.Println("exec")
}

func main() {
	fmt.Printf("# File generated by index_script_gen.go\n")
	var i int = 0
	for {
		generateUniqueGenericTree(i)
		i++
	}
	fmt.Println("done")
}
<commit_msg>Now a reasonable baseline test for unique tree index.<commit_after>/*
 Generate input test scripts to verify index behaviour.

 Current functionality:
 - random width (1 to maxIdxColumn), random type schemas
 - prng values for all columns
 - "is", "ls", "if" commands on unique indexes

 todo: The abbrev (int, bint, sint...) handling is weird and
 repeated in the voltIdxTypes table as well as the Abbrev()
 functions. Oddly, NValues don't store any value, they only
 exist as types. Probably they should store their string
 representations.
*/
package main

import (
	"fmt"
	"rand"
	"strings"
)

var (
	// Types known to test tool that are valid index column types
	voltIdxTypes = [...]string{
		"int",
		"bint",
		"sint",
		"tint",
		"dec",
		"str4",
		"str128"}

	// Maximum number of indexed columns
	maxIdxColumns = 50
)

// Interface all values implement
type NValue interface {
	Create() string
	Abbrev() string
}

// INTEGER
type IntValue struct{}

func (nv IntValue) Create() string {
	val := rand.Int() * sign()
	return fmt.Sprint(val)
}

func (nv IntValue) Abbrev() string {
	return "int"
}

// BIGINT
type BintValue struct{}

func (nv BintValue) Create() string {
	val := rand.Int63()
	val = val * int64(sign())
	return fmt.Sprint(val)
}

func (nv BintValue) Abbrev() string {
	return "bint"
}

// SMALLINT
type SintValue struct{}

func (nv SintValue) Create() string {
	val := rand.Intn(0x7FFF)
	signstr := ""
	if val != 0 && rand.Intn(10) < 5 {
		signstr = "-"
	}
	return fmt.Sprintf("%s%d", signstr, val)
}

func (nv SintValue) Abbrev() string {
	return "sint"
}

// TINYINT
type TintValue struct{}

func (nv TintValue) Create() string {
	val := rand.Intn(0x7F)
	signstr := ""
	if val != 0 && rand.Intn(10) < 5 {
		signstr = "-"
	}
	return fmt.Sprintf("%s%d", signstr, val)
}

func (nv TintValue) Abbrev() string {
	return "tint"
}

// DECIMAL
type DecValue struct{}

func (nv DecValue) Create() string {
	vals := make([]string, 3)
	lhs := rand.Int() * sign()
	vals[0] = fmt.Sprint(lhs)
	vals[1] = "."
	vals[2] = fmt.Sprint(rand.Intn(99999999))
	return strings.Join(vals, "")
}

func (nv DecValue) Abbrev() string {
	return "dec"
}

// VARCHAR(4|128)
type StrValue struct {
	size int
}

func (nv StrValue) Create() string {
	if nv.size == 128 {
		substrs := [...]string{"ning", "izzy", "ariel", "nick",
			"mazur", "ryan", "hugg", "yankeesfan", "volt", "runs",
			"with", "scissors", "blue", "awesome", "weak", "sauce",
			"chicken", "strength", "vikram", "bobbi", "jarr", "bungee",
			"banjo", "arrow", "trinity", "coffee", "pvc"}
		cnt := (rand.Int() % 10) + 1
		vals := make([]string, cnt)

		for i := 0; i < cnt; i++ {
			vals[i] = substrs[rand.Int()%len(substrs)]
		}
		return strings.Join(vals, "")
	} else if nv.size == 4 {
		substrs := [...]string{"a", "b", "c", "d"}
		cnt := 4
		vals := make([]string, cnt)

		for i := 0; i < cnt; i++ {
			vals[i] = substrs[rand.Int()%len(substrs)]
		}
		return strings.Join(vals, "")
	}
	panic("Invalid string size.")
}

func (nv StrValue) Abbrev() string {
	return fmt.Sprintf("%s%d", "str", nv.size)
}

// Utility Functions

func nvalueFactory(abbrev string) NValue {
	switch abbrev {
	case "int":
		return &IntValue{}
	case "bint":
		return &BintValue{}
	case "sint":
		return &SintValue{}
	case "tint":
		return &TintValue{}
	case "dec":
		return &DecValue{}
	case "str4":
		return &StrValue{4}
	case "str128":
		return &StrValue{128}
	}
	panic(abbrev)
}

func printSliceAsList(slice []string) {
	for i := 0; i < len(slice); i++ {
		fmt.Printf(slice[i])
		if i == len(slice)-1 {
			fmt.Printf("\n")
		} else {
			fmt.Printf(",")
		}
	}
}

func createSchema() ([]NValue, []string) {
	schema := make([]NValue, (rand.Int()%maxIdxColumns)+1)
	abbrev := make([]string, len(schema))
	for ii := 0; ii < len(schema); ii++ {
		schemaType := voltIdxTypes[rand.Int()%len(voltIdxTypes)]
		schema[ii] = nvalueFactory(schemaType)
		abbrev[ii] = schemaType
	}
	return schema, abbrev
}

func createTuple(schema []NValue) []string {
	tuple := make([]string, len(schema))
	for ii := 0; ii < len(schema); ii++ {
		tuple[ii] = schema[ii].Create()
	}
	return tuple
}

// Create a reasonable serialization given a schema a tuple
func tupleKey(schema []NValue, tuple []string) string {
	parts := make([]string, len(schema)*2)
	pi := 0
	for si := 0; si < len(schema); si++ {
		parts[pi], parts[pi+1] = schema[si].Abbrev(), tuple[si]
		pi = pi + 2
	}
	return strings.Join(parts, ":")
}

func sign() int {
	sign := rand.Int() % 2
	if sign == 0 {
		return 1
	}
	return -1
}

/*
 * Commands known to the test harness:
 *
 * is : insert success
 * if : insert failure
 * ls : lookup success
 * lf : lookup failure
 * us : update success
 * uf : update failure
 */

func generateUniqueGenericTree(testrun int) {
	// map where the keys are strings and the values are slices of strings
	tuples := make(map[string][]string)
	schema, abbrev := createSchema()

	// print the test introduction
	fmt.Printf("begin TestUniqueGenericTree_%d UniqueGenericTree ", testrun)
	printSliceAsList(abbrev)

	// create tuples. push them into a map to uniqify them.
	// (this is a unique index test)
	for cmd := 0; cmd < 10; cmd++ {
		tuple := createTuple(schema)
		tuplekey := tupleKey(schema, tuple)
		tuples[tuplekey] = tuple
	}

	// is commands
	for _, v := range tuples {
		fmt.Printf("is ")
		printSliceAsList(v)
	}

	// ls commands
	for _, v := range tuples {
		fmt.Printf("ls ")
		printSliceAsList(v)
	}

	// if commands (reinserting existing keys should fail)
	for _, v := range tuples {
		fmt.Printf("if ")
		printSliceAsList(v)
	}

	// ds commands (delete success)
	for _, v := range tuples {
		fmt.Printf("ds ")
		printSliceAsList(v)
	}
	for _, v := range tuples {
		fmt.Printf("df ")
		printSliceAsList(v)
	}
	for _, v := range tuples {
		fmt.Printf("lf ")
		printSliceAsList(v)
	}

	// print the test conclusion
	fmt.Println("exec")
}

func main() {
	fmt.Printf("# File generated by index_script_gen.go\n")
	var i int = 0
	for {
		generateUniqueGenericTree(i)
		i++
	}
	fmt.Println("done")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"

	"gopkg.in/alecthomas/kingpin.v2"

	"github.com/dpecos/dotback/models"
	. "github.com/dpecos/dotback/utils"
)

func ReadConfig(recipe string) []models.Recipe {
	file, err := ioutil.ReadFile(path.Join(HomeDir(), ".dotfiles", "config.json"))
	CheckError("Could not read config.json file", err)

	var config []models.Recipe
	err = json.Unmarshal(file, &config)
	CheckError("Could not parse config.json file", err)

	if recipe != "" {
		for _, r := range config {
			if r.Name == recipe {
				fmt.Printf("Executing only recipe '%s' (skipping the rest)\n", r.Name)
				config = []models.Recipe{r}
			}
		}
	}

	return config
}

func ExecRecipes(config []models.Recipe) {
	for _, recipe := range config {
		recipe.Exec()
	}
}

var (
	app = kingpin.New("dotback", "Handle your dot files like a boss")
	// initialize = app.Command("init", "Use a Git repository to initialize your dotfiles folder")
	pull    = app.Command("pull", "Fetch latest changes from remote dotfiles repository")
	push    = app.Command("push", "Send latest changes to remote dotfiles repository")
	list    = app.Command("list", "List the actions defined in your ~/.dotfiles/config.json")
	install = app.Command("install", "Performs the actions defined in your ~/.dotfiles/config.json")
	recipe  = install.Arg("recipe", "Execute only this recipe").String()
	// add        = app.Command("add", "Creates a new recipe")
	// delete     = app.Command("delete", "Remove a recipe")
)

func main() {

	switch kingpin.MustParse(app.Parse(os.Args[1:])) {
	case "pull":
		path := path.Join(HomeDir(), ".dotfiles")
		err := Execute(fmt.Sprintf("cd %s && git pull", path))
		CheckError("Error updating repository", err)
	case "push":
		path := path.Join(HomeDir(), ".dotfiles")
		err := Execute(fmt.Sprintf("cd %s && git push", path))
		CheckError("Error updating repository", err)
	case "list":
		config := ReadConfig(*recipe)
		for _, recipe := range config {
			fmt.Printf("%s -> \n", recipe.Name)

			for _, action := range recipe.Actions {
				if action.Link != "" {
					fmt.Printf("   Link: %s\n", action.Link)
				}
				if action.Cmd != "" {
					fmt.Printf("   Cmd: %s\n", action.Cmd)
				}
				if action.Git != "" {
					fmt.Printf("   Git: %s\n", action.Git)
				}
			}
		}
	case "install":
		config := ReadConfig(*recipe)
		ExecRecipes(config)
	}

}
<commit_msg>Return empty config if user requested recipe is not found<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"

	"gopkg.in/alecthomas/kingpin.v2"

	"github.com/dpecos/dotback/models"
	. "github.com/dpecos/dotback/utils"
)

func ReadConfig(recipe string) []models.Recipe {
	file, err := ioutil.ReadFile(path.Join(HomeDir(), ".dotfiles", "config.json"))
	CheckError("Could not read config.json file", err)

	var config []models.Recipe
	err = json.Unmarshal(file, &config)
	CheckError("Could not parse config.json file", err)

	if recipe != "" {
		oldConfig := config
		config = nil

		for _, r := range oldConfig {
			if r.Name == recipe {
				fmt.Printf("Executing only recipe '%s' (skipping the rest)\n", r.Name)
				config = []models.Recipe{r}
			}
		}

		if config == nil {
			Error("Recipe %s not found", recipe)
			os.Exit(-1)
		}
	}

	return config
}

func ExecRecipes(config []models.Recipe) {
	for _, recipe := range config {
		recipe.Exec()
	}
}

var (
	app = kingpin.New("dotback", "Handle your dot files like a boss")
	// initialize = app.Command("init", "Use a Git repository to initialize your dotfiles folder")
	pull    = app.Command("pull", "Fetch latest changes from remote dotfiles repository")
	push    = app.Command("push", "Send latest changes to remote dotfiles repository")
	list    = app.Command("list", "List the actions defined in your ~/.dotfiles/config.json")
	install = app.Command("install", "Performs the actions defined in your ~/.dotfiles/config.json")
	recipe  = install.Arg("recipe", "Execute only this recipe").String()
	// add        = app.Command("add", "Creates a new recipe")
	// delete     = app.Command("delete", "Remove a recipe")
)

func main() {

	switch kingpin.MustParse(app.Parse(os.Args[1:])) {
	case "pull":
		path := path.Join(HomeDir(), ".dotfiles")
		err := Execute(fmt.Sprintf("cd %s && git pull", path))
		CheckError("Error updating repository", err)
	case "push":
		path := path.Join(HomeDir(), ".dotfiles")
		err := Execute(fmt.Sprintf("cd %s && git push", path))
		CheckError("Error updating repository", err)
	case "list":
		config := ReadConfig(*recipe)
		for _, recipe := range config {
			fmt.Printf("%s -> \n", recipe.Name)

			for _, action := range recipe.Actions {
				if action.Link != "" {
					fmt.Printf("   Link: %s\n", action.Link)
				}
				if action.Cmd != "" {
					fmt.Printf("   Cmd: %s\n", action.Cmd)
				}
				if action.Git != "" {
					fmt.Printf("   Git: %s\n", action.Git)
				}
			}
		}
	case "install":
		config := ReadConfig(*recipe)
		ExecRecipes(config)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/fatih/color"
	"github.com/urfave/cli"
	"net"
	"os"
	"io"
	"os/signal"
	"syscall"
)

func getLocalAddrs() ([]net.IP, error) {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return nil, err
	}
	var list []net.IP
	for _, addr := range addrs {
		v := addr.(*net.IPNet)
		if v.IP.To4() != nil {
			list = append(list, v.IP)
		}
	}
	return list, nil
}

func fwd(src net.Conn, remote string, proto string) {
	dst, err := net.Dial(proto, remote)
	errHandler(err)
	go func() {
		_, err = io.Copy(src, dst)
		errPrinter(err)
	}()
	go func() {
		_, err = io.Copy(dst, src)
		errPrinter(err)
	}()
}

func errHandler(err error) {
	if err != nil {
		color.Set(color.FgRed)
		fmt.Fprintf(os.Stderr, "[Error] %s\n", err.Error())
		color.Unset()
		os.Exit(1)
	}
}

// TODO: merge error handling functions
func errPrinter(err error) {
	if err != nil {
		color.Set(color.FgRed)
		fmt.Fprintf(os.Stderr, "[Error] %s\n", err.Error())
		color.Unset()
	}
}

func tcpStart(from string, to string) {
	proto := "tcp"

	localAddress, err := net.ResolveTCPAddr(proto, from)
	errHandler(err)

	remoteAddress, err := net.ResolveTCPAddr(proto, to)
	errHandler(err)

	listener, err := net.ListenTCP(proto, localAddress)
	errHandler(err)

	defer listener.Close()

	fmt.Printf("Forwarding %s traffic from '%v' to '%v'\n", proto, localAddress, remoteAddress)
	color.Set(color.FgYellow)
	fmt.Println("<CTRL+C> to exit")
	fmt.Println()
	color.Unset()

	for {
		src, err := listener.Accept()
		errHandler(err)
		fmt.Printf("New connection established from '%v'\n", src.RemoteAddr())
		go fwd(src, to, proto)
	}
}

func udpStart(from string, to string) {
	proto := "udp"

	localAddress, err := net.ResolveUDPAddr(proto, from)
	errHandler(err)

	remoteAddress, err := net.ResolveUDPAddr(proto, to)
	errHandler(err)

	listener, err := net.ListenUDP(proto, localAddress)
	errHandler(err)
	defer listener.Close()

	dst, err := net.DialUDP(proto, nil, remoteAddress)
	errHandler(err)
	defer dst.Close()

	fmt.Printf("Forwarding %s traffic from '%v' to '%v'\n", proto, localAddress, remoteAddress)
	color.Set(color.FgYellow)
	fmt.Println("<CTRL+C> to exit")
	fmt.Println()
	color.Unset()

	buf := make([]byte, 512)
	for {
		rnum, err := listener.Read(buf[0:])
		errHandler(err)

		_, err = dst.Write(buf[:rnum])
		errHandler(err)

		fmt.Printf("%d bytes forwared\n", rnum)
	}
}

func ctrlc() {
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		sig := <-sigs
		color.Set(color.FgGreen)
		fmt.Println("\nExecution stopped by", sig)
		color.Unset()
		os.Exit(0)
	}()
}

func main() {
	app := cli.NewApp()
	app.Name = "fwd"
	app.Version = "0.1.2"
	app.Usage = "The little forwarder that could"
	app.UsageText = "fwd --from localhost:2222 --to 192.168.1.254:22"
	app.Copyright = "MIT License"
	app.Authors = []cli.Author{
		cli.Author{
			Name:  "Joel Bastos",
			Email: "kintoandar@gmail.com",
		},
	}
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "from, f",
			Value:  "127.0.0.1:8000",
			EnvVar: "FWD_FROM",
			Usage:  "source HOST:PORT",
		},
		cli.StringFlag{
			Name:   "to, t",
			EnvVar: "FWD_TO",
			Usage:  "destination HOST:PORT",
		},
		cli.BoolFlag{
			Name:  "list, l",
			Usage: "list local addresses",
		},
		cli.BoolFlag{
			Name:  "udp, u",
			Usage: "enable udp forwarding (tcp by default)",
		},
	}
	app.Action = func(c *cli.Context) error {
		defer color.Unset()
		color.Set(color.FgGreen)
		if c.Bool("list") {
			list, err := getLocalAddrs()
			errHandler(err)
			fmt.Println("Available local addresses:")
			color.Unset()
			for _, ip := range list {
				fmt.Println(ip)
			}
			return nil
		} else if c.String("to") == "" {
			color.Unset()
			cli.ShowAppHelp(c)
			return nil
		} else {
			ctrlc()
			if c.Bool("udp") {
				udpStart(c.String("from"), c.String("to"))

			} else {
				tcpStart(c.String("from"), c.String("to"))
			}
			return nil
		}
	}
	app.Run(os.Args)
}
<commit_msg>Add build information flag<commit_after>package main

import (
	"fmt"
	"github.com/fatih/color"
	"github.com/urfave/cli"
	"net"
	"os"
	"io"
	"os/signal"
	"syscall"
	"runtime"
)

func getLocalAddrs() ([]net.IP, error) {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return nil, err
	}
	var list []net.IP
	for _, addr := range addrs {
		v := addr.(*net.IPNet)
		if v.IP.To4() != nil {
			list = append(list, v.IP)
		}
	}
	return list, nil
}

func fwd(src net.Conn, remote string, proto string) {
	dst, err := net.Dial(proto, remote)
	errHandler(err)
	go func() {
		_, err = io.Copy(src, dst)
		errPrinter(err)
	}()
	go func() {
		_, err = io.Copy(dst, src)
		errPrinter(err)
	}()
}

func errHandler(err error) {
	if err != nil {
		color.Set(color.FgRed)
		fmt.Fprintf(os.Stderr, "[Error] %s\n", err.Error())
		color.Unset()
		os.Exit(1)
	}
}

// TODO: merge error handling functions
func errPrinter(err error) {
	if err != nil {
		color.Set(color.FgRed)
		fmt.Fprintf(os.Stderr, "[Error] %s\n", err.Error())
		color.Unset()
	}
}

func tcpStart(from string, to string) {
	proto := "tcp"

	localAddress, err := net.ResolveTCPAddr(proto, from)
	errHandler(err)

	remoteAddress, err := net.ResolveTCPAddr(proto, to)
	errHandler(err)

	listener, err := net.ListenTCP(proto, localAddress)
	errHandler(err)

	defer listener.Close()

	fmt.Printf("Forwarding %s traffic from '%v' to '%v'\n", proto, localAddress, remoteAddress)
	color.Set(color.FgYellow)
	fmt.Println("<CTRL+C> to exit")
	fmt.Println()
	color.Unset()

	for {
		src, err := listener.Accept()
		errHandler(err)
		fmt.Printf("New connection established from '%v'\n", src.RemoteAddr())
		go fwd(src, to, proto)
	}
}

func udpStart(from string, to string) {
	proto := "udp"

	localAddress, err := net.ResolveUDPAddr(proto, from)
	errHandler(err)

	remoteAddress, err := net.ResolveUDPAddr(proto, to)
	errHandler(err)

	listener, err := net.ListenUDP(proto, localAddress)
	errHandler(err)
	defer listener.Close()

	dst, err := net.DialUDP(proto, nil, remoteAddress)
	errHandler(err)
	defer dst.Close()

	fmt.Printf("Forwarding %s traffic from '%v' to '%v'\n", proto, localAddress, remoteAddress)
	color.Set(color.FgYellow)
	fmt.Println("<CTRL+C> to exit")
	fmt.Println()
	color.Unset()

	buf := make([]byte, 512)
	for {
		rnum, err := listener.Read(buf[0:])
		errHandler(err)

		_, err = dst.Write(buf[:rnum])
		errHandler(err)

		fmt.Printf("%d bytes forwared\n", rnum)
	}
}

func ctrlc() {
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		sig := <-sigs
		color.Set(color.FgGreen)
		fmt.Println("\nExecution stopped by", sig)
		color.Unset()
		os.Exit(0)
	}()
}

func main() {
	app := cli.NewApp()
	app.Name = "fwd"
	app.Version = "0.1.2"
	app.Usage = "The little forwarder that could"
	app.UsageText = "fwd --from localhost:2222 --to 192.168.1.254:22"
	app.Copyright = "MIT License"
	app.Authors = []cli.Author{
		cli.Author{
			Name:  "Joel Bastos",
			Email: "kintoandar@gmail.com",
		},
	}
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "from, f",
			Value:  "127.0.0.1:8000",
			EnvVar: "FWD_FROM",
			Usage:  "source HOST:PORT",
		},
		cli.StringFlag{
			Name:   "to, t",
			EnvVar: "FWD_TO",
			Usage:  "destination HOST:PORT",
		},
		cli.BoolFlag{
			Name:  "list, l",
			Usage: "list local addresses",
		},
		cli.BoolFlag{
			Name:  "udp, u",
			Usage: "enable udp forwarding (tcp by default)",
		},
		cli.BoolFlag{
			Name:  "build, b",
			Usage: "build information",
		},
	}
	app.Action = func(c *cli.Context) error {
		defer color.Unset()
		color.Set(color.FgGreen)
		if c.Bool("list") {
			list, err := getLocalAddrs()
			errHandler(err)
			fmt.Println("Available local addresses:")
			color.Unset()
			for _, ip := range list {
				fmt.Println(ip)
			}
			return nil
		} else if c.Bool("build") {
			fmt.Println("Built with " + runtime.Version() + " for " + runtime.GOOS + "/" + runtime.GOARCH)
			color.Unset()
			return nil

		} else if c.String("to") == "" {
			color.Unset()
			cli.ShowAppHelp(c)
			return nil
		} else {
			ctrlc()
			if c.Bool("udp") {
				udpStart(c.String("from"), c.String("to"))

			} else {
				tcpStart(c.String("from"), c.String("to"))
			}
			return nil
		}
	}
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package service

import (
	"go2o/core/domain/interface/member"
	"go2o/core/service/auto_gen/rpc/ttype"
	"go2o/core/service/rsi"
	"go2o/core/service/thrift"
	"go2o/tests/ti"
	"testing"
)

var _ = ti.Factory.GetAdRepo()

func TestPagingIntegralLog(t *testing.T) {
	params := &ttype.SPagingParams{
		Opt:        nil,
		OrderField: "",
		OrderDesc:  false,
		Begin:      0,
		Over:       10,
	}
	r, _ := rsi.MemberService.PagingAccountLog(thrift.Context, 1, member.AccountWallet, params)
	t.Logf("%#v", r)
}

func TestPagingWalletLog(t *testing.T) {
	memberId := 77153
	params := &ttype.SPagingParams{
		Opt:        nil,
		OrderField: "",
		OrderDesc:  false,
		Begin:      0,
		Over:       10,
	}
	r, _ := rsi.MemberService.PagingAccountLog(thrift.Context, int64(memberId), member.AccountWallet, params)
	t.Logf("%#v", r)
}
<commit_msg>member_service<commit_after>package service

import (
	"go2o/core/domain/interface/member"
	"go2o/core/infrastructure/domain"
	"go2o/core/service/auto_gen/rpc/ttype"
	"go2o/core/service/rsi"
	"go2o/core/service/thrift"
	"go2o/tests/ti"
	"testing"
)

var _ = ti.Factory.GetAdRepo()

func TestPagingIntegralLog(t *testing.T) {
	params := &ttype.SPagingParams{
		Opt:        nil,
		OrderField: "",
		OrderDesc:  false,
		Begin:      0,
		Over:       10,
	}
	r, _ := rsi.MemberService.PagingAccountLog(thrift.Context, 1, member.AccountWallet, params)
	t.Logf("%#v", r)
}

func TestPagingWalletLog(t *testing.T) {
	memberId := 77153
	params := &ttype.SPagingParams{
		Opt:        nil,
		OrderField: "",
		OrderDesc:  false,
		Begin:      0,
		Over:       10,
	}
	r, _ := rsi.MemberService.PagingAccountLog(thrift.Context, int64(memberId), member.AccountWallet, params)
	t.Logf("%#v", r)
}

func TestCheckTradePwd(t *testing.T) {
	memberId := 22149
	pwd := domain.Md5("123456")
	//r2,_ := rsi.MemberService.ModifyTradePwd(thrift.Context,int64(memberId),"",pwd)
	//t.Logf("%#v", r2)

	r, _ := rsi.MemberService.CheckTradePwd(thrift.Context, int64(memberId), pwd)
	t.Logf("%#v", r)
}<|endoftext|>