<commit_before>package main

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/printer"
	"go/token"
	"os"
	"reflect"
	"strconv"
)

type changeType uint8

const (
	changeUnknown changeType = iota
	changeNone
	changeNonBreaking
	changeBreaking
)

func (c changeType) String() string {
	switch c {
	case changeUnknown:
		return "unknowable"
	case changeNone:
		return "no change"
	case changeNonBreaking:
		return "non-breaking change"
	}
	return "breaking change"
}

type operation uint8

const (
	opAdd operation = iota
	opRemove
	opChange
)

func (op operation) String() string {
	switch op {
	case opAdd:
		return "added"
	case opRemove:
		return "removed"
	}
	return "changed"
}

// change is the ast declaration containing the before and after
type change struct {
	id         string
	summary    string
	op         operation
	changeType changeType
	before     ast.Decl
	after      ast.Decl
}

func (c change) String() string {
	fset := token.FileSet{} // only require non-nil fset
	pcfg := printer.Config{Mode: printer.RawFormat, Indent: 1}
	buf := bytes.Buffer{}

	if c.op == opChange {
		fmt.Fprintf(&buf, "%s (%s - %s)\n", c.op, c.changeType, c.summary)
	} else {
		fmt.Fprintln(&buf, c.op)
	}

	if c.before != nil {
		pcfg.Fprint(&buf, &fset, c.before)
		fmt.Fprintln(&buf)
	}
	if c.after != nil {
		pcfg.Fprint(&buf, &fset, c.after)
		fmt.Fprintln(&buf)
	}
	return buf.String()
}

// byID implements sort.Interface for []change based on the id field
type byID []change

func (a byID) Len() int           { return len(a) }
func (a byID) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byID) Less(i, j int) bool { return a[i].id < a[j].id }

// decls is a map of an identifier to actual ast, where the id is a unique
// name to match declarations for before and after
type decls map[string]ast.Decl

func diff(bdecls, adecls decls) []change {
	var changes []change
	for id, decl := range bdecls {
		if _, ok := adecls[id]; !ok {
			// in before, not in after, therefore it was removed
			changes = append(changes, change{id: id, op: opRemove, before: decl})
			continue
		}

		// in before and in after, check if there's a difference
		changeType, summary := compareDecl(bdecls[id], adecls[id])
		if changeType == changeNone || changeType == changeUnknown {
			continue
		}

		changes = append(changes, change{
			id:         id,
			op:         opChange,
			changeType: changeType,
			summary:    summary,
			before:     decl,
			after:      adecls[id]},
		)
	}

	for id, decl := range adecls {
		if _, ok := bdecls[id]; !ok {
			// in after, not in before, therefore it was added
			changes = append(changes, change{id: id, op: opAdd, after: decl})
		}
	}

	return changes
}

// equal compares two declarations and returns true if they do not have
// incompatible changes. For example, comments aren't compared, names of
// arguments aren't compared etc.
func compareDecl(before, after ast.Decl) (changeType, string) {
	// compare types, ignore comments etc, so reflect.DeepEqual isn't good enough

	if reflect.TypeOf(before) != reflect.TypeOf(after) {
		// Declaration type changed, such as GenDecl to FuncDecl (eg var/const to func)
		return changeBreaking, "changed declaration"
	}

	switch b := before.(type) {
	case *ast.GenDecl:
		a := after.(*ast.GenDecl)

		if reflect.TypeOf(b.Specs[0]) != reflect.TypeOf(a.Specs[0]) {
			// Spec changed, such as ValueSpec to TypeSpec (eg var/const to struct)
			return changeBreaking, "changed spec"
		}

		switch bspec := b.Specs[0].(type) {
		case *ast.ValueSpec:
			aspec := a.Specs[0].(*ast.ValueSpec)
			// refactoring opportunity here with equalFieldTypes

			if bspec.Type == nil || aspec.Type == nil {
				// eg: var ErrSomeError = errors.New("Some Error")
				// cannot currently determine the type
				return changeUnknown, "cannot currently determine type"
			}

			if reflect.TypeOf(bspec.Type) != reflect.TypeOf(aspec.Type) {
				// eg change from int to []int
				return changeBreaking, "changed value spec type"
			}

			// var / const
			switch btype := bspec.Type.(type) {
			case *ast.Ident:
				// int/string/etc
				atype := aspec.Type.(*ast.Ident)
				if btype.Name != atype.Name {
					// type changed
					return changeBreaking, "changed type"
				}
			case *ast.ArrayType:
				// slice/array
				atype := aspec.Type.(*ast.ArrayType)
				// compare length
				if !exprEqual(btype.Len, atype.Len) {
					// change of length, or between array and slice
					return changeBreaking, "changed of array's length"
				}
				// compare array's element's type
				if !exprEqual(btype.Elt, atype.Elt) {
					return changeBreaking, "changed of array's element's type"
				}
			case *ast.MapType:
				// map
				atype := aspec.Type.(*ast.MapType)

				if !exprEqual(btype.Key, atype.Key) {
					return changeBreaking, "changed map's key's type"
				}
				if !exprEqual(btype.Value, atype.Value) {
					return changeBreaking, "changed map's value's type"
				}
			default:
				panic(fmt.Errorf("Unknown val spec type: %T", btype))
			}
		case *ast.TypeSpec:
			aspec := a.Specs[0].(*ast.TypeSpec)

			// type struct/interface/aliased

			if reflect.TypeOf(bspec.Type) != reflect.TypeOf(aspec.Type) {
				// Spec change, such as from StructType to InterfaceType or different aliased types
				return changeBreaking, "changed type of value spec"
			}

			switch btype := bspec.Type.(type) {
			case *ast.InterfaceType:
				atype := aspec.Type.(*ast.InterfaceType)

				// interfaces don't care if methods are removed
				added, removed, changed := diffFields(btype.Methods.List, atype.Methods.List)
				if len(added) > 0 {
					// Fields were added
					return changeBreaking, "members added"
				} else if len(changed) > 0 {
					// Fields changed types
					return changeBreaking, "members changed types"
				} else if len(removed) > 0 {
					return changeNonBreaking, "members removed"
				}
			case *ast.StructType:
				atype := aspec.Type.(*ast.StructType)

				// structs don't care if fields were added
				added, removed, changed := diffFields(btype.Fields.List, atype.Fields.List)
				if len(removed) > 0 {
					// Fields were removed
					return changeBreaking, "members removed"
				} else if len(changed) > 0 {
					// Fields changed types
					return changeBreaking, "members changed types"
				} else if len(added) > 0 {
					return changeNonBreaking, "members added"
				}
			case *ast.Ident:
				// alias
				atype := aspec.Type.(*ast.Ident)
				if btype.Name != atype.Name {
					// Alias typing changed underlying types
					return changeBreaking, "alias changed its underlying type"
				}
			}
		}
	case *ast.FuncDecl:
		a := after.(*ast.FuncDecl)
		added, removed, changed := diffFields(b.Type.Params.List, a.Type.Params.List)
		if len(added) > 0 || len(removed) > 0 || len(changed) > 0 {
			return changeBreaking, "parameters types changed"
		}

		if b.Type.Results != nil {
			if a.Type.Results == nil {
				// removed return parameter
				return changeBreaking, "removed return parameter"
			}

			_, removed, changed := diffFields(b.Type.Results.List, a.Type.Results.List)
			// Only check if we're changing/removing return parameters
			if len(removed) > 0 || len(changed) > 0 {
				return changeBreaking, "changed or removed return parameter"
			}
		}
	default:
		panic(fmt.Errorf("Unknown type: %T", before))
	}
	return changeNone, ""
}

func diffFields(before, after []*ast.Field) (added, removed, changed []*ast.Field) {
	// Presort after for quicker matching of fieldname -> type, may not be worthwhile
	AfterMembers := make(map[string]*ast.Field)
	for i, field := range after {
		AfterMembers[fieldKey(field, i)] = field
	}

	for i, bfield := range before {
		bkey := fieldKey(bfield, i)
		if afield, ok := AfterMembers[bkey]; ok {
			if !exprEqual(bfield.Type, afield.Type) {
				// changed
				changed = append(changed, bfield)
			}
			delete(AfterMembers, bkey)
			continue
		}

		// Removed
		removed = append(removed, bfield)
	}

	// What's left in afterMembers has added
	for _, afield := range AfterMembers {
		added = append(added, afield)
	}

	return added, removed, changed
}

// Return an appropriate identifier for a field, if it has an ident (name)
// such as in the case of a struct/interface member, else, use it's provided
// position i, such as the case of a function's parameter or result list
func fieldKey(field *ast.Field, i int) string {
	if len(field.Names) > 0 {
		return field.Names[0].Name
	}
	// No name, probably a function, return position
	return strconv.FormatInt(int64(i), 10)
}

// exprEqual compares two ast.Expr to determine if they are equal
func exprEqual(before, after ast.Expr) bool {
	// For the moment just use typeToString and compare strings
	return typeToString(before) == typeToString(after)
}

// typeToString returns a type, such as ident or function and returns a string
// representation (without superfluous variable names when necessary).
//
// This is designed to make comparisons simpler by not having to handle all
// the various ast permutations, but this is the slowest method and may have
// its own set of undesirable properties (including a performance penalty).
// See the equivalent func equalFieldTypes in b3b41cc470d4258b38372b87f22d87845ecfecb6
// for an example of what it might have been (it was missing some checks though)
func typeToString(ident ast.Expr) string {
	fset := token.FileSet{} // only require non-nil fset
	pcfg := printer.Config{Mode: printer.RawFormat}
	buf := bytes.Buffer{}

	switch v := ident.(type) {
	case *ast.FuncType:
		// strip variable names in functions
		for i := range v.Params.List {
			v.Params.List[i].Names = []*ast.Ident{}
		}
		if v.Results != nil {
			for i := range v.Results.List {
				v.Results.List[i].Names = []*ast.Ident{}
			}
		}
	}
	pcfg.Fprint(&buf, &fset, ident)

	return buf.String()
}

// printast is a debug helper to quickly print the go source of an ast
func printast(ast interface{}) {
	pcfg := printer.Config{Mode: printer.RawFormat}
	pcfg.Fprint(os.Stdout, &token.FileSet{}, ast)
}
<commit_msg>Debug and error handling<commit_after>package main

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/printer"
	"go/token"
	"reflect"
	"strconv"
)

type changeType uint8

const (
	changeUnknown changeType = iota
	changeNone
	changeNonBreaking
	changeBreaking
)

func (c changeType) String() string {
	switch c {
	case changeUnknown:
		return "unknowable"
	case changeNone:
		return "no change"
	case changeNonBreaking:
		return "non-breaking change"
	}
	return "breaking change"
}

type operation uint8

const (
	opAdd operation = iota
	opRemove
	opChange
)

func (op operation) String() string {
	switch op {
	case opAdd:
		return "added"
	case opRemove:
		return "removed"
	}
	return "changed"
}

// change is the ast declaration containing the before and after
type change struct {
	id         string
	summary    string
	op         operation
	changeType changeType
	before     ast.Decl
	after      ast.Decl
}

func (c change) String() string {
	fset := token.FileSet{} // only require non-nil fset
	pcfg := printer.Config{Mode: printer.RawFormat, Indent: 1}
	buf := bytes.Buffer{}

	if c.op == opChange {
		fmt.Fprintf(&buf, "%s (%s - %s)\n", c.op, c.changeType, c.summary)
	} else {
		fmt.Fprintln(&buf, c.op)
	}

	if c.before != nil {
		pcfg.Fprint(&buf, &fset, c.before)
		fmt.Fprintln(&buf)
	}
	if c.after != nil {
		pcfg.Fprint(&buf, &fset, c.after)
		fmt.Fprintln(&buf)
	}
	return buf.String()
}

// byID implements sort.Interface for []change based on the id field
type byID []change

func (a byID) Len() int           { return len(a) }
func (a byID) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byID) Less(i, j int) bool { return a[i].id < a[j].id }

// decls is a map of an identifier to actual ast, where the id is a unique
// name to match declarations for before and after
type decls map[string]ast.Decl

func diff(bdecls, adecls decls) []change {
	var changes []change
	for id, decl := range bdecls {
		if _, ok := adecls[id]; !ok {
			// in before, not in after, therefore it was removed
			changes = append(changes, change{id: id, op: opRemove, before: decl})
			continue
		}

		// in before and in after, check if there's a difference
		changeType, summary := compareDecl(bdecls[id], adecls[id])
		if changeType == changeNone || changeType == changeUnknown {
			continue
		}

		changes = append(changes, change{
			id:         id,
			op:         opChange,
			changeType: changeType,
			summary:    summary,
			before:     decl,
			after:      adecls[id]},
		)
	}

	for id, decl := range adecls {
		if _, ok := bdecls[id]; !ok {
			// in after, not in before, therefore it was added
			changes = append(changes, change{id: id, op: opAdd, after: decl})
		}
	}

	return changes
}

// equal compares two declarations and returns true if they do not have
// incompatible changes. For example, comments aren't compared, names of
// arguments aren't compared etc.
func compareDecl(before, after ast.Decl) (changeType, string) {
	// compare types, ignore comments etc, so reflect.DeepEqual isn't good enough

	if reflect.TypeOf(before) != reflect.TypeOf(after) {
		// Declaration type changed, such as GenDecl to FuncDecl (eg var/const to func)
		return changeBreaking, "changed declaration"
	}

	switch b := before.(type) {
	case *ast.GenDecl:
		a := after.(*ast.GenDecl)

		if reflect.TypeOf(b.Specs[0]) != reflect.TypeOf(a.Specs[0]) {
			// Spec changed, such as ValueSpec to TypeSpec (eg var/const to struct)
			return changeBreaking, "changed spec"
		}

		if len(b.Specs) != 1 {
			// getDecls should've already flattened var/const blocks
			panic("unexpected number of specs")
		}

		switch bspec := b.Specs[0].(type) {
		case *ast.ValueSpec:
			aspec := a.Specs[0].(*ast.ValueSpec)

			if bspec.Type == nil || aspec.Type == nil {
				// eg: var ErrSomeError = errors.New("Some Error")
				// cannot currently determine the type
				return changeUnknown, "cannot currently determine type"
			}

			// TODO perhaps just make this entire thing use
			// exprEqual(bspec.Type, aspect.Type) but we'll lose some details

			if reflect.TypeOf(bspec.Type) != reflect.TypeOf(aspec.Type) {
				// eg change from int to []int
				return changeBreaking, "changed value spec type"
			}

			// var / const
			switch btype := bspec.Type.(type) {
			case *ast.Ident:
				// int/string/etc
				atype := aspec.Type.(*ast.Ident)
				if btype.Name != atype.Name {
					// type changed
					return changeBreaking, "changed type"
				}
			case *ast.ArrayType:
				// slice/array
				atype := aspec.Type.(*ast.ArrayType)
				// compare length
				if !exprEqual(btype.Len, atype.Len) {
					// change of length, or between array and slice
					return changeBreaking, "changed of array's length"
				}
				// compare array's element's type
				if !exprEqual(btype.Elt, atype.Elt) {
					return changeBreaking, "changed of array's element's type"
				}
			case *ast.MapType:
				// map
				atype := aspec.Type.(*ast.MapType)

				if !exprEqual(btype.Key, atype.Key) {
					return changeBreaking, "changed map's key's type"
				}
				if !exprEqual(btype.Value, atype.Value) {
					return changeBreaking, "changed map's value's type"
				}
			default:
				panic(fmt.Errorf("Unknown val spec type: %T, source: %s", btype, astString(btype)))
			}
		case *ast.TypeSpec:
			aspec := a.Specs[0].(*ast.TypeSpec)

			// type struct/interface/aliased

			if reflect.TypeOf(bspec.Type) != reflect.TypeOf(aspec.Type) {
				// Spec change, such as from StructType to InterfaceType or different aliased types
				return changeBreaking, "changed type of value spec"
			}

			switch btype := bspec.Type.(type) {
			case *ast.InterfaceType:
				atype := aspec.Type.(*ast.InterfaceType)

				// interfaces don't care if methods are removed
				added, removed, changed := diffFields(btype.Methods.List, atype.Methods.List)
				if len(added) > 0 {
					// Fields were added
					return changeBreaking, "members added"
				} else if len(changed) > 0 {
					// Fields changed types
					return changeBreaking, "members changed types"
				} else if len(removed) > 0 {
					return changeNonBreaking, "members removed"
				}
			case *ast.StructType:
				atype := aspec.Type.(*ast.StructType)

				// structs don't care if fields were added
				added, removed, changed := diffFields(btype.Fields.List, atype.Fields.List)
				if len(removed) > 0 {
					// Fields were removed
					return changeBreaking, "members removed"
				} else if len(changed) > 0 {
					// Fields changed types
					return changeBreaking, "members changed types"
				} else if len(added) > 0 {
					return changeNonBreaking, "members added"
				}
			case *ast.Ident:
				// alias
				atype := aspec.Type.(*ast.Ident)
				if btype.Name != atype.Name {
					// Alias typing changed underlying types
					return changeBreaking, "alias changed its underlying type"
				}
			}
		}
	case *ast.FuncDecl:
		a := after.(*ast.FuncDecl)
		added, removed, changed := diffFields(b.Type.Params.List, a.Type.Params.List)
		if len(added) > 0 || len(removed) > 0 || len(changed) > 0 {
			return changeBreaking, "parameters types changed"
		}

		if b.Type.Results != nil {
			if a.Type.Results == nil {
				// removed return parameter
				return changeBreaking, "removed return parameter"
			}

			_, removed, changed := diffFields(b.Type.Results.List, a.Type.Results.List)
			// Only check if we're changing/removing return parameters
			if len(removed) > 0 || len(changed) > 0 {
				return changeBreaking, "changed or removed return parameter"
			}
		}
	default:
		panic(fmt.Errorf("Unknown type: %T", before))
	}
	return changeNone, ""
}

func diffFields(before, after []*ast.Field) (added, removed, changed []*ast.Field) {
	// Presort after for quicker matching of fieldname -> type, may not be worthwhile
	AfterMembers := make(map[string]*ast.Field)
	for i, field := range after {
		AfterMembers[fieldKey(field, i)] = field
	}

	for i, bfield := range before {
		bkey := fieldKey(bfield, i)
		if afield, ok := AfterMembers[bkey]; ok {
			if !exprEqual(bfield.Type, afield.Type) {
				// changed
				changed = append(changed, bfield)
			}
			delete(AfterMembers, bkey)
			continue
		}

		// Removed
		removed = append(removed, bfield)
	}

	// What's left in afterMembers has added
	for _, afield := range AfterMembers {
		added = append(added, afield)
	}

	return added, removed, changed
}

// Return an appropriate identifier for a field, if it has an ident (name)
// such as in the case of a struct/interface member, else, use it's provided
// position i, such as the case of a function's parameter or result list
func fieldKey(field *ast.Field, i int) string {
	if len(field.Names) > 0 {
		return field.Names[0].Name
	}
	// No name, probably a function, return position
	return strconv.FormatInt(int64(i), 10)
}

// exprEqual compares two ast.Expr to determine if they are equal
func exprEqual(before, after ast.Expr) bool {
	// For the moment just use typeToString and compare strings
	return typeToString(before) == typeToString(after)
}

// typeToString returns a type, such as ident or function and returns a string
// representation (without superfluous variable names when necessary).
//
// This is designed to make comparisons simpler by not having to handle all
// the various ast permutations, but this is the slowest method and may have
// its own set of undesirable properties (including a performance penalty).
// See the equivalent func equalFieldTypes in b3b41cc470d4258b38372b87f22d87845ecfecb6
// for an example of what it might have been (it was missing some checks though)
func typeToString(ident ast.Expr) string {
	fset := token.FileSet{} // only require non-nil fset
	pcfg := printer.Config{Mode: printer.RawFormat}
	buf := bytes.Buffer{}

	switch v := ident.(type) {
	case *ast.FuncType:
		// strip variable names in functions
		for i := range v.Params.List {
			v.Params.List[i].Names = []*ast.Ident{}
		}
		if v.Results != nil {
			for i := range v.Results.List {
				v.Results.List[i].Names = []*ast.Ident{}
			}
		}
	}
	pcfg.Fprint(&buf, &fset, ident)

	return buf.String()
}

// astString is a debug helper to return the go source of an ast
func astString(ast interface{}) string {
	pcfg := printer.Config{Mode: printer.RawFormat}
	buf := bytes.Buffer{}
	pcfg.Fprint(&buf, &token.FileSet{}, ast)
	return buf.String()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	flags "github.com/jessevdk/go-flags"
	"github.com/labstack/echo"
	"github.com/lestrrat/go-server-starter/listener"
	"github.com/monochromegane/gannoy"
)

type Options struct {
	DataDir           string `short:"d" long:"data-dir" default:"." description:"Specify the directory where the meta files are located."`
	WithServerStarter bool   `short:"s" long:"server-starter" default:"false" description:"Use server-starter listener for server address."`
	ShutDownTimeout   int    `short:"t" long:"timeout" default:"10" description:"Specify the number of seconds for shutdown timeout."`
}

var opts Options

type Feature struct {
	W []float64 `json:"features"`
}

func main() {
	_, err := flags.Parse(&opts)
	if err != nil {
		os.Exit(1)
	}

	files, err := ioutil.ReadDir(opts.DataDir)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	databases := map[string]gannoy.GannoyIndex{}
	for _, file := range files {
		if file.IsDir() || filepath.Ext(file.Name()) != ".meta" {
			continue
		}
		key := strings.TrimSuffix(file.Name(), ".meta")
		gannoy, err := gannoy.NewGannoyIndex(filepath.Join(opts.DataDir, file.Name()), gannoy.Angular{}, gannoy.RandRandom{})
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		databases[key] = gannoy
	}

	e := echo.New()
	e.GET("/search", func(c echo.Context) error {
		database := c.QueryParam("database")
		if _, ok := databases[database]; !ok {
			return c.NoContent(http.StatusNotFound)
		}
		key, err := strconv.Atoi(c.QueryParam("key"))
		if err != nil {
			key = -1
		}
		limit, err := strconv.Atoi(c.QueryParam("limit"))
		if err != nil {
			limit = 10
		}

		gannoy := databases[database]
		r, err := gannoy.GetNnsByKey(key, limit, -1)
		if err != nil || len(r) == 0 {
			return c.NoContent(http.StatusNotFound)
		}

		return c.JSON(http.StatusOK, r)
	})

	e.PUT("/databases/:database/features/:key", func(c echo.Context) error {
		database := c.Param("database")
		if _, ok := databases[database]; !ok {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		key, err := strconv.Atoi(c.Param("key"))
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		feature := new(Feature)
		if err := c.Bind(feature); err != nil {
			return err
		}

		gannoy := databases[database]
		err = gannoy.AddItem(key, feature.W)
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		return c.NoContent(http.StatusOK)
	})

	e.DELETE("/databases/:database/features/:key", func(c echo.Context) error {
		database := c.Param("database")
		if _, ok := databases[database]; !ok {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		key, err := strconv.Atoi(c.Param("key"))
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		gannoy := databases[database]
		err = gannoy.RemoveItem(key)
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}

		return c.NoContent(http.StatusOK)
	})

	address := ":1323"
	sig := os.Interrupt
	if opts.WithServerStarter {
		address = ""
		sig = syscall.SIGTERM
		listeners, err := listener.ListenAll()
		if err != nil && err != listener.ErrNoListeningTarget {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		e.Listener = listeners[0]
	}

	go func() {
		if err := e.Start(address); err != nil {
			e.Logger.Info("shutting down the server")
		}
	}()

	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, sig)
	<-sigCh

	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(opts.ShutDownTimeout)*time.Second)
	defer cancel()
	if err := e.Shutdown(ctx); err != nil {
		e.Logger.Fatal(err)
	}
}
<commit_msg>Added log-dir option for gannoy-server command.<commit_after>package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	flags "github.com/jessevdk/go-flags"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/lestrrat/go-server-starter/listener"
	"github.com/monochromegane/gannoy"
)

type Options struct {
	DataDir           string `short:"d" long:"data-dir" default:"." description:"Specify the directory where the meta files are located."`
	LogDir            string `short:"l" long:"log-dir" default-mask:"os.Stdout" description:"Specify the log output directory."`
	WithServerStarter bool   `short:"s" long:"server-starter" default:"false" description:"Use server-starter listener for server address."`
	ShutDownTimeout   int    `short:"t" long:"timeout" default:"10" description:"Specify the number of seconds for shutdown timeout."`
}

var opts Options

type Feature struct {
	W []float64 `json:"features"`
}

func main() {
	_, err := flags.Parse(&opts)
	if err != nil {
		os.Exit(1)
	}

	e := echo.New()

	log, err := initializeLog(opts.LogDir)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	e.Logger.SetOutput(log)
	e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{Output: log}))

	files, err := ioutil.ReadDir(opts.DataDir)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	databases := map[string]gannoy.GannoyIndex{}
	for _, file := range files {
		if file.IsDir() || filepath.Ext(file.Name()) != ".meta" {
			continue
		}
		key := strings.TrimSuffix(file.Name(), ".meta")
		gannoy, err := gannoy.NewGannoyIndex(filepath.Join(opts.DataDir, file.Name()), gannoy.Angular{}, gannoy.RandRandom{})
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		databases[key] = gannoy
	}

	e.GET("/search", func(c echo.Context) error {
		database := c.QueryParam("database")
		if _, ok := databases[database]; !ok {
			return c.NoContent(http.StatusNotFound)
		}
		key, err := strconv.Atoi(c.QueryParam("key"))
		if err != nil {
			key = -1
		}
		limit, err := strconv.Atoi(c.QueryParam("limit"))
		if err != nil {
			limit = 10
		}

		gannoy := databases[database]
		r, err := gannoy.GetNnsByKey(key, limit, -1)
		if err != nil || len(r) == 0 {
			return c.NoContent(http.StatusNotFound)
		}

		return c.JSON(http.StatusOK, r)
	})

	e.PUT("/databases/:database/features/:key", func(c echo.Context) error {
		database := c.Param("database")
		if _, ok := databases[database]; !ok {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		key, err := strconv.Atoi(c.Param("key"))
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		feature := new(Feature)
		if err := c.Bind(feature); err != nil {
			return err
		}

		gannoy := databases[database]
		err = gannoy.AddItem(key, feature.W)
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		return c.NoContent(http.StatusOK)
	})

	e.DELETE("/databases/:database/features/:key", func(c echo.Context) error {
		database := c.Param("database")
		if _, ok := databases[database]; !ok {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		key, err := strconv.Atoi(c.Param("key"))
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		gannoy := databases[database]
		err = gannoy.RemoveItem(key)
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}

		return c.NoContent(http.StatusOK)
	})

	address := ":1323"
	sig := os.Interrupt
	if opts.WithServerStarter {
		address = ""
		sig = syscall.SIGTERM
		listeners, err := listener.ListenAll()
		if err != nil && err != listener.ErrNoListeningTarget {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		e.Listener = listeners[0]
	}

	go func() {
		if err := e.Start(address); err != nil {
			e.Logger.Info("shutting down the server")
		}
	}()

	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, sig)
	<-sigCh

	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(opts.ShutDownTimeout)*time.Second)
	defer cancel()
	if err := e.Shutdown(ctx); err != nil {
		e.Logger.Fatal(err)
	}
}

func initializeLog(logDir string) (*os.File, error) {
	if logDir == "" {
		return os.Stdout, nil
	}
	if err := os.MkdirAll(logDir, os.ModePerm); err != nil {
		return nil, err
	}
	return os.OpenFile("access.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
}
<|endoftext|>


<commit_before>package command

import (
	"bufio"
	"flag"
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/columnize"
	"github.com/funkygao/gafka/cmd/kateway/structs"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/gofmt"
	"github.com/funkygao/golib/pipestream"
	consulapi "github.com/hashicorp/consul/api"
	"github.com/pmylund/sortutil"
)

type hostLoadInfo struct {
	host            string
	cluster         string
	topicPartitions []structs.TopicPartition
	qps             int64
}

type hostOffsetInfo struct {
	host      string
	brokerIDs map[string]int32                            // cluster:brokerID
	offsetMap map[string]map[structs.TopicPartition]int64 // cluster:tp:offset
}

func (ho hostOffsetInfo) Clusters() []string {
	var r []string
	for cluster, _ := range ho.offsetMap {
		r = append(r, cluster)
	}
	return r
}

func (ho hostOffsetInfo) Total() (t int64) {
	for _, tps := range ho.offsetMap {
		for _, qps := range tps {
			t += qps
		}
	}

	return
}

func (ho hostOffsetInfo) ClusterPartitions(cluster string) int {
	return len(ho.offsetMap[cluster])
}

func (ho hostOffsetInfo) ClusterTotal(cluster string) (t int64) {
	for _, qps := range ho.offsetMap[cluster] {
		t += qps
	}

	return
}

type Balance struct {
	Ui  cli.Ui
	Cmd string

	zone, cluster     string
	interval          time.Duration
	detailMode        bool
	host              string
	atLeastTps        int64
	hideZeroClusetr   bool
	skipKafkaInternal bool

	loadAvgMap   map[string]float64
	loadAvgReady chan struct{}

	offsets     map[string]int64 // host => offset sum TODO
	lastOffsets map[string]int64

	allHostsTps map[string]hostOffsetInfo

	hostOffsetCh chan map[string]hostOffsetInfo // key is host
	signalsCh    map[string]chan struct{}
}

func (this *Balance) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("balance", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.BoolVar(&this.hideZeroClusetr, "nozero", true, "")
	cmdFlags.DurationVar(&this.interval, "i", time.Second*5, "")
	cmdFlags.StringVar(&this.host, "host", "", "")
	cmdFlags.BoolVar(&this.skipKafkaInternal, "skipk", true, "")
	cmdFlags.Int64Var(&this.atLeastTps, "over", 0, "")
	cmdFlags.BoolVar(&this.detailMode, "l", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	this.loadAvgReady = make(chan struct{})
	this.loadAvgMap = make(map[string]float64)

	this.signalsCh = make(map[string]chan struct{})
	this.hostOffsetCh = make(chan map[string]hostOffsetInfo)

	this.allHostsTps = make(map[string]hostOffsetInfo)
	this.offsets = make(map[string]int64)
	this.lastOffsets = make(map[string]int64)

	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		if !patternMatched(zkcluster.Name(), this.cluster) {
			return
		}

		this.signalsCh[zkcluster.Name()] = make(chan struct{})

		go this.clusterTopProducers(zkcluster)
	})

	this.drawBalance()

	return
}

func (this *Balance) startAll() {
	for _, ch := range this.signalsCh {
		ch <- struct{}{}
	}
}

func (this *Balance) collectAll(seq int) {
	for _, _ = range this.signalsCh {
		offsets := <-this.hostOffsetCh
		if seq == 0 {
			// record into allHostsTps
			for host, offsetInfo := range offsets {
				if _, present := this.allHostsTps[host]; !present {
					this.allHostsTps[host] = hostOffsetInfo{
						host:      host,
						brokerIDs: make(map[string]int32),
						offsetMap: make(map[string]map[structs.TopicPartition]int64),
					}
				}

				for cluster, tps := range offsetInfo.offsetMap {
					if _, present := this.allHostsTps[host].offsetMap[cluster]; !present {
						this.allHostsTps[host].offsetMap[cluster] = make(map[structs.TopicPartition]int64)
					}

					for tp, off := range tps {
						// 1st loop, offset
						this.allHostsTps[host].offsetMap[cluster][tp] = off
						this.allHostsTps[host].brokerIDs[cluster] = offsetInfo.brokerIDs[cluster]
					}
				}
			}
		} else {
			for host, offsetInfo := range offsets {
				for cluster, tps := range offsetInfo.offsetMap {
					for tp, off := range tps {
						// 2nd loop, qps
						// FIXME hard coding
						this.allHostsTps[host].offsetMap[cluster][tp] = (off - this.allHostsTps[host].offsetMap[cluster][tp]) / int64(this.interval.Seconds())
					}
				}
			}
		}
	}

}

func (this *Balance) drawBalance() {
	go this.fetchLoadAvg()

	for i := 0; i < 2; i++ {
		this.startAll()
		time.Sleep(this.interval)
		this.collectAll(i)
	}

	type hostTps struct {
		host string
		tps  int64
	}
	var sortedHosts []hostTps
	for host, info := range this.allHostsTps {
		sortedHosts = append(sortedHosts, hostTps{host, info.Total()})
	}
	sortutil.AscByField(sortedHosts, "tps")

	var hosts []string
	for _, h := range sortedHosts {
		if this.atLeastTps > 0 && h.tps < this.atLeastTps {
			continue
		}

		hosts = append(hosts, h.host)
	}

	<-this.loadAvgReady

	if !this.detailMode {
		this.drawSummary(hosts)
		return
	}

	this.drawDetail(hosts)
}

func (this *Balance) drawDetail(sortedHosts []string) {
	type hostSummary struct {
		cluster  string
		tp       structs.TopicPartition
		qps      int64
		brokerID int32
	}

	for _, host := range sortedHosts {
		var summary []hostSummary
		offsetInfo := this.allHostsTps[host]

		for cluster, tps := range offsetInfo.offsetMap {
			for tp, qps := range tps {
				summary = append(summary, hostSummary{cluster, tp, qps, offsetInfo.brokerIDs[cluster]})
			}
		}

		sortutil.DescByField(summary, "qps")

		this.Ui.Output(color.Green("%16s %8s %+v", host, gofmt.Comma(offsetInfo.Total()), offsetInfo.Clusters()))
		for _, sum := range summary {
			if sum.qps < 100 {
				continue
			}

			this.Ui.Output(fmt.Sprintf("    %30s %2d %8s %s#%d", sum.cluster, sum.brokerID,
				gofmt.Comma(sum.qps), sum.tp.Topic, sum.tp.PartitionID))
		}
	}

}

type clusterQps struct {
	cluster    string
	qps        int64
	partitions int
}

func (c clusterQps) String() string {
	partitions := fmt.Sprintf("%d", c.partitions)
	if c.qps < 1000 {
		return fmt.Sprintf("%s#%s/%d", c.cluster, partitions, c.qps)
	} else if c.qps < 5000 {
		return fmt.Sprintf("%s#%s/%s", c.cluster, partitions, color.Yellow("%d", c.qps))
	} else if c.qps < 10000 {
		return fmt.Sprintf("%s#%s/%s", c.cluster, partitions, color.Magenta("%d", c.qps))
	}

	return fmt.Sprintf("%s/%s/%s", c.cluster, partitions, color.Red("%d", c.qps))
}

func (this *Balance) drawSummary(sortedHosts []string) {
	type hintInfo struct {
		cluster     string
		host        string
		brokerID    int32
		clusterQps  int64
		clustersQps int64 // all clusters on this host qps sum up
	}

	var hints []hintInfo
	var hostHasClusters = make(map[string]int)

	lines := []string{"Broker|Load1m|P|TPS|Cluster/OPS"}

	var totalTps int64
	var totalPartitions int
	for _, host := range sortedHosts {
		hostPartitions := 0
		offsetInfo := this.allHostsTps[host]
		var clusters []clusterQps
		for cluster, _ := range offsetInfo.offsetMap {
			clusterTps := offsetInfo.ClusterTotal(cluster)
			clusterPartitions := offsetInfo.ClusterPartitions(cluster)
			hostPartitions += clusterPartitions
			totalTps += clusterTps
			totalPartitions += clusterPartitions

			hostHasClusters[host]++
			hints = append(hints, hintInfo{
				cluster:     cluster,
				host:        host,
				brokerID:    offsetInfo.brokerIDs[cluster],
				clusterQps:  clusterTps,
				clustersQps: offsetInfo.Total(),
			})

			if this.hideZeroClusetr && clusterTps == 0 {
				continue
			}

			clusters = append(clusters, clusterQps{cluster, clusterTps, clusterPartitions})
		}

		sortutil.AscByField(clusters, "cluster")

		lines = append(lines, fmt.Sprintf("%s|%5.1f|%d|%s|%+v",
			host, this.loadAvgMap[host], hostPartitions,
			gofmt.Comma(offsetInfo.Total()), clusters))
	}
	this.Ui.Output(columnize.SimpleFormat(lines))
	this.Ui.Output(fmt.Sprintf("-Total- Hosts:%d Partitions:%d Tps:%s",
		len(sortedHosts), totalPartitions, gofmt.Comma(totalTps)))

	// give the rebalance hints by cluster
	this.Ui.Output("")
	sortutil.DescByField(hints, "clustersQps")
	lastHost := ""
	var suspectedHosts []string
	for _, h := range hints {
		if hostHasClusters[h.host] < 2 {
			// already dedicated broker for the cluster
			continue
		}

		if h.clustersQps < 1500 {
			// we don't care about non-busy broker
			continue
		}

		if h.host == lastHost {
			continue
		}

		lastHost = h.host

		suspectedHosts = append(suspectedHosts, h.host)
	}
	this.Ui.Outputf("suspected brokers: %+v", suspectedHosts)
}

func (this *Balance) fetchLoadAvg() {
	defer close(this.loadAvgReady)

	// get members host ip
	cf := consulapi.DefaultConfig()
	client, _ := consulapi.NewClient(cf)
	members, _ := client.Agent().Members(false)

	nodeHostMap := make(map[string]string, len(members))
	for _, member := range members {
		nodeHostMap[member.Name] = member.Addr
	}

	cmd := pipestream.New("consul", "exec", "uptime", "|", "grep", "load")
	cmd.Open()
	if cmd.Reader() == nil {
		return
	}
	defer cmd.Close()

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		fields := strings.Fields(line)
		node := fields[0]
		parts := strings.Split(line, "load average:")
		if len(parts) < 2 {
			continue
		}
		if strings.HasSuffix(node, ":") {
			node = strings.TrimRight(node, ":")
		}

		load1m, err := ctx.ExtractLoadAvg1m(line)
		if err != nil {
			continue
		}

		host := nodeHostMap[node]
		this.loadAvgMap[host] = load1m
	}

}

func (this *Balance) clusterTopProducers(zkcluster *zk.ZkCluster) {
	kfk, err := sarama.NewClient(zkcluster.BrokerList(), sarama.NewConfig())
	if err != nil {
		return
	}
	defer kfk.Close()

	for {
		hostOffsets := make(map[string]hostOffsetInfo)

		topics, err := kfk.Topics()
		swallow(err)

		<-this.signalsCh[zkcluster.Name()]

		for _, topic := range topics {
			if topic == "__consumer_offsets" {
				// skip kafka intennal topics
				continue
			}

			partions, err := kfk.WritablePartitions(topic)
			swallow(err)
			for _, partitionID := range partions {
				leader, err := kfk.Leader(topic, partitionID)
				swallow(err)

				latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest)
				swallow(err)

				host, _, err := net.SplitHostPort(leader.Addr())
				swallow(err)

				if !patternMatched(host, this.host) {
					continue
				}

				if _, present := hostOffsets[host]; !present {
					hostOffsets[host] = hostOffsetInfo{
						host:      host,
						offsetMap: make(map[string]map[structs.TopicPartition]int64),
						brokerIDs: make(map[string]int32),
					}
				}
				if _, present := hostOffsets[host].offsetMap[zkcluster.Name()]; !present {
					hostOffsets[host].offsetMap[zkcluster.Name()] = make(map[structs.TopicPartition]int64)
					hostOffsets[host].brokerIDs[zkcluster.Name()] = leader.ID()
				}

				tp := structs.TopicPartition{Topic: topic, PartitionID: partitionID}
				hostOffsets[host].offsetMap[zkcluster.Name()][tp] = latestOffset
			}
		}

		this.hostOffsetCh <- hostOffsets

		kfk.RefreshMetadata(topics...)
	}
}

func (*Balance) Synopsis() string {
	return "Balance topics distribution according to load instead of count"
}

func (this *Balance) Help() string {
	help := fmt.Sprintf(`
Usage: %s balance [options]

    %s

Options:

    -z zone
      Default %s

    -c cluster pattern

    -host broker ip

    -l
      Use a long listing format.

    -over number
      Only display brokers whose TPS over the number.

    -nozero
      Hide 0 OPS clusters. True by default.

    -skipk
      Skip kafka internal topic: __consumer_offsets. True by default.

`, this.Cmd, this.Synopsis(), ctx.ZkDefaultZone())
	return strings.TrimSpace(help)
}
<commit_msg>enhancement of the suspect broker: more obvious recognization<commit_after>package command

import (
	"bufio"
	"flag"
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/columnize"
	"github.com/funkygao/gafka/cmd/kateway/structs"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/gofmt"
	"github.com/funkygao/golib/pipestream"
	consulapi "github.com/hashicorp/consul/api"
	"github.com/pmylund/sortutil"
)

type hostLoadInfo struct {
	host            string
	cluster         string
	topicPartitions []structs.TopicPartition
	qps             int64
}

type hostOffsetInfo struct {
	host      string
	brokerIDs map[string]int32                            // cluster:brokerID
	offsetMap map[string]map[structs.TopicPartition]int64 // cluster:tp:offset
}

func (ho hostOffsetInfo) Clusters() []string {
	var r []string
	for cluster, _ := range ho.offsetMap {
		r = append(r, cluster)
	}
	return r
}

func (ho hostOffsetInfo) MightProblematic() bool {
	if len(ho.offsetMap) < 2 {
		return false
	}
	bigClusters := 0
	for cluster, _ := range ho.offsetMap {
		if ho.ClusterTotal(cluster) > 1000 {
			bigClusters++
		}
	}
	return bigClusters > 1
}

func (ho hostOffsetInfo) Total() (t int64) {
	for _, tps := range ho.offsetMap {
		for _, qps := range tps {
			t += qps
		}
	}

	return
}

func (ho hostOffsetInfo) ClusterPartitions(cluster string) int {
	return len(ho.offsetMap[cluster])
}

func (ho hostOffsetInfo) ClusterTotal(cluster string) (t int64) {
	for _, qps := range ho.offsetMap[cluster] {
		t += qps
	}

	return
}

type Balance struct {
	Ui  cli.Ui
	Cmd string

	zone, cluster     string
	interval          time.Duration
	detailMode        bool
	host              string
	atLeastTps        int64
	hideZeroClusetr   bool
	skipKafkaInternal bool

	loadAvgMap   map[string]float64
	loadAvgReady chan struct{}

	offsets     map[string]int64 // host => offset sum TODO
	lastOffsets map[string]int64

	allHostsTps map[string]hostOffsetInfo

	hostOffsetCh chan map[string]hostOffsetInfo // key is host
	signalsCh    map[string]chan struct{}
}

func (this *Balance) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("balance", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.BoolVar(&this.hideZeroClusetr, "nozero", false, "")
	cmdFlags.DurationVar(&this.interval, "i", time.Second*5, "")
	cmdFlags.StringVar(&this.host, "host", "", "")
	cmdFlags.BoolVar(&this.skipKafkaInternal, "skipk", true, "")
	cmdFlags.Int64Var(&this.atLeastTps, "over", 0, "")
	cmdFlags.BoolVar(&this.detailMode, "l", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	this.loadAvgReady = make(chan struct{})
	this.loadAvgMap = make(map[string]float64)

	this.signalsCh = make(map[string]chan struct{})
	this.hostOffsetCh = make(chan map[string]hostOffsetInfo)

	this.allHostsTps = make(map[string]hostOffsetInfo)
	this.offsets = make(map[string]int64)
	this.lastOffsets = make(map[string]int64)

	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		if !patternMatched(zkcluster.Name(), this.cluster) {
			return
		}

		this.signalsCh[zkcluster.Name()] = make(chan struct{})

		go this.clusterTopProducers(zkcluster)
	})

	this.drawBalance()

	return
}

func (this *Balance) startAll() {
	for _, ch := range this.signalsCh {
		ch <- struct{}{}
	}
}

func (this *Balance) collectAll(seq int) {
	for _, _ = range this.signalsCh {
		offsets := <-this.hostOffsetCh
		if seq == 0 {
			// record into allHostsTps
			for host, offsetInfo := range offsets {
				if _, present := this.allHostsTps[host]; !present {
					this.allHostsTps[host] = hostOffsetInfo{
						host:      host,
						brokerIDs: make(map[string]int32),
						offsetMap: make(map[string]map[structs.TopicPartition]int64),
					}
				}

				for cluster, tps := range offsetInfo.offsetMap {
					if _, present := this.allHostsTps[host].offsetMap[cluster]; !present {
						this.allHostsTps[host].offsetMap[cluster] = make(map[structs.TopicPartition]int64)
					}

					for tp, off := range tps {
						// 1st loop, offset
						this.allHostsTps[host].offsetMap[cluster][tp] = off
						this.allHostsTps[host].brokerIDs[cluster] = offsetInfo.brokerIDs[cluster]
					}
				}
			}
		} else {
			for host, offsetInfo := range offsets {
				for cluster, tps := range offsetInfo.offsetMap {
					for tp, off := range tps {
						// 2nd loop, qps
						// FIXME hard coding
						this.allHostsTps[host].offsetMap[cluster][tp] = (off - this.allHostsTps[host].offsetMap[cluster][tp]) / int64(this.interval.Seconds())
					}
				}
			}
		}
	}

}

func (this *Balance) drawBalance() {
	go this.fetchLoadAvg()

	for i := 0; i < 2; i++ {
		this.startAll()
		time.Sleep(this.interval)
		this.collectAll(i)
	}

	type hostTps struct {
		host string
		tps  int64
	}
	var sortedHosts []hostTps
	for host, info := range this.allHostsTps {
		sortedHosts = append(sortedHosts, hostTps{host, info.Total()})
	}
	sortutil.AscByField(sortedHosts, "tps")

	var hosts []string
	for _, h := range sortedHosts {
		if this.atLeastTps > 0 && h.tps < this.atLeastTps {
			continue
		}

		hosts = append(hosts, h.host)
	}

	<-this.loadAvgReady

	if !this.detailMode {
		this.drawSummary(hosts)
		return
	}

	this.drawDetail(hosts)
}

func (this *Balance) drawDetail(sortedHosts []string) {
	type hostSummary struct {
		cluster  string
		tp       structs.TopicPartition
		qps      int64
		brokerID int32
	}

	for _, host := range sortedHosts {
		var summary []hostSummary
		offsetInfo := this.allHostsTps[host]

		for cluster, tps := range offsetInfo.offsetMap {
			for tp, qps := range tps {
				summary = append(summary, hostSummary{cluster, tp, qps, offsetInfo.brokerIDs[cluster]})
			}
		}

		sortutil.DescByField(summary, "qps")

		this.Ui.Output(color.Green("%16s %8s %+v", host, gofmt.Comma(offsetInfo.Total()), offsetInfo.Clusters()))
		for _, sum := range summary {
			if sum.qps < 100 {
				continue
			}

			this.Ui.Output(fmt.Sprintf("    %30s %2d %8s %s#%d", sum.cluster, sum.brokerID,
				gofmt.Comma(sum.qps), sum.tp.Topic, sum.tp.PartitionID))
		}
	}

}

type clusterQps struct {
	cluster    string
	qps        int64
	partitions int
}

func (c clusterQps) String() string {
	partitions := fmt.Sprintf("%d", c.partitions)
	if c.qps < 1000 {
		return fmt.Sprintf("%s#%s/%d", c.cluster, partitions, c.qps)
	} else if c.qps < 5000 {
		return fmt.Sprintf("%s#%s/%s", c.cluster, partitions, color.Yellow("%d", c.qps))
	} else if c.qps < 10000 {
		return fmt.Sprintf("%s#%s/%s", c.cluster, partitions, color.Magenta("%d", c.qps))
	}

	return fmt.Sprintf("%s/%s/%s", c.cluster, partitions, color.Red("%d", c.qps))
}

func (this *Balance) drawSummary(sortedHosts []string) {
	lines := []string{"Broker|Load1m|P|TPS|Cluster/OPS"}
	var totalTps int64
	var totalPartitions int
	for _, host := range sortedHosts {
		hostPartitions := 0
		offsetInfo := this.allHostsTps[host]
		var clusters []clusterQps
		for cluster, _ := range offsetInfo.offsetMap {
			clusterTps := offsetInfo.ClusterTotal(cluster)
			clusterPartitions := offsetInfo.ClusterPartitions(cluster)
			hostPartitions += clusterPartitions
			totalTps += clusterTps
			totalPartitions += clusterPartitions

			if this.hideZeroClusetr && clusterTps == 0 {
				continue
			}

			clusters = append(clusters, clusterQps{cluster, clusterTps, clusterPartitions})
		}

		sortutil.AscByField(clusters, "cluster")

		if offsetInfo.MightProblematic() {
			lines = append(lines, fmt.Sprintf("%s|%5.1f|%d|%s|%+v",
				color.Green(host), this.loadAvgMap[host], hostPartitions,
				gofmt.Comma(offsetInfo.Total()), clusters))
		} else {
			lines = append(lines, fmt.Sprintf("%s|%5.1f|%d|%s|%+v",
				host, this.loadAvgMap[host], hostPartitions,
				gofmt.Comma(offsetInfo.Total()), clusters))
		}

	}
	this.Ui.Output(columnize.SimpleFormat(lines))
	this.Ui.Output(fmt.Sprintf("-Total- Hosts:%d Partitions:%d Tps:%s",
		len(sortedHosts), totalPartitions, gofmt.Comma(totalTps)))
}

func (this *Balance) fetchLoadAvg() {
	defer close(this.loadAvgReady)

	// get members host ip
	cf := consulapi.DefaultConfig()
	client, _ := consulapi.NewClient(cf)
	members, _ := client.Agent().Members(false)

	nodeHostMap := make(map[string]string, len(members))
	for _, member := range members {
		nodeHostMap[member.Name] = member.Addr
	}

	cmd := pipestream.New("consul", "exec", "uptime", "|", "grep", "load")
	cmd.Open()
	if cmd.Reader() == nil {
		return
	}
	defer cmd.Close()

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		fields := strings.Fields(line)
		node := fields[0]
		parts := strings.Split(line, "load average:")
		if len(parts) < 2 {
			continue
		}
		if strings.HasSuffix(node, ":") {
			node = strings.TrimRight(node, ":")
		}

		load1m, err := ctx.ExtractLoadAvg1m(line)
		if err != nil {
			continue
		}

		host := nodeHostMap[node]
		this.loadAvgMap[host] = load1m
	}

}

func (this *Balance) clusterTopProducers(zkcluster *zk.ZkCluster) {
	kfk, err := sarama.NewClient(zkcluster.BrokerList(), sarama.NewConfig())
	if err != nil {
		return
	}
	defer kfk.Close()

	for {
		hostOffsets := make(map[string]hostOffsetInfo)

		topics, err := kfk.Topics()
		swallow(err)

		<-this.signalsCh[zkcluster.Name()]

		for _, topic := range topics {
			if topic == "__consumer_offsets" {
				// skip kafka intennal topics
				continue
			}

			partions, err := kfk.WritablePartitions(topic)
			swallow(err)
			for _, partitionID := range partions {
				leader, err := kfk.Leader(topic, partitionID)
				swallow(err)

				latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest)
				swallow(err)

				host, _, err := net.SplitHostPort(leader.Addr())
				swallow(err)

				if !patternMatched(host, this.host) {
					continue
				}

				if _, present := hostOffsets[host]; !present {
					hostOffsets[host] = hostOffsetInfo{
						host:      host,
						offsetMap: make(map[string]map[structs.TopicPartition]int64),
						brokerIDs: make(map[string]int32),
					}
				}
				if _, present := hostOffsets[host].offsetMap[zkcluster.Name()]; !present {
					hostOffsets[host].offsetMap[zkcluster.Name()] = make(map[structs.TopicPartition]int64)
					hostOffsets[host].brokerIDs[zkcluster.Name()] = leader.ID()
				}

				tp := structs.TopicPartition{Topic: topic, PartitionID: partitionID}
				hostOffsets[host].offsetMap[zkcluster.Name()][tp] = latestOffset
			}
		}

		this.hostOffsetCh <- hostOffsets

		kfk.RefreshMetadata(topics...)
	}
}

func (*Balance) Synopsis() string {
	return "Balance topics distribution according to load instead of count"
}

func (this *Balance) Help() string {
	help := fmt.Sprintf(`
Usage: %s balance [options]

    %s

Options:

    -z zone
      Default %s

    -c cluster pattern

    -host broker ip

    -l
      Use a long listing format.

    -over number
      Only display brokers whose TPS over the number.

    -nozero
      Hide 0 OPS clusters. False by default.

    -skipk
      Skip kafka internal topic: __consumer_offsets. True by default.

`, this.Cmd, this.Synopsis(), ctx.ZkDefaultZone())
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"path/filepath"

	"github.com/pedro-r-marques/k8s-service-proxy/pkg/proxy"
)

type options struct {
	Port          int
	HTTPStaticDir string
}

func defineFlags(opt *options) {
	flag.IntVar(&opt.Port, "port", 8080, "Listening port")
	flag.StringVar(&opt.HTTPStaticDir, "http-static-dir", "/var/www", "Directory for static http content")
}

func main() {
	var opt options
	defineFlags(&opt)
	flag.Parse()

	log.Print("Listening on port ", opt.Port)

	mux := http.NewServeMux()
	mux.Handle(proxy.SvcProxyHTTPPath, http.FileServer(http.Dir(opt.HTTPStaticDir)))
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/" {
			http.Redirect(w, r, proxy.SvcProxyHTTPPath+"status.html", http.StatusSeeOther)
		} else {
			w.WriteHeader(http.StatusNotFound)
			if msg, err := ioutil.ReadFile(filepath.Join(opt.HTTPStaticDir, proxy.SvcProxyHTTPPath+"error_404.html")); err == nil {
				w.Write(msg)
			} else {
				log.Println(err)
			}
		}
	})
	svcProxy := proxy.NewKubernetesServiceProxy(mux)
	http.ListenAndServe(fmt.Sprintf(":%d", opt.Port), svcProxy)
}
<commit_msg>Enable expvar and debug/pprof<commit_after>package main

import (
	_ "expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	_ "net/http/pprof"
	"path/filepath"

	"github.com/pedro-r-marques/k8s-service-proxy/pkg/proxy"
)

type options struct {
	Port          int
	HTTPStaticDir string
}

func defineFlags(opt *options) {
	flag.IntVar(&opt.Port, "port", 8080, "Listening port")
	flag.StringVar(&opt.HTTPStaticDir, "http-static-dir", "/var/www", "Directory for static http content")
}

func defaultMuxServeHTTP(w http.ResponseWriter, r *http.Request) {
	req := r.Clone(r.Context())
	req.URL.Path = req.URL.Path[len(proxy.SvcProxyHTTPPath)-1:]
	http.DefaultServeMux.ServeHTTP(w, req)
}

func main() {
	var opt options
	defineFlags(&opt)
	flag.Parse()

	log.Print("Listening on port ", opt.Port)

	mux := http.NewServeMux()
	mux.Handle(proxy.SvcProxyHTTPPath, http.FileServer(http.Dir(opt.HTTPStaticDir)))
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/" {
			http.Redirect(w, r, proxy.SvcProxyHTTPPath+"status.html", http.StatusSeeOther)
		} else {
			w.WriteHeader(http.StatusNotFound)
			if msg, err := ioutil.ReadFile(filepath.Join(opt.HTTPStaticDir, proxy.SvcProxyHTTPPath+"error_404.html")); err == nil {
				w.Write(msg)
			} else {
				log.Println(err)
			}
		}
	})

	mux.HandleFunc(proxy.SvcProxyHTTPPath+"debug/", defaultMuxServeHTTP)

	svcProxy := proxy.NewKubernetesServiceProxy(mux)
	http.ListenAndServe(fmt.Sprintf(":%d", opt.Port), svcProxy)
}
<|endoftext|>


<commit_before>package main

const NGINX_BUILD_VERSION = "0.0.6"

// nginx
const (
	NGINX_VERSION             = "1.7.6"
	NGINX_DOWNLOAD_URL_PREFIX = "http://nginx.org/download"
)

// pcre
const (
	PCRE_VERSION             = "8.36"
	PCRE_DOWNLOAD_URL_PREFIX = "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre"
)

// openssl
const (
	OPENSSL_VERSION             = "1.0.1i"
	OPENSSL_DOWNLOAD_URL_PREFIX = "http://www.openssl.org/source"
)

// zlib
const (
	ZLIB_VERSION             = "1.2.8"
	ZLIB_DOWNLOAD_URL_PREFIX = "http://zlib.net"
)

// component enumerations
const (
	COMPONENT_NGINX = iota
	COMPONENT_PCRE
	COMPONENT_OPENSSL
	COMPONENT_ZLIB
	COMPONENT_MAX
)
<commit_msg>bumped openssl version to 1.0.1j<commit_after>package main

const NGINX_BUILD_VERSION = "0.0.6"

// nginx
const (
	NGINX_VERSION             = "1.7.6"
	NGINX_DOWNLOAD_URL_PREFIX = "http://nginx.org/download"
)

// pcre
const (
	PCRE_VERSION             = "8.36"
	PCRE_DOWNLOAD_URL_PREFIX = "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre"
)

// openssl
const (
	OPENSSL_VERSION             = "1.0.1j"
	OPENSSL_DOWNLOAD_URL_PREFIX = "http://www.openssl.org/source"
)

// zlib
const (
	ZLIB_VERSION             = "1.2.8"
	ZLIB_DOWNLOAD_URL_PREFIX = "http://zlib.net"
)

// component enumerations
const (
	COMPONENT_NGINX = iota
	COMPONENT_PCRE
	COMPONENT_OPENSSL
	COMPONENT_ZLIB
	COMPONENT_MAX
)
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"github.com/spf13/cobra"
	"github.com/spf13/cobra/doc"
)

type GenHelpDocsCmd struct {
	cobraCommand *cobra.Command
	OutDir       string
}

var genHelpDocsCmd = GenHelpDocsCmd{
	cobraCommand: &cobra.Command{
		Use:    "genhelpdocs",
		Short:  "Generate CLI help docs",
		Hidden: true,
	},
}

func init() {
	cmd := genHelpDocsCmd.cobraCommand
	rootCommand.cobraCommand.AddCommand(cmd)

	cmd.Run = func(cmd *cobra.Command, args []string) {
		err := genHelpDocsCmd.Run()
		if err != nil {
			exitWithError(err)
		}
	}

	cmd.Flags().StringVar(&genHelpDocsCmd.OutDir, "out", "", "path to write out to.")
}

func (c *GenHelpDocsCmd) Run() error {
	rootCommand.cobraCommand.DisableAutoGenTag = true
	rootCommand.RegistryPath = ""
	err := doc.GenMarkdownTree(rootCommand.cobraCommand, c.OutDir)

	return err
}
<commit_msg>Fixes #2502: unset default value of state flag when docs are generated<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"github.com/spf13/cobra"
	"github.com/spf13/cobra/doc"
)

type GenHelpDocsCmd struct {
	cobraCommand *cobra.Command
	OutDir       string
}

var genHelpDocsCmd = GenHelpDocsCmd{
	cobraCommand: &cobra.Command{
		Use:    "genhelpdocs",
		Short:  "Generate CLI help docs",
		Hidden: true,
	},
}

func init() {
	cmd := genHelpDocsCmd.cobraCommand
	rootCommand.cobraCommand.AddCommand(cmd)

	cmd.Run = func(cmd *cobra.Command, args []string) {
		err := genHelpDocsCmd.Run()
		if err != nil {
			exitWithError(err)
		}
	}

	cmd.Flags().StringVar(&genHelpDocsCmd.OutDir, "out", "", "path to write out to.")
}

func (c *GenHelpDocsCmd) Run() error {
	rootCommand.cobraCommand.DisableAutoGenTag = true

	// unset KOPS_STATE_STORE from default value
	for _, c := range rootCommand.cobraCommand.Commands() {
		c.Flag("state").DefValue = ""
	}
	return doc.GenMarkdownTree(rootCommand.cobraCommand, c.OutDir)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net/http"
	"time"

	"github.com/go-graphite/protocol/carbonapi_v2_pb"
	"github.com/go-graphite/protocol/carbonapi_v3_pb"
	ogórek "github.com/lomik/og-rek"
	"go.uber.org/zap"

	"github.com/go-graphite/carbonapi/zipper/httpHeaders"
)

func (cfg *listener) renderHandler(wr http.ResponseWriter, req *http.Request) {
	hdrs := make(map[string][]string)

	for n, v := range req.Header {
		hdrs[n] = v
	}

	logger := cfg.logger.With(
		zap.String("function", "renderHandler"),
		zap.String("method", req.Method),
		zap.String("path", req.URL.Path),
		zap.Any("headers", hdrs),
	)

	logger.Info("got request")
	if cfg.Code != http.StatusOK {
		wr.WriteHeader(cfg.Code)
		return
	}

	format, err := getFormat(req)
	if err != nil {
		logger.Error("bad request, failed to parse format")
		wr.WriteHeader(http.StatusBadRequest)
		_, _ = wr.Write([]byte(err.Error()))
		return
	}

	targets := req.Form["target"]
	maxDataPoints := int64(0)

	if format == protoV3Format {
		body, err := ioutil.ReadAll(req.Body)
		if err != nil {
			logger.Error("bad request, failed to read request body",
				zap.Error(err),
			)
			http.Error(wr, "bad request (failed to read request body): "+err.Error(), http.StatusBadRequest)
			return
		}

		var pv3Request carbonapi_v3_pb.MultiFetchRequest
		err = pv3Request.Unmarshal(body)

		if err != nil {
			logger.Error("bad request, failed to unmarshal request",
				zap.Error(err),
			)
			http.Error(wr, "bad request (failed to parse format): "+err.Error(), http.StatusBadRequest)
			return
		}

		targets = make([]string, len(pv3Request.Metrics))
		for i, r := range pv3Request.Metrics {
			targets[i] = r.PathExpression
		}
		maxDataPoints = pv3Request.Metrics[0].MaxDataPoints
	}

	logger.Info("request details",
		zap.Strings("target", targets),
		zap.String("format", format.String()),
		zap.Int64("maxDataPoints", maxDataPoints),
	)

	multiv2 := carbonapi_v2_pb.MultiFetchResponse{
		Metrics: []carbonapi_v2_pb.FetchResponse{},
	}

	multiv3 := carbonapi_v3_pb.MultiFetchResponse{
		Metrics: []carbonapi_v3_pb.FetchResponse{},
	}

	newCfg := Listener{
		Code:        cfg.Code,
		EmptyBody:   cfg.EmptyBody,
		Expressions: copyMap(cfg.Expressions),
	}

	for _, target := range targets {
		response, ok := newCfg.Expressions[target]
		if !ok {
			wr.WriteHeader(http.StatusNotFound)
			_, _ = wr.Write([]byte("Not found"))
			return
		}
		if response.ReplyDelayMS > 0 {
			delay := time.Duration(response.ReplyDelayMS) * time.Millisecond
			logger.Info("will add extra delay",
				zap.Duration("delay", delay),
			)
			time.Sleep(delay)
		}
		for _, m := range response.Data {
			startTime := m.StartTime
			if startTime == 0 {
				startTime = 1
			}
			step := m.Step
			if step == 0 {
				step = 1
			}
			isAbsent := make([]bool, 0, len(m.Values))
			protov2Values := make([]float64, 0, len(m.Values))
			for i := range m.Values {
				if math.IsNaN(m.Values[i]) {
					isAbsent = append(isAbsent, true)
					protov2Values = append(protov2Values, 0.0)
				} else {
					isAbsent = append(isAbsent, false)
					protov2Values = append(protov2Values, m.Values[i])
				}
			}
			fr2 := carbonapi_v2_pb.FetchResponse{
				Name:      m.MetricName,
				StartTime: int32(startTime),
				StopTime:  int32(step * (startTime + len(protov2Values) - 1)),
				StepTime:  int32(step),
				Values:    protov2Values,
				IsAbsent:  isAbsent,
			}

			fr3 := carbonapi_v3_pb.FetchResponse{
				Name:                    m.MetricName,
				PathExpression:          target,
				ConsolidationFunc:       "avg",
				StartTime:               int64(startTime),
				StopTime:                int64(step * (startTime + len(m.Values) - 1)),
				StepTime:                int64(step),
				XFilesFactor:            0,
				HighPrecisionTimestamps: false,
				Values:                  m.Values,
				RequestStartTime:        1,
				RequestStopTime:         int64(step * (startTime + len(m.Values) - 1)),
			}

			multiv2.Metrics = append(multiv2.Metrics, fr2)
			multiv3.Metrics = append(multiv3.Metrics, fr3)
		}
	}

	if cfg.Listener.ShuffleResults {
		rand.Shuffle(len(multiv2.Metrics), func(i, j int) {
			multiv2.Metrics[i], multiv2.Metrics[j] = multiv2.Metrics[j], multiv2.Metrics[i]
		})
		rand.Shuffle(len(multiv3.Metrics), func(i, j int) {
			multiv3.Metrics[i], multiv3.Metrics[j] = multiv3.Metrics[j], multiv3.Metrics[i]
		})
	}

	contentType, d := cfg.marshalResponse(wr, logger, format, multiv3, multiv2)
	if d == nil {
		return
	}
	wr.Header().Set("Content-Type", contentType)
	_, _ = wr.Write(d)
}

func (cfg *listener) marshalResponse(wr http.ResponseWriter, logger *zap.Logger, format responseFormat, multiv3 carbonapi_v3_pb.MultiFetchResponse, multiv2 carbonapi_v2_pb.MultiFetchResponse) (string, []byte) {
	var d []byte
	var contentType string
	var err error
	switch format {
	case pickleFormat:
		contentType = httpHeaders.ContentTypePickle
		if cfg.EmptyBody {
			break
		}
		var response []map[string]interface{}

		for _, metric := range multiv3.GetMetrics() {
			m := make(map[string]interface{})
			m["start"] = metric.StartTime
			m["step"] = metric.StepTime
			m["end"] = metric.StopTime
			m["name"] = metric.Name
			m["pathExpression"] = metric.PathExpression
			m["xFilesFactor"] = 0.5
			m["consolidationFunc"] = "avg"

			mv := make([]interface{}, len(metric.Values))
			for i, p := range metric.Values {
				if math.IsNaN(p) {
					mv[i] = nil
				} else {
					mv[i] = p
				}
			}

			m["values"] = mv
			log.Printf("%+v\n\n", m)
			response = append(response, m)
		}

		var buf bytes.Buffer
		logger.Info("request will be served",
			zap.String("format", "pickle"),
			zap.Any("content", response),
		)
		pEnc := ogórek.NewEncoder(&buf)
		err = pEnc.Encode(response)
		if err != nil {
			wr.WriteHeader(http.StatusBadGateway)
			_, _ = wr.Write([]byte(err.Error()))
			return "", nil
		}
		d = buf.Bytes()
	case protoV2Format:
		contentType = httpHeaders.ContentTypeCarbonAPIv2PB
		if cfg.EmptyBody {
			break
		}
		logger.Info("request will be served",
			zap.String("format", "protov2"),
			zap.Any("content", multiv2),
		)
		d, err = multiv2.Marshal()
		if err != nil {
			wr.WriteHeader(http.StatusBadGateway)
			_, _ = wr.Write([]byte(err.Error()))
			return "", nil
		}
	case protoV3Format:
		contentType = httpHeaders.ContentTypeCarbonAPIv3PB
		if cfg.EmptyBody {
			break
		}
		logger.Info("request will be served",
			zap.String("format", "protov3"),
			zap.Any("content", multiv3),
		)
		d, err = multiv3.Marshal()
		if err != nil {
			wr.WriteHeader(http.StatusBadGateway)
			_, _ = wr.Write([]byte(err.Error()))
			return "", nil
		}
	case jsonFormat:
		contentType = "application/json"
		if cfg.EmptyBody {
			break
		}
		logger.Info("request will be served",
			zap.String("format", "json"),
			zap.Any("content", multiv2),
		)
		d, err = json.Marshal(multiv2)
		if err != nil {
			wr.WriteHeader(http.StatusBadGateway)
			_, _ = wr.Write([]byte(err.Error()))
			return "", nil
		}
	default:
		logger.Error("format is not supported",
			zap.Any("format", format),
		)
		return "", nil
	}
	return contentType, d
}
<commit_msg>Adjust mockbackend<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net/http"
	"time"

	"github.com/go-graphite/protocol/carbonapi_v2_pb"
	"github.com/go-graphite/protocol/carbonapi_v3_pb"
	ogórek "github.com/lomik/og-rek"
	"go.uber.org/zap"

	"github.com/go-graphite/carbonapi/zipper/httpHeaders"
)

func (cfg *listener) renderHandler(wr http.ResponseWriter, req *http.Request) {
	hdrs := make(map[string][]string)

	for n, v := range req.Header {
		hdrs[n] = v
	}

	logger := cfg.logger.With(
		zap.String("function", "renderHandler"),
		zap.String("method", req.Method),
		zap.String("path", req.URL.Path),
		zap.Any("headers", hdrs),
	)

	logger.Info("got request")
	if cfg.Code != http.StatusOK {
		wr.WriteHeader(cfg.Code)
		return
	}

	format, err := getFormat(req)
	if err != nil {
		logger.Error("bad request, failed to parse format")
		wr.WriteHeader(http.StatusBadRequest)
		_, _ = wr.Write([]byte(err.Error()))
		return
	}

	targets := req.Form["target"]
	maxDataPoints := int64(0)

	if format == protoV3Format {
		body, err := ioutil.ReadAll(req.Body)
		if err != nil {
			logger.Error("bad request, failed to read request body",
				zap.Error(err),
			)
			http.Error(wr, "bad request (failed to read request body): "+err.Error(), http.StatusBadRequest)
			return
		}

		var pv3Request carbonapi_v3_pb.MultiFetchRequest
		err = pv3Request.Unmarshal(body)

		if err != nil {
			logger.Error("bad request, failed to unmarshal request",
				zap.Error(err),
			)
			http.Error(wr, "bad request (failed to parse format): "+err.Error(), http.StatusBadRequest)
			return
		}

		targets = make([]string, len(pv3Request.Metrics))
		for i, r := range pv3Request.Metrics {
			targets[i] = r.PathExpression
		}
		maxDataPoints = pv3Request.Metrics[0].MaxDataPoints
	}

	logger.Info("request details",
		zap.Strings("target", targets),
		zap.String("format", format.String()),
		zap.Int64("maxDataPoints", maxDataPoints),
	)

	multiv2 := carbonapi_v2_pb.MultiFetchResponse{
		Metrics: []carbonapi_v2_pb.FetchResponse{},
	}

	multiv3 := carbonapi_v3_pb.MultiFetchResponse{
		Metrics: []carbonapi_v3_pb.FetchResponse{},
	}

	newCfg := Listener{
		Code:        cfg.Code,
		EmptyBody:   cfg.EmptyBody,
		Expressions: copyMap(cfg.Expressions),
	}

	for _, target := range targets {
		response, ok := newCfg.Expressions[target]
		if !ok {
			wr.WriteHeader(http.StatusNotFound)
			_, _ = wr.Write([]byte("Not found"))
			return
		}
		if response.ReplyDelayMS > 0 {
			delay := time.Duration(response.ReplyDelayMS) * time.Millisecond
			logger.Info("will add extra delay",
				zap.Duration("delay", delay),
			)
			time.Sleep(delay)
		}
		for _, m := range response.Data {
			step := m.Step
			if step == 0 {
				step = 1
			}
			startTime := m.StartTime
			if startTime == 0 {
				startTime = step
			}
			isAbsent := make([]bool, 0, len(m.Values))
			protov2Values := make([]float64, 0, len(m.Values))
			for i := range m.Values {
				if math.IsNaN(m.Values[i]) {
					isAbsent = append(isAbsent, true)
					protov2Values = append(protov2Values, 0.0)
				} else {
					isAbsent = append(isAbsent, false)
					protov2Values = append(protov2Values, m.Values[i])
				}
			}
			fr2 := carbonapi_v2_pb.FetchResponse{
				Name:      m.MetricName,
				StartTime: int32(startTime),
				StopTime:  int32(startTime + step*len(protov2Values)),
				StepTime:  int32(step),
				Values:    protov2Values,
				IsAbsent:  isAbsent,
			}

			fr3 := carbonapi_v3_pb.FetchResponse{
				Name:                    m.MetricName,
				PathExpression:          target,
				ConsolidationFunc:       "avg",
				StartTime:               int64(startTime),
				StopTime:                int64(startTime + step*len(m.Values)),
				StepTime:                int64(step),
				XFilesFactor:            0,
				HighPrecisionTimestamps: false,
				Values:                  m.Values,
				RequestStartTime:        1,
				RequestStopTime:         int64(startTime + step*len(m.Values)),
			}

			multiv2.Metrics = append(multiv2.Metrics, fr2)
			multiv3.Metrics = append(multiv3.Metrics, fr3)
		}
	}

	if cfg.Listener.ShuffleResults {
		rand.Shuffle(len(multiv2.Metrics), func(i, j int) {
			multiv2.Metrics[i], multiv2.Metrics[j] = multiv2.Metrics[j], multiv2.Metrics[i]
		})
		rand.Shuffle(len(multiv3.Metrics), func(i, j int) {
			multiv3.Metrics[i], multiv3.Metrics[j] = multiv3.Metrics[j], multiv3.Metrics[i]
		})
	}

	contentType, d := cfg.marshalResponse(wr, logger, format, multiv3, multiv2)
	if d == nil {
		return
	}
	wr.Header().Set("Content-Type", contentType)
	_, _ = wr.Write(d)
}

func (cfg *listener) marshalResponse(wr http.ResponseWriter, logger *zap.Logger, format responseFormat, multiv3 carbonapi_v3_pb.MultiFetchResponse, multiv2 carbonapi_v2_pb.MultiFetchResponse) (string, []byte) {
	var d []byte
	var contentType string
	var err error
	switch format {
	case pickleFormat:
		contentType = httpHeaders.ContentTypePickle
		if cfg.EmptyBody {
			break
		}
		var response []map[string]interface{}

		for _, metric := range multiv3.GetMetrics() {
			m := make(map[string]interface{})
			m["start"] = metric.StartTime
			m["step"] = metric.StepTime
			m["end"] = metric.StopTime
			m["name"] = metric.Name
			m["pathExpression"] = metric.PathExpression
			m["xFilesFactor"] = 0.5
			m["consolidationFunc"] = "avg"

			mv := make([]interface{}, len(metric.Values))
			for i, p := range metric.Values {
				if math.IsNaN(p) {
					mv[i] = nil
				} else {
					mv[i] = p
				}
			}

			m["values"] = mv
			log.Printf("%+v\n\n", m)
			response = append(response, m)
		}

		var buf bytes.Buffer
		logger.Info("request will be served",
			zap.String("format", "pickle"),
			zap.Any("content", response),
		)
		pEnc := ogórek.NewEncoder(&buf)
		err = pEnc.Encode(response)
		if err != nil {
			wr.WriteHeader(http.StatusBadGateway)
			_, _ = wr.Write([]byte(err.Error()))
			return "", nil
		}
		d = buf.Bytes()
	case protoV2Format:
		contentType = httpHeaders.ContentTypeCarbonAPIv2PB
		if cfg.EmptyBody {
			break
		}
		logger.Info("request will be served",
			zap.String("format", "protov2"),
			zap.Any("content", multiv2),
		)
		d, err = multiv2.Marshal()
		if err != nil {
			wr.WriteHeader(http.StatusBadGateway)
			_, _ = wr.Write([]byte(err.Error()))
			return "", nil
		}
	case protoV3Format:
		contentType = httpHeaders.ContentTypeCarbonAPIv3PB
		if cfg.EmptyBody {
			break
		}
		logger.Info("request will be served",
			zap.String("format", "protov3"),
			zap.Any("content", multiv3),
		)
		d, err = multiv3.Marshal()
		if err != nil {
			wr.WriteHeader(http.StatusBadGateway)
			_, _ = wr.Write([]byte(err.Error()))
			return "", nil
		}
	case jsonFormat:
		contentType = "application/json"
		if cfg.EmptyBody {
			break
		}
		logger.Info("request will be served",
			zap.String("format", "json"),
			zap.Any("content", multiv2),
		)
		d, err = json.Marshal(multiv2)
		if err != nil {
			wr.WriteHeader(http.StatusBadGateway)
			_, _ = wr.Write([]byte(err.Error()))
			return "", nil
		}
	default:
		logger.Error("format is not supported",
			zap.Any("format", format),
		)
		return "", nil
	}
	return contentType, d
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"sort"
	"strings"

	"golang.org/x/oauth2"
	"golang.org/x/sync/errgroup"

	"github.com/haya14busa/go-actions-toolkit/core"
	"github.com/reviewdog/reviewdog"
	"github.com/reviewdog/reviewdog/cienv"
	"github.com/reviewdog/reviewdog/doghouse"
	"github.com/reviewdog/reviewdog/doghouse/client"
	"github.com/reviewdog/reviewdog/project"
)

func runDoghouse(ctx context.Context, r io.Reader, w io.Writer, opt *option, isProject bool, forPr bool) error {
	ghInfo, isPr, err := cienv.GetBuildInfo()
	if err != nil {
		return err
	}
	if !isPr && forPr {
		fmt.Fprintln(os.Stderr, "reviewdog: this is not PullRequest build.")
		return nil
	}
	if !forPr {
		ghInfo.PullRequest = 0
	}
	resultSet, err := checkResultSet(ctx, r, opt, isProject)
	if err != nil {
		return err
	}
	cli, err := newDoghouseCli(ctx)
	if err != nil {
		return err
	}
	filteredResultSet, err := postResultSet(ctx, resultSet, ghInfo, cli)
	if err != nil {
		return err
	}
	if foundResultInDiff := reportResults(w, filteredResultSet); foundResultInDiff {
		return errors.New("found at least one result in diff")
	}
	return nil
}

func newDoghouseCli(ctx context.Context) (client.DogHouseClientInterface, error) {
	// If skipDoghouseServer is true, run doghouse code directly instead of talking to
	// the doghouse server because provided GitHub API Token has Check API scope.
	skipDoghouseServer := cienv.IsInGitHubAction() && os.Getenv("REVIEWDOG_TOKEN") == ""
	if skipDoghouseServer {
		token, err := nonEmptyEnv("REVIEWDOG_GITHUB_API_TOKEN")
		if err != nil {
			return nil, err
		}
		ghcli, err := githubClient(ctx, token)
		if err != nil {
			return nil, err
		}
		return &client.GitHubClient{Client: ghcli}, nil
	}
	return newDoghouseServerCli(ctx), nil
}

func newDoghouseServerCli(ctx context.Context) *client.DogHouseClient {
	httpCli := http.DefaultClient
	if token := os.Getenv("REVIEWDOG_TOKEN"); token != "" {
		ts := oauth2.StaticTokenSource(
			&oauth2.Token{AccessToken: token},
		)
		httpCli = oauth2.NewClient(ctx, ts)
	}
	return client.New(httpCli)
}

var projectRunAndParse = project.RunAndParse

func checkResultSet(ctx context.Context, r io.Reader, opt *option, isProject bool) (*reviewdog.ResultMap, error) {
	resultSet := new(reviewdog.ResultMap)
	if isProject {
		conf, err := projectConfig(opt.conf)
		if err != nil {
			return nil, err
		}
		resultSet, err = projectRunAndParse(ctx, conf, buildRunnersMap(opt.runners), opt.level)
		if err != nil {
			return nil, err
		}
	} else {
		p, err := newParserFromOpt(opt)
		if err != nil {
			return nil, err
		}
		rs, err := p.Parse(r)
		if err != nil {
			return nil, err
		}
		resultSet.Store(toolName(opt), &reviewdog.Result{
			Level:        opt.level,
			CheckResults: rs,
		})
	}
	return resultSet, nil
}

func postResultSet(ctx context.Context, resultSet *reviewdog.ResultMap, ghInfo *cienv.BuildInfo, cli client.DogHouseClientInterface) (*reviewdog.FilteredResultMap, error) {
	var g errgroup.Group
	wd, _ := os.Getwd()
	filteredResultSet := new(reviewdog.FilteredResultMap)
	resultSet.Range(func(name string, result *reviewdog.Result) {
		checkResults := result.CheckResults
		as := make([]*doghouse.Annotation, 0, len(checkResults))
		for _, r := range checkResults {
			as = append(as, checkResultToAnnotation(r, wd))
		}
		req := &doghouse.CheckRequest{
			Name:        name,
			Owner:       ghInfo.Owner,
			Repo:        ghInfo.Repo,
			PullRequest: ghInfo.PullRequest,
			SHA:         ghInfo.SHA,
			Branch:      ghInfo.Branch,
			Annotations: as,
			Level:       result.Level,
		}
		g.Go(func() error {
			res, err := cli.Check(ctx, req)
			if err != nil {
				return fmt.Errorf("post failed for %s: %v", name, err)
			}
			if res.ReportURL != "" {
				log.Printf("[%s] reported: %s", name, res.ReportURL)
			}
			if res.CheckedResults != nil {
				filteredResultSet.Store(name, &reviewdog.FilteredResult{
					Level:         result.Level,
					FilteredCheck: res.CheckedResults,
				})
			}
			if res.ReportURL == "" && res.CheckedResults == nil {
				return fmt.Errorf("no result found for %q", name)
			}
			return nil
		})
	})
	return filteredResultSet, g.Wait()
}

func checkResultToAnnotation(c *reviewdog.CheckResult, wd string) *doghouse.Annotation {
	return &doghouse.Annotation{
		Path:       reviewdog.CleanPath(c.Path, wd),
		Line:       c.Lnum,
		Message:    c.Message,
		RawMessage: strings.Join(c.Lines, "\n"),
	}
}

// reportResults reports results to given io.Writer and return true if at least
// one annotation result is in diff.
func reportResults(w io.Writer, filteredResultSet *reviewdog.FilteredResultMap) bool {
	// Sort names to get deterministic result.
	var names []string
	filteredResultSet.Range(func(name string, results *reviewdog.FilteredResult) {
		names = append(names, name)
	})
	sort.Strings(names)

	foundInDiff := false
	for _, name := range names {
		results, err := filteredResultSet.Load(name)
		if err != nil {
			// Should not happen.
			log.Printf("reviewdog: result not found for %q", name)
			continue
		}
		fmt.Fprintf(w, "reviewdog: Reporting results for %q\n", name)
		foundResultPerName := false
		filteredNum := 0
		for _, result := range results.FilteredCheck {
			if !result.InDiff {
				filteredNum++
				continue
			}
			foundInDiff = true
			foundResultPerName = true
			if cienv.IsInGitHubAction() {
				reportResultsInGitHubActions(name, results.Level, result)
			} else {
				// Output original lines.
				for _, line := range result.Lines {
					fmt.Fprintln(w, line)
				}
			}
		}
		if !foundResultPerName {
			fmt.Fprintf(w, "reviewdog: No results found for %q. %d results found outside diff.\n", name, filteredNum)
		}
	}
	return foundInDiff
}

// Report results via logging command to create annotations.
// https://help.github.com/en/actions/automating-your-workflow-with-github-actions/development-tools-for-github-actions#example-5
func reportResultsInGitHubActions(toolName, level string, result *reviewdog.FilteredCheck) {
	mes := fmt.Sprintf("[%s] reported by reviewdog 🐶\n%s",
		toolName, strings.Join(result.Lines, "\n"))
	opt := &core.LogOption{
		File: result.Path,
		Line: result.Lnum,
		Col:  result.Col,
	}
	// no info command with location data.
	switch level {
	case "error":
		core.Error(mes, opt)
	default:
		core.Warning(mes, opt)
	}
}
<commit_msg>improve status handling and do not exit with 0 if level is warning or info<commit_after>package main

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"sort"
	"strings"

	"golang.org/x/oauth2"
	"golang.org/x/sync/errgroup"

	"github.com/haya14busa/go-actions-toolkit/core"
	"github.com/reviewdog/reviewdog"
	"github.com/reviewdog/reviewdog/cienv"
	"github.com/reviewdog/reviewdog/doghouse"
	"github.com/reviewdog/reviewdog/doghouse/client"
	"github.com/reviewdog/reviewdog/project"
)

func runDoghouse(ctx context.Context, r io.Reader, w io.Writer, opt *option, isProject bool, forPr bool) error {
	ghInfo, isPr, err := cienv.GetBuildInfo()
	if err != nil {
		return err
	}
	if !isPr && forPr {
		fmt.Fprintln(os.Stderr, "reviewdog: this is not PullRequest build.")
		return nil
	}
	if !forPr {
		ghInfo.PullRequest = 0
	}
	resultSet, err := checkResultSet(ctx, r, opt, isProject)
	if err != nil {
		return err
	}
	cli, err := newDoghouseCli(ctx)
	if err != nil {
		return err
	}
	filteredResultSet, err := postResultSet(ctx, resultSet, ghInfo, cli)
	if err != nil {
		return err
	}
	if foundResultInDiff := reportResults(w, filteredResultSet); foundResultInDiff {
		return errors.New("found at least one result in diff")
	}
	return nil
}

func newDoghouseCli(ctx context.Context) (client.DogHouseClientInterface, error) {
	// If skipDoghouseServer is true, run doghouse code directly instead of talking to
	// the doghouse server because provided GitHub API Token has Check API scope.
	skipDoghouseServer := cienv.IsInGitHubAction() && os.Getenv("REVIEWDOG_TOKEN") == ""
	if skipDoghouseServer {
		token, err := nonEmptyEnv("REVIEWDOG_GITHUB_API_TOKEN")
		if err != nil {
			return nil, err
		}
		ghcli, err := githubClient(ctx, token)
		if err != nil {
			return nil, err
		}
		return &client.GitHubClient{Client: ghcli}, nil
	}
	return newDoghouseServerCli(ctx), nil
}

func newDoghouseServerCli(ctx context.Context) *client.DogHouseClient {
	httpCli := http.DefaultClient
	if token := os.Getenv("REVIEWDOG_TOKEN"); token != "" {
		ts := oauth2.StaticTokenSource(
			&oauth2.Token{AccessToken: token},
		)
		httpCli = oauth2.NewClient(ctx, ts)
	}
	return client.New(httpCli)
}

var projectRunAndParse = project.RunAndParse

func checkResultSet(ctx context.Context, r io.Reader, opt *option, isProject bool) (*reviewdog.ResultMap, error) {
	resultSet := new(reviewdog.ResultMap)
	if isProject {
		conf, err := projectConfig(opt.conf)
		if err != nil {
			return nil, err
		}
		resultSet, err = projectRunAndParse(ctx, conf, buildRunnersMap(opt.runners), opt.level)
		if err != nil {
			return nil, err
		}
	} else {
		p, err := newParserFromOpt(opt)
		if err != nil {
			return nil, err
		}
		rs, err := p.Parse(r)
		if err != nil {
			return nil, err
		}
		resultSet.Store(toolName(opt), &reviewdog.Result{
			Level:        opt.level,
			CheckResults: rs,
		})
	}
	return resultSet, nil
}

func postResultSet(ctx context.Context, resultSet *reviewdog.ResultMap, ghInfo *cienv.BuildInfo, cli client.DogHouseClientInterface) (*reviewdog.FilteredResultMap, error) {
	var g errgroup.Group
	wd, _ := os.Getwd()
	filteredResultSet := new(reviewdog.FilteredResultMap)
	resultSet.Range(func(name string, result *reviewdog.Result) {
		checkResults := result.CheckResults
		as := make([]*doghouse.Annotation, 0, len(checkResults))
		for _, r := range checkResults {
			as = append(as, checkResultToAnnotation(r, wd))
		}
		req := &doghouse.CheckRequest{
			Name:        name,
			Owner:       ghInfo.Owner,
			Repo:        ghInfo.Repo,
			PullRequest: ghInfo.PullRequest,
			SHA:         ghInfo.SHA,
			Branch:      ghInfo.Branch,
			Annotations: as,
			Level:       result.Level,
		}
		g.Go(func() error {
			res, err := cli.Check(ctx, req)
			if err != nil {
				return fmt.Errorf("post failed for %s: %v", name, err)
			}
			if res.ReportURL != "" {
				log.Printf("[%s] reported: %s", name, res.ReportURL)
			}
			if res.CheckedResults != nil {
				filteredResultSet.Store(name, &reviewdog.FilteredResult{
					Level:         result.Level,
					FilteredCheck: res.CheckedResults,
				})
			}
			if res.ReportURL == "" && res.CheckedResults == nil {
				return fmt.Errorf("no result found for %q", name)
			}
			return nil
		})
	})
	return filteredResultSet, g.Wait()
}

func checkResultToAnnotation(c *reviewdog.CheckResult, wd string) *doghouse.Annotation {
	return &doghouse.Annotation{
		Path:       reviewdog.CleanPath(c.Path, wd),
		Line:       c.Lnum,
		Message:    c.Message,
		RawMessage: strings.Join(c.Lines, "\n"),
	}
}

// reportResults reports results to given io.Writer and return true if
// reviewdog should exit with 1.
// e.g. At least one annotation result is in diff.
func reportResults(w io.Writer, filteredResultSet *reviewdog.FilteredResultMap) bool {
	// Sort names to get deterministic result.
	var names []string
	filteredResultSet.Range(func(name string, results *reviewdog.FilteredResult) {
		names = append(names, name)
	})
	sort.Strings(names)

	shouldFail := false
	for _, name := range names {
		results, err := filteredResultSet.Load(name)
		if err != nil {
			// Should not happen.
			log.Printf("reviewdog: result not found for %q", name)
			continue
		}
		fmt.Fprintf(w, "reviewdog: Reporting results for %q\n", name)
		foundResultPerName := false
		filteredNum := 0
		for _, result := range results.FilteredCheck {
			if !result.InDiff {
				filteredNum++
				continue
			}
			// If it's not running in GitHub Actions, reviewdog should exit with 1
			// if there are at least one result in diff regardless of error level.
			shouldFail = shouldFail || !cienv.IsInGitHubAction() ||
				!(results.Level == "warning" || results.Level == "info")
			foundResultPerName = true
			if cienv.IsInGitHubAction() {
				reportResultsInGitHubActions(name, results.Level, result)
			} else {
				// Output original lines.
				for _, line := range result.Lines {
					fmt.Fprintln(w, line)
				}
			}
		}
		if !foundResultPerName {
			fmt.Fprintf(w, "reviewdog: No results found for %q. %d results found outside diff.\n", name, filteredNum)
		}
	}
	return shouldFail
}

// Report results via logging command to create annotations.
// https://help.github.com/en/actions/automating-your-workflow-with-github-actions/development-tools-for-github-actions#example-5
func reportResultsInGitHubActions(toolName, level string, result *reviewdog.FilteredCheck) {
	mes := fmt.Sprintf("[%s] reported by reviewdog 🐶\n%s",
		toolName, strings.Join(result.Lines, "\n"))
	opt := &core.LogOption{
		File: result.Path,
		Line: result.Lnum,
		Col:  result.Col,
	}
	switch level {
	// no info command with location data.
	case "warning", "info":
		core.Warning(mes, opt)
	case "error", "":
		core.Error(mes, opt)
	default:
		core.Error(fmt.Sprintf("Unknown level: %s", level), nil)
		core.Error(mes, opt)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"github.com/whosonfirst/go-whosonfirst-csv"
	"gopkg.in/redis.v1"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	_ "time"
)

func main() {

	var dryrun = flag.Bool("dryrun", false, "...")

	var redis_host = flag.String("redis-host", "localhost", "Redis host")
	var redis_port = flag.Int("redis-port", 6379, "Redis port")
	var redis_channel = flag.String("redis-channel", "updated", "Redis channel")

	var repo = flag.String("repo", "", "...")
	var start_commit = flag.String("start-commit", "", "...")
	var stop_commit = flag.String("stop-commit", "", "...")

	flag.Parse()

	_, err := os.Stat(*repo)

	if os.IsNotExist(err) {
		log.Fatal("Repo does not exist", *repo)
	}

	cwd, err := os.Getwd()

	if err != nil {
		log.Fatal(err)
	}

	// I wish this wasn't necessary. I wish I could make sense of the
	// libgit2 documentation... (20161222/thisisaaronland)

	err = os.Chdir(*repo)

	if err != nil {
		log.Fatal(err)
	}

	// https://git-scm.com/docs/git-diff

	git_args := []string{
		// "diff", "--pretty=format:", "--name-only",
		"log", "--pretty=format:#%H", "--name-only",
	}

	if *start_commit != "" {

		stop := "HEAD"

		if *stop_commit != "" {
			stop = *stop_commit
		}

		commit_range := fmt.Sprintf("%s^...%s", *start_commit, stop)
		git_args = append(git_args, commit_range)
	}

	log.Println(strings.Join(git_args, " "))

	cmd := exec.Command("git", git_args...)
	out, err := cmd.Output()

	os.Chdir(cwd)

	if err != nil {
		log.Fatal(err)
	}

	var b bytes.Buffer
	buf := bufio.NewWriter(&b)

	fieldnames := []string{"hash", "repo", "path"}
	writer, err := csv.NewDictWriter(buf, fieldnames)

	var hash string

	for _, ln := range strings.Split(string(out), "\n") {

		if strings.HasPrefix(ln, "#") {
			hash = strings.Replace(ln, "#", "", 1)
		}

		if strings.HasSuffix(ln, ".geojson") {

			row := make(map[string]string)
			row["hash"] = hash
			row["repo"] = filepath.Base(*repo)
			row["path"] = ln

			writer.WriteRow(row)
		}
	}

	buf.Flush()

	log.Println(b.String())

	if !*dryrun {
		redis_endpoint := fmt.Sprintf("%s:%d", *redis_host, *redis_port)

		redis_client := redis.NewTCPClient(&redis.Options{
			Addr: redis_endpoint,
		})

		defer redis_client.Close()

		redis_client.Publish(*redis_channel, b.String())
	}

	// log.Printf("%s", files)
}
<commit_msg>notes to self inre issue #1<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"github.com/whosonfirst/go-whosonfirst-csv"
	"gopkg.in/redis.v1"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	_ "time"
)

func main() {

	var dryrun = flag.Bool("dryrun", false, "...")

	var redis_host = flag.String("redis-host", "localhost", "Redis host")
	var redis_port = flag.Int("redis-port", 6379, "Redis port")
	var redis_channel = flag.String("redis-channel", "updated", "Redis channel")

	var repo = flag.String("repo", "", "...")
	var start_commit = flag.String("start-commit", "", "...")
	var stop_commit = flag.String("stop-commit", "", "...")

	flag.Parse()

	_, err := os.Stat(*repo)

	if os.IsNotExist(err) {
		log.Fatal("Repo does not exist", *repo)
	}

	cwd, err := os.Getwd()

	if err != nil {
		log.Fatal(err)
	}

	// I wish this wasn't necessary. I wish I could make sense of the
	// libgit2 documentation... (20161222/thisisaaronland)

	// See also: https://github.com/whosonfirst/go-whosonfirst-updated/issues/1

	err = os.Chdir(*repo)

	if err != nil {
		log.Fatal(err)
	}

	// https://git-scm.com/docs/git-diff

	git_args := []string{
		// "diff", "--pretty=format:", "--name-only",
		"log", "--pretty=format:#%H", "--name-only",
	}

	if *start_commit != "" {

		stop := "HEAD"

		if *stop_commit != "" {
			stop = *stop_commit
		}

		commit_range := fmt.Sprintf("%s^...%s", *start_commit, stop)
		git_args = append(git_args, commit_range)
	}

	log.Println(strings.Join(git_args, " "))

	cmd := exec.Command("git", git_args...)
	out, err := cmd.Output()

	os.Chdir(cwd)

	if err != nil {
		log.Fatal(err)
	}

	var b bytes.Buffer
	buf := bufio.NewWriter(&b)

	fieldnames := []string{"hash", "repo", "path"}
	writer, err := csv.NewDictWriter(buf, fieldnames)

	var hash string

	for _, ln := range strings.Split(string(out), "\n") {

		if strings.HasPrefix(ln, "#") {
			hash = strings.Replace(ln, "#", "", 1)
		}

		if strings.HasSuffix(ln, ".geojson") {

			row := make(map[string]string)
			row["hash"] = hash
			row["repo"] = filepath.Base(*repo)
			row["path"] = ln

			writer.WriteRow(row)
		}
	}

	buf.Flush()

	log.Println(b.String())

	if !*dryrun {
		redis_endpoint := fmt.Sprintf("%s:%d", *redis_host, *redis_port)

		redis_client := redis.NewTCPClient(&redis.Options{
			Addr: redis_endpoint,
		})

		defer redis_client.Close()

		redis_client.Publish(*redis_channel, b.String())
	}

	// log.Printf("%s", files)
}
<|endoftext|>


<commit_before>// Copyright 2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"flag"
	"fmt"
	"strconv"

	"golang.org/x/sys/unix"
)

const defaultPerms = 0660

func parseDevices(args []string, devtype string) (int, error) {
	if len(args) != 4 {
		return 0, fmt.Errorf("device type %v requires a major and minor number", devtype)
	}
	major, err := strconv.ParseUint(args[2], 10, 8)
	if err != nil {
		return 0, err
	}
	minor, err := strconv.ParseUint(args[3], 10, 8)
	if err != nil {
		return 0, err
	}
	return int((major << 8) | minor), nil
}

func mknod() error {
	flag.Parse()
	a := flag.Args()
	if len(a) != 2 && len(a) != 4 {
		return errors.New("Usage: mknod path type [major minor]")
	}
	path := a[0]
	devtype := a[1]

	var err error
	var mode uint32

	mode = defaultPerms
	var dev int

	switch devtype {
	case "b":
		// This is a block device. A major/minor number is needed.
		mode |= unix.S_IFBLK
		dev, err = parseDevices(a, devtype)
		if err != nil {
			return err
		}
	case "c", "u":
		// This is a character/unbuffered device. A major/minor number is needed.
		mode |= unix.S_IFCHR
		dev, err = parseDevices(a, devtype)
		if err != nil {
			return err
		}
	case "p":
		// This is a pipe. A major and minor number must not be supplied
		mode |= unix.S_IFIFO
		if len(a) != 2 {
			return fmt.Errorf("device type %v requires no other arguments", devtype)
		}
	default:
		return fmt.Errorf("device type not recognized: %v", devtype)
	}

	if err := unix.Mknod(path, mode, dev); err != nil {
		return fmt.Errorf("mknod :%s: mode %x: %v", path, mode, err)
	}
	return nil
}
<commit_msg>mknod: fix bits for major and minor<commit_after>// Copyright 2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"flag"
	"fmt"
	"strconv"

	"golang.org/x/sys/unix"
)

const defaultPerms = 0660

func parseDevices(args []string, devtype string) (int, error) {
	if len(args) != 4 {
		return 0, fmt.Errorf("device type %v requires a major and minor number", devtype)
	}
	major, err := strconv.ParseUint(args[2], 10, 12)
	if err != nil {
		return 0, err
	}
	minor, err := strconv.ParseUint(args[3], 10, 20)
	if err != nil {
		return 0, err
	}
	return int((major << 8) | minor), nil
}

func mknod() error {
	flag.Parse()
	a := flag.Args()
	if len(a) != 2 && len(a) != 4 {
		return errors.New("Usage: mknod path type [major minor]")
	}
	path := a[0]
	devtype := a[1]

	var err error
	var mode uint32

	mode = defaultPerms
	var dev int

	switch devtype {
	case "b":
		// This is a block device. A major/minor number is needed.
		mode |= unix.S_IFBLK
		dev, err = parseDevices(a, devtype)
		if err != nil {
			return err
		}
	case "c", "u":
		// This is a character/unbuffered device. A major/minor number is needed.
		mode |= unix.S_IFCHR
		dev, err = parseDevices(a, devtype)
		if err != nil {
			return err
		}
	case "p":
		// This is a pipe. A major and minor number must not be supplied
		mode |= unix.S_IFIFO
		if len(a) != 2 {
			return fmt.Errorf("device type %v requires no other arguments", devtype)
		}
	default:
		return fmt.Errorf("device type not recognized: %v", devtype)
	}

	if err := unix.Mknod(path, mode, dev); err != nil {
		return fmt.Errorf("mknod :%s: mode %x: %v", path, mode, err)
	}
	return nil
}
<|endoftext|>