<commit_before>package gorethink

import (
	"fmt"
	"strconv"
	"strings"

	p "github.com/dancannon/gorethink/ql2"
)

type Query struct {
	Type  p.Query_QueryType
	Token int64
	Term  *Term
	Opts  map[string]interface{}
}

func (q *Query) build() []interface{} {
	res := []interface{}{int(q.Type)}
	if q.Term != nil {
		res = append(res, q.Term.build())
	}

	if len(q.Opts) > 0 {
		res = append(res, q.Opts)
	}

	return res
}

type termsList []Term
type termsObj map[string]Term
type Term struct {
	name     string
	rootTerm bool
	termType p.Term_TermType
	data     interface{}
	args     []Term
	optArgs  map[string]Term
}

// build takes the query tree and prepares it to be sent as a JSON
// expression
func (t Term) build() interface{} {
	switch t.termType {
	case p.Term_DATUM:
		return t.data
	case p.Term_MAKE_OBJ:
		res := map[string]interface{}{}
		for k, v := range t.optArgs {
			res[k] = v.build()
		}
		return res
	case p.Term_BINARY:
		if len(t.args) == 0 {
			return map[string]interface{}{
				"$reql_type$": "BINARY",
				"data":        t.data,
			}
		}
	}

	args := []interface{}{}
	optArgs := map[string]interface{}{}

	for _, v := range t.args {
		args = append(args, v.build())
	}

	for k, v := range t.optArgs {
		optArgs[k] = v.build()
	}

	ret := []interface{}{int(t.termType)}

	if len(args) > 0 {
		ret = append(ret, args)
	}
	if len(optArgs) > 0 {
		ret = append(ret, optArgs)
	}

	return ret
}

// String returns a string representation of the query tree
func (t Term) String() string {
	switch t.termType {
	case p.Term_MAKE_ARRAY:
		return fmt.Sprintf("[%s]", strings.Join(argsToStringSlice(t.args), ", "))
	case p.Term_MAKE_OBJ:
		return fmt.Sprintf("{%s}", strings.Join(optArgsToStringSlice(t.optArgs), ", "))
	case p.Term_FUNC:
		// Get string representation of each argument
		args := []string{}
		for _, v := range t.args[0].args {
			args = append(args, fmt.Sprintf("var_%d", v.data))
		}

		return fmt.Sprintf("func(%s r.Term) r.Term { return %s }",
			strings.Join(args, ", "),
			t.args[1].String(),
		)
	case p.Term_VAR:
		return fmt.Sprintf("var_%s", t.args[0])
	case p.Term_IMPLICIT_VAR:
		return "r.Row"
	case p.Term_DATUM:
		switch v := t.data.(type) {
		case string:
			return strconv.Quote(v)
		default:
			return fmt.Sprintf("%v", v)
		}
	case p.Term_BINARY:
		if len(t.args) == 0 {
			return fmt.Sprintf("r.binary(<data>)")
		}
	}

	if t.rootTerm {
		return fmt.Sprintf("r.%s(%s)", t.name, strings.Join(allArgsToStringSlice(t.args, t.optArgs), ", "))
	}
	return fmt.Sprintf("%s.%s(%s)", t.args[0].String(), t.name, strings.Join(allArgsToStringSlice(t.args[1:], t.optArgs), ", "))
}

type OptArgs interface {
	toMap() map[string]interface{}
}

type WriteResponse struct {
	Errors        int            `gorethink:"errors"`
	Inserted      int            `gorethink:"inserted"`
	Updated       int            `gorethink:"updadte"`
	Unchanged     int            `gorethink:"unchanged"`
	Replaced      int            `gorethink:"replaced"`
	Renamed       int            `gorethink:"renamed"`
	Skipped       int            `gorethink:"skipped"`
	Deleted       int            `gorethink:"deleted"`
	Created       int            `gorethink:"created"`
	DBsCreated    int            `gorethink:"dbs_created"`
	TablesCreated int            `gorethink:"tables_created"`
	Dropped       int            `gorethink:"dropped"`
	DBsDropped    int            `gorethink:"dbs_dropped"`
	TablesDropped int            `gorethink:"tables_dropped"`
	GeneratedKeys []string       `gorethink:"generated_keys"`
	FirstError    string         `gorethink:"first_error"` // populated if Errors > 0
	ConfigChanges []WriteChanges `gorethink:"config_changes"`
	Changes       []WriteChanges
}

type WriteChanges struct {
	NewValue interface{} `gorethink:"new_val"`
	OldValue interface{} `gorethink:"old_val"`
}

type RunOpts struct {
	Db             interface{} `gorethink:"db,omitempty"`
	Profile        interface{} `gorethink:"profile,omitempty"`
	UseOutdated    interface{} `gorethink:"use_outdated,omitempty"`
	ArrayLimit     interface{} `gorethink:"array_limit,omitempty"`
	TimeFormat     interface{} `gorethink:"time_format,omitempty"`
	GroupFormat    interface{} `gorethink:"group_format,omitempty"`
	BinaryFormat   interface{} `gorethink:"binary_format,omitempty"`
	GeometryFormat interface{} `gorethink:"geometry_format,omitempty"`

	MinBatchRows              interface{} `gorethink:"min_batch_rows,omitempty"`
	MaxBatchRows              interface{} `gorethink:"max_batch_rows,omitempty"`
	MaxBatchBytes             interface{} `gorethink:"max_batch_bytes,omitempty"`
	MaxBatchSeconds           interface{} `gorethink:"max_batch_seconds,omitempty"`
	FirstBatchScaledownFactor interface{} `gorethink:"first_batch_scaledown_factor,omitempty"`
}

func (o *RunOpts) toMap() map[string]interface{} {
	return optArgsToMap(o)
}

// Run runs a query using the given connection.
//
//	rows, err := query.Run(sess)
//	if err != nil {
//		// error
//	}
//
//  var doc MyDocumentType
//	for rows.Next(&doc) {
//      // Do something with document
//	}
func (t Term) Run(s *Session, optArgs ...RunOpts) (*Cursor, error) {
	opts := map[string]interface{}{}
	if len(optArgs) >= 1 {
		opts = optArgs[0].toMap()
	}

	return s.Query(s.newQuery(t, opts))
}

// RunWrite runs a query using the given connection but unlike Run automatically
// scans the result into a variable of type WriteResponss. This function should be used
// if you are running a write query (such as Insert,  Update, TableCreate, etc...)
//
//	res, err := r.Db("database").Table("table").Insert(doc).RunWrite(sess)
func (t Term) RunWrite(s *Session, optArgs ...RunOpts) (WriteResponse, error) {
	var response WriteResponse
	res, err := t.Run(s, optArgs...)
	if err == nil {
		err = res.One(&response)
	}
	return response, err
}

// ExecOpts inherits its options from RunOpts, the only difference is the
// addition of the NoReply field.
//
// When NoReply is true it causes the driver not to wait to receive the result
// and return immediately.
type ExecOpts struct {
	Db             interface{} `gorethink:"db,omitempty"`
	Profile        interface{} `gorethink:"profile,omitempty"`
	UseOutdated    interface{} `gorethink:"use_outdated,omitempty"`
	ArrayLimit     interface{} `gorethink:"array_limit,omitempty"`
	TimeFormat     interface{} `gorethink:"time_format,omitempty"`
	GroupFormat    interface{} `gorethink:"group_format,omitempty"`
	BinaryFormat   interface{} `gorethink:"binary_format,omitempty"`
	GeometryFormat interface{} `gorethink:"geometry_format,omitempty"`

	MinBatchRows              interface{} `gorethink:"min_batch_rows,omitempty"`
	MaxBatchRows              interface{} `gorethink:"max_batch_rows,omitempty"`
	MaxBatchBytes             interface{} `gorethink:"max_batch_bytes,omitempty"`
	MaxBatchSeconds           interface{} `gorethink:"max_batch_seconds,omitempty"`
	FirstBatchScaledownFactor interface{} `gorethink:"first_batch_scaledown_factor,omitempty"`

	NoReply interface{} `gorethink:"noreply,omitempty"`
}

func (o *ExecOpts) toMap() map[string]interface{} {
	return optArgsToMap(o)
}

// Exec runs the query but does not return the result. Exec will still wait for
// the response to be received unless the NoReply field is true.
//
//	res, err := r.Db("database").Table("table").Insert(doc).Exec(sess, r.ExecOpts{
//		NoReply: true,
//	})
func (t Term) Exec(s *Session, optArgs ...ExecOpts) error {
	opts := map[string]interface{}{}
	if len(optArgs) >= 1 {
		opts = optArgs[0].toMap()
	}

	return s.Exec(s.newQuery(t, opts))
}
<commit_msg>Fix documentation on func (Term) Exec<commit_after>package gorethink

import (
	"fmt"
	"strconv"
	"strings"

	p "github.com/dancannon/gorethink/ql2"
)

type Query struct {
	Type  p.Query_QueryType
	Token int64
	Term  *Term
	Opts  map[string]interface{}
}

func (q *Query) build() []interface{} {
	res := []interface{}{int(q.Type)}
	if q.Term != nil {
		res = append(res, q.Term.build())
	}

	if len(q.Opts) > 0 {
		res = append(res, q.Opts)
	}

	return res
}

type termsList []Term
type termsObj map[string]Term
type Term struct {
	name     string
	rootTerm bool
	termType p.Term_TermType
	data     interface{}
	args     []Term
	optArgs  map[string]Term
}

// build takes the query tree and prepares it to be sent as a JSON
// expression
func (t Term) build() interface{} {
	switch t.termType {
	case p.Term_DATUM:
		return t.data
	case p.Term_MAKE_OBJ:
		res := map[string]interface{}{}
		for k, v := range t.optArgs {
			res[k] = v.build()
		}
		return res
	case p.Term_BINARY:
		if len(t.args) == 0 {
			return map[string]interface{}{
				"$reql_type$": "BINARY",
				"data":        t.data,
			}
		}
	}

	args := []interface{}{}
	optArgs := map[string]interface{}{}

	for _, v := range t.args {
		args = append(args, v.build())
	}

	for k, v := range t.optArgs {
		optArgs[k] = v.build()
	}

	ret := []interface{}{int(t.termType)}

	if len(args) > 0 {
		ret = append(ret, args)
	}
	if len(optArgs) > 0 {
		ret = append(ret, optArgs)
	}

	return ret
}

// String returns a string representation of the query tree
func (t Term) String() string {
	switch t.termType {
	case p.Term_MAKE_ARRAY:
		return fmt.Sprintf("[%s]", strings.Join(argsToStringSlice(t.args), ", "))
	case p.Term_MAKE_OBJ:
		return fmt.Sprintf("{%s}", strings.Join(optArgsToStringSlice(t.optArgs), ", "))
	case p.Term_FUNC:
		// Get string representation of each argument
		args := []string{}
		for _, v := range t.args[0].args {
			args = append(args, fmt.Sprintf("var_%d", v.data))
		}

		return fmt.Sprintf("func(%s r.Term) r.Term { return %s }",
			strings.Join(args, ", "),
			t.args[1].String(),
		)
	case p.Term_VAR:
		return fmt.Sprintf("var_%s", t.args[0])
	case p.Term_IMPLICIT_VAR:
		return "r.Row"
	case p.Term_DATUM:
		switch v := t.data.(type) {
		case string:
			return strconv.Quote(v)
		default:
			return fmt.Sprintf("%v", v)
		}
	case p.Term_BINARY:
		if len(t.args) == 0 {
			return fmt.Sprintf("r.binary(<data>)")
		}
	}

	if t.rootTerm {
		return fmt.Sprintf("r.%s(%s)", t.name, strings.Join(allArgsToStringSlice(t.args, t.optArgs), ", "))
	}
	return fmt.Sprintf("%s.%s(%s)", t.args[0].String(), t.name, strings.Join(allArgsToStringSlice(t.args[1:], t.optArgs), ", "))
}

type OptArgs interface {
	toMap() map[string]interface{}
}

type WriteResponse struct {
	Errors        int            `gorethink:"errors"`
	Inserted      int            `gorethink:"inserted"`
	Updated       int            `gorethink:"updadte"`
	Unchanged     int            `gorethink:"unchanged"`
	Replaced      int            `gorethink:"replaced"`
	Renamed       int            `gorethink:"renamed"`
	Skipped       int            `gorethink:"skipped"`
	Deleted       int            `gorethink:"deleted"`
	Created       int            `gorethink:"created"`
	DBsCreated    int            `gorethink:"dbs_created"`
	TablesCreated int            `gorethink:"tables_created"`
	Dropped       int            `gorethink:"dropped"`
	DBsDropped    int            `gorethink:"dbs_dropped"`
	TablesDropped int            `gorethink:"tables_dropped"`
	GeneratedKeys []string       `gorethink:"generated_keys"`
	FirstError    string         `gorethink:"first_error"` // populated if Errors > 0
	ConfigChanges []WriteChanges `gorethink:"config_changes"`
	Changes       []WriteChanges
}

type WriteChanges struct {
	NewValue interface{} `gorethink:"new_val"`
	OldValue interface{} `gorethink:"old_val"`
}

type RunOpts struct {
	Db             interface{} `gorethink:"db,omitempty"`
	Profile        interface{} `gorethink:"profile,omitempty"`
	UseOutdated    interface{} `gorethink:"use_outdated,omitempty"`
	ArrayLimit     interface{} `gorethink:"array_limit,omitempty"`
	TimeFormat     interface{} `gorethink:"time_format,omitempty"`
	GroupFormat    interface{} `gorethink:"group_format,omitempty"`
	BinaryFormat   interface{} `gorethink:"binary_format,omitempty"`
	GeometryFormat interface{} `gorethink:"geometry_format,omitempty"`

	MinBatchRows              interface{} `gorethink:"min_batch_rows,omitempty"`
	MaxBatchRows              interface{} `gorethink:"max_batch_rows,omitempty"`
	MaxBatchBytes             interface{} `gorethink:"max_batch_bytes,omitempty"`
	MaxBatchSeconds           interface{} `gorethink:"max_batch_seconds,omitempty"`
	FirstBatchScaledownFactor interface{} `gorethink:"first_batch_scaledown_factor,omitempty"`
}

func (o *RunOpts) toMap() map[string]interface{} {
	return optArgsToMap(o)
}

// Run runs a query using the given connection.
//
//	rows, err := query.Run(sess)
//	if err != nil {
//		// error
//	}
//
//  var doc MyDocumentType
//	for rows.Next(&doc) {
//      // Do something with document
//	}
func (t Term) Run(s *Session, optArgs ...RunOpts) (*Cursor, error) {
	opts := map[string]interface{}{}
	if len(optArgs) >= 1 {
		opts = optArgs[0].toMap()
	}

	return s.Query(s.newQuery(t, opts))
}

// RunWrite runs a query using the given connection but unlike Run automatically
// scans the result into a variable of type WriteResponss. This function should be used
// if you are running a write query (such as Insert,  Update, TableCreate, etc...)
//
//	res, err := r.Db("database").Table("table").Insert(doc).RunWrite(sess)
func (t Term) RunWrite(s *Session, optArgs ...RunOpts) (WriteResponse, error) {
	var response WriteResponse
	res, err := t.Run(s, optArgs...)
	if err == nil {
		err = res.One(&response)
	}
	return response, err
}

// ExecOpts inherits its options from RunOpts, the only difference is the
// addition of the NoReply field.
//
// When NoReply is true it causes the driver not to wait to receive the result
// and return immediately.
type ExecOpts struct {
	Db             interface{} `gorethink:"db,omitempty"`
	Profile        interface{} `gorethink:"profile,omitempty"`
	UseOutdated    interface{} `gorethink:"use_outdated,omitempty"`
	ArrayLimit     interface{} `gorethink:"array_limit,omitempty"`
	TimeFormat     interface{} `gorethink:"time_format,omitempty"`
	GroupFormat    interface{} `gorethink:"group_format,omitempty"`
	BinaryFormat   interface{} `gorethink:"binary_format,omitempty"`
	GeometryFormat interface{} `gorethink:"geometry_format,omitempty"`

	MinBatchRows              interface{} `gorethink:"min_batch_rows,omitempty"`
	MaxBatchRows              interface{} `gorethink:"max_batch_rows,omitempty"`
	MaxBatchBytes             interface{} `gorethink:"max_batch_bytes,omitempty"`
	MaxBatchSeconds           interface{} `gorethink:"max_batch_seconds,omitempty"`
	FirstBatchScaledownFactor interface{} `gorethink:"first_batch_scaledown_factor,omitempty"`

	NoReply interface{} `gorethink:"noreply,omitempty"`
}

func (o *ExecOpts) toMap() map[string]interface{} {
	return optArgsToMap(o)
}

// Exec runs the query but does not return the result. Exec will still wait for
// the response to be received unless the NoReply field is true.
//
//	err := r.Db("database").Table("table").Insert(doc).Exec(sess, r.ExecOpts{
//		NoReply: true,
//	})
func (t Term) Exec(s *Session, optArgs ...ExecOpts) error {
	opts := map[string]interface{}{}
	if len(optArgs) >= 1 {
		opts = optArgs[0].toMap()
	}

	return s.Exec(s.newQuery(t, opts))
}
<|endoftext|>


<commit_before>package events

import (
	"container/list"
	"sync"

	"github.com/Sirupsen/logrus"
)

// Queue accepts all messages into a queue for asynchronous consumption
// by a sink. It is unbounded and thread safe but the sink must be reliable or
// events will be dropped.
type Queue struct {
	dst    Sink
	events *list.List
	cond   *sync.Cond
	mu     sync.Mutex
	closed bool
}

// NewQueue returns a queue to the provided Sink dst.
func NewQueue(dst Sink) *Queue {
	eq := Queue{
		dst:    dst,
		events: list.New(),
	}

	eq.cond = sync.NewCond(&eq.mu)
	go eq.run()
	return &eq
}

// Write accepts the events into the queue, only failing if the queue has
// beend closed.
func (eq *Queue) Write(event Event) error {
	eq.mu.Lock()
	defer eq.mu.Unlock()

	if eq.closed {
		return ErrSinkClosed
	}

	eq.events.PushBack(event)
	eq.cond.Signal() // signal waiters

	return nil
}

// Close shutsdown the event queue, flushing
func (eq *Queue) Close() error {
	eq.mu.Lock()
	defer eq.mu.Unlock()

	if eq.closed {
		return ErrSinkClosed
	}

	// set closed flag
	eq.closed = true
	eq.cond.Signal() // signal flushes queue
	eq.cond.Wait()   // wait for signal from last flush
	return eq.dst.Close()
}

// run is the main goroutine to flush events to the target sink.
func (eq *Queue) run() {
	for {
		event := eq.next()

		if event == nil {
			return // nil block means event queue is closed.
		}

		if err := eq.dst.Write(event); err != nil {
			logrus.WithFields(logrus.Fields{
				"event": event,
				"sink":  eq.dst,
			}).WithError(err).Warnf("eventqueue: dropped event")
		}
	}
}

// next encompasses the critical section of the run loop. When the queue is
// empty, it will block on the condition. If new data arrives, it will wake
// and return a block. When closed, a nil slice will be returned.
func (eq *Queue) next() Event {
	eq.mu.Lock()
	defer eq.mu.Unlock()

	for eq.events.Len() < 1 {
		if eq.closed {
			eq.cond.Broadcast()
			return nil
		}

		eq.cond.Wait()
	}

	front := eq.events.Front()
	block := front.Value.(Event)
	eq.events.Remove(front)

	return block
}
<commit_msg>Change "dropped events" warning to debug-level log message<commit_after>package events

import (
	"container/list"
	"sync"

	"github.com/Sirupsen/logrus"
)

// Queue accepts all messages into a queue for asynchronous consumption
// by a sink. It is unbounded and thread safe but the sink must be reliable or
// events will be dropped.
type Queue struct {
	dst    Sink
	events *list.List
	cond   *sync.Cond
	mu     sync.Mutex
	closed bool
}

// NewQueue returns a queue to the provided Sink dst.
func NewQueue(dst Sink) *Queue {
	eq := Queue{
		dst:    dst,
		events: list.New(),
	}

	eq.cond = sync.NewCond(&eq.mu)
	go eq.run()
	return &eq
}

// Write accepts the events into the queue, only failing if the queue has
// beend closed.
func (eq *Queue) Write(event Event) error {
	eq.mu.Lock()
	defer eq.mu.Unlock()

	if eq.closed {
		return ErrSinkClosed
	}

	eq.events.PushBack(event)
	eq.cond.Signal() // signal waiters

	return nil
}

// Close shutsdown the event queue, flushing
func (eq *Queue) Close() error {
	eq.mu.Lock()
	defer eq.mu.Unlock()

	if eq.closed {
		return ErrSinkClosed
	}

	// set closed flag
	eq.closed = true
	eq.cond.Signal() // signal flushes queue
	eq.cond.Wait()   // wait for signal from last flush
	return eq.dst.Close()
}

// run is the main goroutine to flush events to the target sink.
func (eq *Queue) run() {
	for {
		event := eq.next()

		if event == nil {
			return // nil block means event queue is closed.
		}

		if err := eq.dst.Write(event); err != nil {
			// TODO(aaronl): Dropping events could be bad depending
			// on the application. We should have a way of
			// communicating this condition. However, logging
			// at a log level above debug may not be appropriate.
			// Eventually, go-events should not use logrus at all,
			// and should bubble up conditions like this through
			// error values.
			logrus.WithFields(logrus.Fields{
				"event": event,
				"sink":  eq.dst,
			}).WithError(err).Debug("eventqueue: dropped event")
		}
	}
}

// next encompasses the critical section of the run loop. When the queue is
// empty, it will block on the condition. If new data arrives, it will wake
// and return a block. When closed, a nil slice will be returned.
func (eq *Queue) next() Event {
	eq.mu.Lock()
	defer eq.mu.Unlock()

	for eq.events.Len() < 1 {
		if eq.closed {
			eq.cond.Broadcast()
			return nil
		}

		eq.cond.Wait()
	}

	front := eq.events.Front()
	block := front.Value.(Event)
	eq.events.Remove(front)

	return block
}
<|endoftext|>


<commit_before>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package index

import (
	"fmt"
	"sort"
	"strings"
	"unicode"

	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
)

// Path separator is a string used to separate path components.
const PathSeparator string = ":"

// Path is type which represents a position in the index heirarchy.  Each level has a key, and so the path
// is a slice of strings where each element is the key of some index element (group or track).
type Path []Key

// String implements Stringer.
func (p Path) String() string {
	return p.Encode()
}

// Encode returns a string representation of the Path.
func (p Path) Encode() string {
	s := ""
	if len(p) == 0 {
		return s
	}
	for _, k := range p[:len(p)-1] {
		s += string(k) + PathSeparator
	}
	s += string(p[len(p)-1])
	return s
}

// PathSlice is a wrapper type implementing sort.Interface (and index.Swapper)
type PathSlice []Path

// Swap implements sort.Interface (and index.Swapper).
func (p PathSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

// Less implements sort.Interface.
func (p PathSlice) Less(i, j int) bool { return p[i].Encode() < p[j].Encode() }

// Len implements sort.Interface.
func (p PathSlice) Len() int { return len(p) }

func isMn(r rune) bool {
	return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks
}

var transformer = transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC)

func removeNonAlphaNumeric(s string) string {
	in := []rune(s)
	res := make([]rune, len(in))
	i := 0
	for _, x := range s {
		if x == '-' {
			res[i] = ' '
			i++
			continue
		}
		if unicode.IsLetter(x) || unicode.IsDigit(x) || unicode.IsSpace(x) {
			res[i] = unicode.ToLower(x)
			i++
		}
	}
	result, _, _ := transform.Bytes(transformer, []byte(string(res[:i])))
	return string(result)
}

// Searcher is an interface which defines the Search method.
type Searcher interface {
	// Search uses the given string to filter a list of paths.
	Search(string) []Path
}

// WordIndex is an interface which defines the Words method.
type WordIndex interface {
	// Words returns all the words in the index.
	Words() []string
}

type wordIndex struct {
	fields []string
	words  map[string][]Path // mapping from word -> paths
}

func (s *wordIndex) Words() []string {
	words := make([]string, 0, len(s.words))
	for k := range s.words {
		words = append(words, k)
	}
	return words
}

// Search returns a list of paths for the given word.
func (s *wordIndex) Search(w string) []Path {
	return s.words[w]
}

// AddWord adds the given string (assumed to be a single word) to the index.
func (s *wordIndex) AddWord(w string, p Path) {
	s.words[w] = append(s.words[w], p)
}

// AddString adds the string to the index, removing non-alphanumeric characters,
// normalising modified characters, and splitting into words.
func (s *wordIndex) AddString(x string, p Path) {
	x = removeNonAlphaNumeric(x)
	w := strings.Fields(x)
	for _, x := range w {
		s.AddWord(x, p)
	}
}

func (s *wordIndex) AddCollection(c Collection, p Path) {
	for _, k := range c.Keys() {
		np := make(Path, len(p), len(p)+1)
		copy(np, p)
		np = append(np, k)
		s.AddGroup(c.Get(k), np)
	}
}

// AddGroup adds the given group to the word index, using the Path as root
func (s *wordIndex) AddGroup(g Group, p Path) {
	if c, ok := g.(Collection); ok {
		s.AddCollection(c, p)
		return
	}
	// for i, t := range g.Tracks() {
	for _, t := range g.Tracks() {
		// np := make(Path, len(p), len(p)+1)
		// copy(p, np)
		// np = append(np, strconv.Itoa(i))
		for _, f := range s.fields {
			// s.AddString(t.GetString(f), np)
			s.AddString(t.GetString(f), p)
		}
	}
}

// Expander is an interface which implements the Expand method.
type Expander interface {
	// Expand the given string, returning the result and true if succesful, or false
	// otherwise.
	Expand(string) ([]string, bool)
}

// MinPrefix is the minimum number of characters that can be used in a prefix.
const MinPrefix = 3

// PrefixMultiExpand is a type which implements Expander
type PrefixMultiExpand struct {
	words map[string][]string
	size  int
}

// BuildPrefixMultiExpander builds an Expander with given length n.  All words mapped to by prefixes will
// have length greater than or equal to MinPrefix.
func BuildPrefixMultiExpander(words []string, n int) PrefixMultiExpand {
	if n < MinPrefix {
		panic(fmt.Sprintf("Size must be greater than MinPrefix (%d)", MinPrefix))
	}

	m := make(map[string][]string)
	for _, w := range words {
		if len(w) >= MinPrefix {
			last := n
			if len(w) <= n {
				last = len(w)
			}
			for i := MinPrefix; i <= last; i++ {
				m[w[:i]] = append(m[w[:i]], w)
			}
		}
	}

	return PrefixMultiExpand{
		words: m,
		size:  n,
	}
}

// Expand uses the prefix mapping to return a list of words which can be expanded from s.
func (p PrefixMultiExpand) Expand(s string) ([]string, bool) {
	if len(s) < MinPrefix {
		return nil, false
	}
	if len(s) > p.size {
		// TODO: filter this with edit distance
		return p.words[s[:p.size]], true
	}
	return p.words[s], true
}

// expandSearcher is an implementation of Searcher which applies the Expander to search
// input and then performs a search on each of the expanded outputs, union the results
// and returns as the Search result.
type expandSearcher struct {
	Expander
	Searcher
}

// Search implements Searcher, and uses the internal Expander to expand all words
// in the search expression and unions the results (deduping). Returns nil when
// the search term isn't above the MinPrefix.
func (es *expandSearcher) Search(s string) []Path {
	e, ok := es.Expander.Expand(s)
	if !ok {
		return nil
	}
	ps := make([][]Path, len(e))
	for i, w := range e {
		ps[i] = es.Searcher.Search(w)
	}
	return Union(ps...)
}

// BuildPrefixExpandSearcher constructs a prefix expander which wraps the given Searcher
// by expanding each word in the search input using the WordIndex.
func BuildPrefixExpandSearcher(s Searcher, w WordIndex, n int) Searcher {
	return &expandSearcher{BuildPrefixMultiExpander(w.Words(), n), s}
}

// BuildWordIndex creates a *wordIndex ()
func BuildWordIndex(c Collection, fields []string) *wordIndex {
	wi := &wordIndex{
		fields: fields,
		words:  make(map[string][]Path),
	}
	wi.AddGroup(c, Path([]Key{"Root"}))
	return wi
}

type wordSearchIntersect struct {
	Searcher
	min int // the minimum input string length before search returns something non-trivial.
}

func (s *wordSearchIntersect) Search(x string) []Path {
	if len(x) < s.min {
		return make([]Path, 0)
	}
	words := strings.Fields(strings.ToLower(x))
	paths := make([][]Path, 0, len(words))
	for _, w := range words {
		if len(w) >= s.min {
			paths = append(paths, s.Searcher.Search(w))
		}
	}
	return OrderedIntersection(paths...)
}

// FlatSearcher is a Searcher wrapper which flattens input strings (replaces any accented
// characters with their un-accented equivalents).
type FlatSearcher struct {
	Searcher
}

func (f FlatSearcher) Search(s string) []Path {
	return f.Searcher.Search(removeNonAlphaNumeric(s))
}

// WordsSearchIntersect calls Search on the Searcher for each word in the input string
// and then returns the ordered intersection (Paths are ordered by the number of times
// they appear.
func WordsIntersectSearcher(s Searcher) Searcher {
	return &wordSearchIntersect{
		Searcher: s,
		min:      3,
	}
}

type countKeyPath struct {
	n int
	k string
}

type countKeyPathSlice []countKeyPath

func (c countKeyPathSlice) Len() int      { return len(c) }
func (c countKeyPathSlice) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
func (c countKeyPathSlice) Less(i, j int) bool {
	if c[i].n < c[j].n {
		return true
	}
	if c[i].n > c[j].n {
		return false
	}
	return c[i].k < c[j].k
}

// Compute the intersection of the given lists of paths.
func OrderedIntersection(paths ...[]Path) []Path {
	if len(paths) == 0 {
		return []Path{}
	}

	enc := make(map[string]Path) // Encoding -> Path
	set := make(map[string]bool) // Set of Encoding
	cnt := make(map[string]int)

	for _, v := range paths[0] {
		e := v.Encode()
		set[e] = true
		enc[e] = v
		cnt[e]++
	}

	if len(paths) > 1 {
		for _, list := range paths[1:] {
			miss := make(map[string]bool)
			for k := range set {
				miss[k] = true
			}

			for _, v := range list {
				e := v.Encode()
				if set[e] {
					delete(miss, e)
					cnt[e]++
				}
			}

			for k := range miss {
				delete(set, k)
				delete(cnt, k)
			}
		}
	}

	result := make([]Path, 0, len(cnt))
	count := make([]countKeyPath, 0, len(cnt))
	for k, v := range cnt {
		result = append(result, enc[k])
		count = append(count, countKeyPath{v, k})
	}

	sort.Sort(ParallelSort(sort.Reverse(countKeyPathSlice(count)), PathSlice(result)))
	return result
}

// Union returns a []Path which is the union (deduped) of the given slices of []Path.
func Union(l ...[]Path) []Path {
	if len(l) == 0 {
		return []Path{}
	}

	done := make(map[string]bool)
	res := make([]Path, 0)
	for _, paths := range l {
		for _, path := range paths {
			e := path.Encode()
			if !done[e] {
				done[e] = true
				res = append(res, path)
			}
		}
	}
	return res
}
<commit_msg>Refactor countKeyPath -> stringFreq.<commit_after>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package index

import (
	"fmt"
	"sort"
	"strings"
	"unicode"

	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
)

// Path separator is a string used to separate path components.
const PathSeparator string = ":"

// Path is type which represents a position in the index heirarchy.  Each level has a key, and so the path
// is a slice of strings where each element is the key of some index element (group or track).
type Path []Key

// String implements Stringer.
func (p Path) String() string {
	return p.Encode()
}

// Encode returns a string representation of the Path.
func (p Path) Encode() string {
	s := ""
	if len(p) == 0 {
		return s
	}
	for _, k := range p[:len(p)-1] {
		s += string(k) + PathSeparator
	}
	s += string(p[len(p)-1])
	return s
}

// PathSlice is a wrapper type implementing sort.Interface (and index.Swapper)
type PathSlice []Path

// Swap implements sort.Interface (and index.Swapper).
func (p PathSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

// Less implements sort.Interface.
func (p PathSlice) Less(i, j int) bool { return p[i].Encode() < p[j].Encode() }

// Len implements sort.Interface.
func (p PathSlice) Len() int { return len(p) }

func isMn(r rune) bool {
	return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks
}

var transformer = transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC)

func removeNonAlphaNumeric(s string) string {
	in := []rune(s)
	res := make([]rune, len(in))
	i := 0
	for _, x := range s {
		if x == '-' {
			res[i] = ' '
			i++
			continue
		}
		if unicode.IsLetter(x) || unicode.IsDigit(x) || unicode.IsSpace(x) {
			res[i] = unicode.ToLower(x)
			i++
		}
	}
	result, _, _ := transform.Bytes(transformer, []byte(string(res[:i])))
	return string(result)
}

// Searcher is an interface which defines the Search method.
type Searcher interface {
	// Search uses the given string to filter a list of paths.
	Search(string) []Path
}

// WordIndex is an interface which defines the Words method.
type WordIndex interface {
	// Words returns all the words in the index.
	Words() []string
}

type wordIndex struct {
	fields []string
	words  map[string][]Path // mapping from word -> paths
}

func (s *wordIndex) Words() []string {
	words := make([]string, 0, len(s.words))
	for k := range s.words {
		words = append(words, k)
	}
	return words
}

// Search returns a list of paths for the given word.
func (s *wordIndex) Search(w string) []Path {
	return s.words[w]
}

// AddWord adds the given string (assumed to be a single word) to the index.
func (s *wordIndex) AddWord(w string, p Path) {
	s.words[w] = append(s.words[w], p)
}

// AddString adds the string to the index, removing non-alphanumeric characters,
// normalising modified characters, and splitting into words.
func (s *wordIndex) AddString(x string, p Path) {
	x = removeNonAlphaNumeric(x)
	w := strings.Fields(x)
	for _, x := range w {
		s.AddWord(x, p)
	}
}

func (s *wordIndex) AddCollection(c Collection, p Path) {
	for _, k := range c.Keys() {
		np := make(Path, len(p), len(p)+1)
		copy(np, p)
		np = append(np, k)
		s.AddGroup(c.Get(k), np)
	}
}

// AddGroup adds the given group to the word index, using the Path as root
func (s *wordIndex) AddGroup(g Group, p Path) {
	if c, ok := g.(Collection); ok {
		s.AddCollection(c, p)
		return
	}
	// for i, t := range g.Tracks() {
	for _, t := range g.Tracks() {
		// np := make(Path, len(p), len(p)+1)
		// copy(p, np)
		// np = append(np, strconv.Itoa(i))
		for _, f := range s.fields {
			// s.AddString(t.GetString(f), np)
			s.AddString(t.GetString(f), p)
		}
	}
}

// Expander is an interface which implements the Expand method.
type Expander interface {
	// Expand the given string, returning the result and true if succesful, or false
	// otherwise.
	Expand(string) ([]string, bool)
}

// MinPrefix is the minimum number of characters that can be used in a prefix.
const MinPrefix = 3

// PrefixMultiExpand is a type which implements Expander
type PrefixMultiExpand struct {
	words map[string][]string
	size  int
}

// BuildPrefixMultiExpander builds an Expander with given length n.  All words mapped to by prefixes will
// have length greater than or equal to MinPrefix.
func BuildPrefixMultiExpander(words []string, n int) PrefixMultiExpand {
	if n < MinPrefix {
		panic(fmt.Sprintf("Size must be greater than MinPrefix (%d)", MinPrefix))
	}

	m := make(map[string][]string)
	for _, w := range words {
		if len(w) >= MinPrefix {
			last := n
			if len(w) <= n {
				last = len(w)
			}
			for i := MinPrefix; i <= last; i++ {
				m[w[:i]] = append(m[w[:i]], w)
			}
		}
	}

	return PrefixMultiExpand{
		words: m,
		size:  n,
	}
}

// Expand uses the prefix mapping to return a list of words which can be expanded from s.
func (p PrefixMultiExpand) Expand(s string) ([]string, bool) {
	if len(s) < MinPrefix {
		return nil, false
	}
	if len(s) > p.size {
		// TODO: filter this with edit distance
		return p.words[s[:p.size]], true
	}
	return p.words[s], true
}

// expandSearcher is an implementation of Searcher which applies the Expander to search
// input and then performs a search on each of the expanded outputs, union the results
// and returns as the Search result.
type expandSearcher struct {
	Expander
	Searcher
}

// Search implements Searcher, and uses the internal Expander to expand all words
// in the search expression and unions the results (deduping). Returns nil when
// the search term isn't above the MinPrefix.
func (es *expandSearcher) Search(s string) []Path {
	e, ok := es.Expander.Expand(s)
	if !ok {
		return nil
	}
	ps := make([][]Path, len(e))
	for i, w := range e {
		ps[i] = es.Searcher.Search(w)
	}
	return Union(ps...)
}

// BuildPrefixExpandSearcher constructs a prefix expander which wraps the given Searcher
// by expanding each word in the search input using the WordIndex.
func BuildPrefixExpandSearcher(s Searcher, w WordIndex, n int) Searcher {
	return &expandSearcher{BuildPrefixMultiExpander(w.Words(), n), s}
}

// BuildWordIndex creates a *wordIndex ()
func BuildWordIndex(c Collection, fields []string) *wordIndex {
	wi := &wordIndex{
		fields: fields,
		words:  make(map[string][]Path),
	}
	wi.AddGroup(c, Path([]Key{"Root"}))
	return wi
}

type wordSearchIntersect struct {
	Searcher
	min int // the minimum input string length before search returns something non-trivial.
}

func (s *wordSearchIntersect) Search(x string) []Path {
	if len(x) < s.min {
		return make([]Path, 0)
	}
	words := strings.Fields(strings.ToLower(x))
	paths := make([][]Path, 0, len(words))
	for _, w := range words {
		if len(w) >= s.min {
			paths = append(paths, s.Searcher.Search(w))
		}
	}
	return OrderedIntersection(paths...)
}

// FlatSearcher is a Searcher wrapper which flattens input strings (replaces any accented
// characters with their un-accented equivalents).
type FlatSearcher struct {
	Searcher
}

func (f FlatSearcher) Search(s string) []Path {
	return f.Searcher.Search(removeNonAlphaNumeric(s))
}

// WordsSearchIntersect calls Search on the Searcher for each word in the input string
// and then returns the ordered intersection (Paths are ordered by the number of times
// they appear.
func WordsIntersectSearcher(s Searcher) Searcher {
	return &wordSearchIntersect{
		Searcher: s,
		min:      3,
	}
}

// stringFreq is a helper type to count the number of occurances of a string.
type stringFreq struct {
	n int
	k string
}

// stringFreqSlice is a convenience type for sorting a slice of stringFreqs by the frequency,
// and then the alphabetical order of the strings.
type stringFreqSlice []stringFreq

func (c stringFreqSlice) Len() int      { return len(c) }
func (c stringFreqSlice) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
func (c stringFreqSlice) Less(i, j int) bool {
	if c[i].n < c[j].n {
		return true
	}
	if c[i].n > c[j].n {
		return false
	}
	return c[i].k < c[j].k
}

// Compute the intersection of the given lists of paths.
func OrderedIntersection(paths ...[]Path) []Path {
	if len(paths) == 0 {
		return []Path{}
	}

	enc := make(map[string]Path) // Encoding -> Path
	set := make(map[string]bool) // Set of Encoding
	cnt := make(map[string]int)

	for _, v := range paths[0] {
		e := v.Encode()
		set[e] = true
		enc[e] = v
		cnt[e]++
	}

	if len(paths) > 1 {
		for _, list := range paths[1:] {
			miss := make(map[string]bool)
			for k := range set {
				miss[k] = true
			}

			for _, v := range list {
				e := v.Encode()
				if set[e] {
					delete(miss, e)
					cnt[e]++
				}
			}

			for k := range miss {
				delete(set, k)
				delete(cnt, k)
			}
		}
	}

	result := make([]Path, 0, len(cnt))
	count := make([]stringFreq, 0, len(cnt))
	for k, v := range cnt {
		result = append(result, enc[k])
		count = append(count, stringFreq{v, k})
	}

	sort.Sort(ParallelSort(sort.Reverse(stringFreqSlice(count)), PathSlice(result)))
	return result
}

// Union returns a []Path which is the union (deduped) of the given slices of []Path.
func Union(l ...[]Path) []Path {
	if len(l) == 0 {
		return []Path{}
	}

	done := make(map[string]bool)
	res := make([]Path, 0)
	for _, paths := range l {
		for _, path := range paths {
			e := path.Encode()
			if !done[e] {
				done[e] = true
				res = append(res, path)
			}
		}
	}
	return res
}
<|endoftext|>


<commit_before>package indexer

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strings"

	"github.com/cardigann/cardigann/config"
)

var (
	ErrUnknownIndexer       = errors.New("Unknown indexer")
	DefaultDefinitionLoader DefinitionLoader
)

type DefinitionLoader interface {
	List() ([]string, error)
	Load(key string) (*IndexerDefinition, error)
}

func init() {
	DefaultDefinitionLoader = &multiLoader{
		newFsLoader(),
		escLoader{Dir(false, "")},
	}
}

type fsLoader struct {
	dirs []string
}

func newFsLoader() DefinitionLoader {
	return &fsLoader{config.GetDefinitionDirs()}
}

func (fs *fsLoader) walkDirectories() (map[string]string, error) {
	defs := map[string]string{}

	for _, dirpath := range fs.dirs {
		dir, err := os.Open(dirpath)
		if os.IsNotExist(err) {
			continue
		}
		files, err := dir.Readdirnames(-1)
		if err != nil {
			continue
		}
		for _, basename := range files {
			if strings.HasSuffix(basename, ".yml") {
				defs[strings.TrimSuffix(basename, ".yml")] = path.Join(dir.Name(), basename)
			}
		}
	}

	return defs, nil
}

func (fs *fsLoader) List() ([]string, error) {
	defs, err := fs.walkDirectories()
	if err != nil {
		return nil, err
	}

	results := []string{}

	for k := range defs {
		results = append(results, k)
	}

	return results, nil
}

func (fs *fsLoader) Load(key string) (*IndexerDefinition, error) {
	defs, err := fs.walkDirectories()
	if err != nil {
		return nil, err
	}

	fileName, ok := defs[key]
	if !ok {
		return nil, ErrUnknownIndexer
	}

	data, err := ioutil.ReadFile(fileName)
	if err != nil {
		return nil, err
	}
	return ParseDefinition(data)
}

type multiLoader []DefinitionLoader

func (ml multiLoader) List() ([]string, error) {
	results := []string{}

	for _, loader := range ml {
		result, err := loader.List()
		if err != nil {
			return nil, err
		}
		results = append(results, result...)
	}

	return results, nil
}

func (ml multiLoader) Load(key string) (*IndexerDefinition, error) {
	for _, loader := range ml {
		def, err := loader.Load(key)
		if err == nil {
			return def, nil
		}
	}
	return nil, ErrUnknownIndexer
}

type escLoader struct {
	http.FileSystem
}

func (el escLoader) List() ([]string, error) {
	results := []string{}

	for key := range _escData {
		results = append(results, key)
	}

	return results, nil
}

func (el escLoader) Load(key string) (*IndexerDefinition, error) {
	f, err := el.Open(fmt.Sprintf("/definitions/%s.yml", key))
	if os.IsNotExist(err) {
		return nil, ErrUnknownIndexer
	} else if err != nil {
		return nil, err
	}

	defer f.Close()
	data, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	return ParseDefinition(data)
}
<commit_msg>escLoader correctly extracts keys from filenames<commit_after>package indexer

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"regexp"
	"strings"

	"github.com/cardigann/cardigann/config"
)

var (
	ErrUnknownIndexer       = errors.New("Unknown indexer")
	DefaultDefinitionLoader DefinitionLoader
)

type DefinitionLoader interface {
	List() ([]string, error)
	Load(key string) (*IndexerDefinition, error)
}

func init() {
	DefaultDefinitionLoader = &multiLoader{
		newFsLoader(),
		escLoader{Dir(false, "")},
	}
}

type fsLoader struct {
	dirs []string
}

func newFsLoader() DefinitionLoader {
	return &fsLoader{config.GetDefinitionDirs()}
}

func (fs *fsLoader) walkDirectories() (map[string]string, error) {
	defs := map[string]string{}

	for _, dirpath := range fs.dirs {
		dir, err := os.Open(dirpath)
		if os.IsNotExist(err) {
			continue
		}
		files, err := dir.Readdirnames(-1)
		if err != nil {
			continue
		}
		for _, basename := range files {
			if strings.HasSuffix(basename, ".yml") {
				defs[strings.TrimSuffix(basename, ".yml")] = path.Join(dir.Name(), basename)
			}
		}
	}

	return defs, nil
}

func (fs *fsLoader) List() ([]string, error) {
	defs, err := fs.walkDirectories()
	if err != nil {
		return nil, err
	}

	results := []string{}

	for k := range defs {
		results = append(results, k)
	}

	return results, nil
}

func (fs *fsLoader) Load(key string) (*IndexerDefinition, error) {
	defs, err := fs.walkDirectories()
	if err != nil {
		return nil, err
	}

	fileName, ok := defs[key]
	if !ok {
		return nil, ErrUnknownIndexer
	}

	data, err := ioutil.ReadFile(fileName)
	if err != nil {
		return nil, err
	}
	return ParseDefinition(data)
}

type multiLoader []DefinitionLoader

func (ml multiLoader) List() ([]string, error) {
	results := []string{}

	for _, loader := range ml {
		result, err := loader.List()
		if err != nil {
			return nil, err
		}
		results = append(results, result...)
	}

	return results, nil
}

func (ml multiLoader) Load(key string) (*IndexerDefinition, error) {
	for _, loader := range ml {
		def, err := loader.Load(key)
		if err == nil {
			return def, nil
		}
	}
	return nil, ErrUnknownIndexer
}

var escFilenameRegex = regexp.MustCompile(`^/definitions/(.+?)\.yml$`)

type escLoader struct {
	http.FileSystem
}

func (el escLoader) List() ([]string, error) {
	results := []string{}

	for filename := range _escData {
		if matches := escFilenameRegex.FindStringSubmatch(filename); matches != nil {
			results = append(results, matches[1])
		}
	}

	return results, nil
}

func (el escLoader) Load(key string) (*IndexerDefinition, error) {
	f, err := el.Open(fmt.Sprintf("/definitions/%s.yml", key))
	if os.IsNotExist(err) {
		return nil, ErrUnknownIndexer
	} else if err != nil {
		return nil, err
	}

	defer f.Close()
	data, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	return ParseDefinition(data)
}
<|endoftext|>


<commit_before>package inject

import (
	"fmt"
	"github.com/mono83/cfg"
	"reflect"
)

type graph struct {
	cfg.Configurer

	buildFunctions map[string]func(Container) (interface{}, error)
	services       map[string]interface{}

	onAfterInitFuncs []func(name string, target interface{}) error
}

// NewGraph builds new graph DI container
func NewGraph(c cfg.Configurer) Container {
	return &graph{
		Configurer:     c,
		services:       map[string]interface{}{},
		buildFunctions: map[string]func(Container) (interface{}, error){},
	}
}

func (g *graph) OnAfterInit(f func(name string, target interface{}) error) {
	if f != nil {
		g.onAfterInitFuncs = append(g.onAfterInitFuncs, f)
	}
}

func (g *graph) HasService(name string) bool {
	_, ok := g.buildFunctions[name]
	if !ok {
		_, ok = g.services[name]
	}

	return ok
}

func (g *graph) getOrBuild(name string) (interface{}, error) {
	if service, ok := g.services[name]; ok {
		return service, nil
	}

	bf, ok := g.buildFunctions[name]
	if ok {
		service, err := bf(g)
		if service == nil && err == nil {
			err = fmt.Errorf("Builder for %s returns nil without error", name)
		}
		if err == nil {
			if len(g.onAfterInitFuncs) > 0 {
				for _, afterInit := range g.onAfterInitFuncs {
					err = afterInit(name, service)
					if err != nil {
						break
					}
				}
			}
			if err == nil {
				g.services[name] = service
			}
		}

		return service, err
	}

	return nil, fmt.Errorf("Service %s not defined", name)
}

func (g *graph) GetService(name string, target interface{}) error {
	service, err := g.getOrBuild(name)
	if err != nil {
		return err
	}

	fg := &failsafeGetter{name: name, source: service, target: target}
	fg.cp()

	return fg.err
}

func (g *graph) MustGetService(name string, target interface{}) {
	err := g.GetService(name, target)
	if err != nil {
		panic(err)
	}
}

func (g *graph) MustGetServices(defs ...Definition) {
	err := g.GetServices(defs...)
	if err != nil {
		panic(err)
	}
}

func (g *graph) GetServices(defs ...Definition) error {
	for _, def := range defs {
		if err := g.GetService(def.Name(), def.Target()); err != nil {
			return err
		}
	}

	return nil
}

func (g *graph) Define(name string, build func(Container) (interface{}, error)) {
	g.buildFunctions[name] = build
}

// failsafeGetter is special component, used to handle panics
// on reflection
type failsafeGetter struct {
	err            error
	name           string
	source, target interface{}
}

func (f *failsafeGetter) cp() {
	defer func() {
		if r := recover(); r != nil {
			f.err = fmt.Errorf("Error copying service %s - %v", f.name, r)
		}
	}()

	s := reflect.TypeOf(f.source)
	t := reflect.TypeOf(f.target)
	if t.Kind() != reflect.Ptr {
		f.err = fmt.Errorf("Error copying service %s - target must be pointer, %T provided", f.name, f.target)
	} else if s.Kind() == reflect.Ptr && t.Elem().Kind() != reflect.Ptr && t.Elem().Kind() != reflect.Interface {
		f.err = fmt.Errorf("Error copying service %s - source is pointer, so target must be pointer to pointer, %T provided", f.name, f.target)
	} else {
		reflect.ValueOf(f.target).Elem().Set(reflect.ValueOf(f.source))
	}
}
<commit_msg>empty configurer on nil<commit_after>package inject

import (
	"fmt"
	"github.com/mono83/cfg"
	"reflect"
)

type graph struct {
	cfg.Configurer

	buildFunctions map[string]func(Container) (interface{}, error)
	services       map[string]interface{}

	onAfterInitFuncs []func(name string, target interface{}) error
}

// NewGraph builds new graph DI container
func NewGraph(c cfg.Configurer) Container {
	if c == nil {
		// Providing empty configurer on nil
		c = cfg.Map(map[string]interface{}{})
	}

	return &graph{
		Configurer:     c,
		services:       map[string]interface{}{},
		buildFunctions: map[string]func(Container) (interface{}, error){},
	}
}

func (g *graph) OnAfterInit(f func(name string, target interface{}) error) {
	if f != nil {
		g.onAfterInitFuncs = append(g.onAfterInitFuncs, f)
	}
}

func (g *graph) HasService(name string) bool {
	_, ok := g.buildFunctions[name]
	if !ok {
		_, ok = g.services[name]
	}

	return ok
}

func (g *graph) getOrBuild(name string) (interface{}, error) {
	if service, ok := g.services[name]; ok {
		return service, nil
	}

	bf, ok := g.buildFunctions[name]
	if ok {
		service, err := bf(g)
		if service == nil && err == nil {
			err = fmt.Errorf("Builder for %s returns nil without error", name)
		}
		if err == nil {
			if len(g.onAfterInitFuncs) > 0 {
				for _, afterInit := range g.onAfterInitFuncs {
					err = afterInit(name, service)
					if err != nil {
						break
					}
				}
			}
			if err == nil {
				g.services[name] = service
			}
		}

		return service, err
	}

	return nil, fmt.Errorf("Service %s not defined", name)
}

func (g *graph) GetService(name string, target interface{}) error {
	service, err := g.getOrBuild(name)
	if err != nil {
		return err
	}

	fg := &failsafeGetter{name: name, source: service, target: target}
	fg.cp()

	return fg.err
}

func (g *graph) MustGetService(name string, target interface{}) {
	err := g.GetService(name, target)
	if err != nil {
		panic(err)
	}
}

func (g *graph) MustGetServices(defs ...Definition) {
	err := g.GetServices(defs...)
	if err != nil {
		panic(err)
	}
}

func (g *graph) GetServices(defs ...Definition) error {
	for _, def := range defs {
		if err := g.GetService(def.Name(), def.Target()); err != nil {
			return err
		}
	}

	return nil
}

func (g *graph) Define(name string, build func(Container) (interface{}, error)) {
	g.buildFunctions[name] = build
}

// failsafeGetter is special component, used to handle panics
// on reflection
type failsafeGetter struct {
	err            error
	name           string
	source, target interface{}
}

func (f *failsafeGetter) cp() {
	defer func() {
		if r := recover(); r != nil {
			f.err = fmt.Errorf("Error copying service %s - %v", f.name, r)
		}
	}()

	s := reflect.TypeOf(f.source)
	t := reflect.TypeOf(f.target)
	if t.Kind() != reflect.Ptr {
		f.err = fmt.Errorf("Error copying service %s - target must be pointer, %T provided", f.name, f.target)
	} else if s.Kind() == reflect.Ptr && t.Elem().Kind() != reflect.Ptr && t.Elem().Kind() != reflect.Interface {
		f.err = fmt.Errorf("Error copying service %s - source is pointer, so target must be pointer to pointer, %T provided", f.name, f.target)
	} else {
		reflect.ValueOf(f.target).Elem().Set(reflect.ValueOf(f.source))
	}
}
<|endoftext|>


<commit_before>package redis

import (
	"errors"
	"strconv"
)

const (
	ErrorReply = 1 << iota
	StatusReply
	IntReply
	BulkReply
	MultiReply
)

type Reply struct {
	Type   int
	Error  error
	Status string
	Int    int64
	Bulk   []byte
	Multi  [][]byte
}

func (r *Redis) Recv() (*Reply, error) {
	data, err := r.Reader.ReadBytes(LF)
	if err != nil {
		return nil, err
	}
	reply := &Reply{}
	head := data[1 : len(data)-2]
	switch data[0] {
	case MINUS:
		reply.Type = ErrorReply
		reply.Error = errors.New(string(data[1 : len(data)-2]))
	case PLUS:
		reply.Type = StatusReply
		reply.Status = string(head)
	case COLON:
		reply.Type = IntReply
		if i, err := strconv.ParseInt(string(head), 10, 64); err != nil {
			reply.Error = err
		} else {
			reply.Int = i
		}
	case DOLLAR:
		reply.Type = BulkReply
		if bulk, err := r.bulk_reply(head); err != nil {
			reply.Error = err
		} else {
			reply.Bulk = bulk
		}
	case STAR:
		reply.Type = MultiReply
		i, err := strconv.Atoi(string(head))
		if err != nil {
			reply.Error = err
		}
		reply.Multi = make([][]byte, i)
		for j := 0; j < i; j++ {
			bulk, err := r.bulk_reply(nil)
			if err != nil {
				reply.Error = err
				break
			}
			reply.Multi[j] = bulk
		}
	}
	return reply, nil
}

func (r *Redis) bulk_reply(head []byte) ([]byte, error) {
	if head == nil {
		data, err := r.Reader.ReadBytes(LF)
		if err != nil {
			return nil, err
		}
		if data[0] != DOLLAR {
			return nil, errors.New("Not bulk head")
		}
		head = data[1 : len(data)-2]
	}
	size, err := strconv.Atoi(string(head))
	if err != nil {
		return nil, err
	}
	if size == -1 {
		return nil, nil
	}
	buf := make([]byte, size+2)
	if _, err := r.Reader.Read(buf); err != nil {
		return nil, err
	} else {
		return buf[:size], nil
	}
}
<commit_msg>fix a `can use head` error<commit_after>package redis

import (
	"errors"
	"strconv"
)

const (
	ErrorReply = 1 << iota
	StatusReply
	IntReply
	BulkReply
	MultiReply
)

type Reply struct {
	Type   int
	Error  error
	Status string
	Int    int64
	Bulk   []byte
	Multi  [][]byte
}

func (r *Redis) Recv() (*Reply, error) {
	data, err := r.Reader.ReadBytes(LF)
	if err != nil {
		return nil, err
	}
	reply := &Reply{}
	head := data[1 : len(data)-2]
	switch data[0] {
	case MINUS:
		reply.Type = ErrorReply
		reply.Error = errors.New(string(head))
	case PLUS:
		reply.Type = StatusReply
		reply.Status = string(head)
	case COLON:
		reply.Type = IntReply
		if i, err := strconv.ParseInt(string(head), 10, 64); err != nil {
			reply.Error = err
		} else {
			reply.Int = i
		}
	case DOLLAR:
		reply.Type = BulkReply
		if bulk, err := r.bulk_reply(head); err != nil {
			reply.Error = err
		} else {
			reply.Bulk = bulk
		}
	case STAR:
		reply.Type = MultiReply
		i, err := strconv.Atoi(string(head))
		if err != nil {
			reply.Error = err
		}
		reply.Multi = make([][]byte, i)
		for j := 0; j < i; j++ {
			bulk, err := r.bulk_reply(nil)
			if err != nil {
				reply.Error = err
				break
			}
			reply.Multi[j] = bulk
		}
	}
	return reply, nil
}

func (r *Redis) bulk_reply(head []byte) ([]byte, error) {
	if head == nil {
		data, err := r.Reader.ReadBytes(LF)
		if err != nil {
			return nil, err
		}
		if data[0] != DOLLAR {
			return nil, errors.New("Not bulk head")
		}
		head = data[1 : len(data)-2]
	}
	size, err := strconv.Atoi(string(head))
	if err != nil {
		return nil, err
	}
	if size == -1 {
		return nil, nil
	}
	buf := make([]byte, size+2)
	if _, err := r.Reader.Read(buf); err != nil {
		return nil, err
	} else {
		return buf[:size], nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/codegangsta/cli"
)

var (
	version   string
	hash      string
	builddate string
	goversion string
)

var CliFlags = []cli.Flag{
	cli.BoolFlag{
		Name:  OPT_SILENT,
		Usage: "if you want do not output messages.",
	},
	cli.BoolFlag{
		Name:  OPT_VERBOSE,
		Usage: "if you want show debug messages.",
	},
}

var Commands = []cli.Command{
	commandInit,
	commandEc2start,
	commandEc2stop,
	commandEc2list,
	commandEc2type,
	commandEc2run,
	commandAttachEIP,
	commandDetachEIP,
}

func main() {
	app := cli.NewApp()
	app.Name = "rnzoo"
	app.Version = version + " (" + hash + ") built with:" + goversion
	app.Usage = "useful commands for ec2."
	app.Author = "reiki4040"
	app.Email = ""
	app.Commands = Commands
	app.Flags = CliFlags
	app.Run(os.Args)
}
<commit_msg>modified version format.<commit_after>package main

import (
	"os"

	"github.com/codegangsta/cli"
)

var (
	version   string
	hash      string
	builddate string
	goversion string
)

var CliFlags = []cli.Flag{
	cli.BoolFlag{
		Name:  OPT_SILENT,
		Usage: "if you want do not output messages.",
	},
	cli.BoolFlag{
		Name:  OPT_VERBOSE,
		Usage: "if you want show debug messages.",
	},
}

var Commands = []cli.Command{
	commandInit,
	commandEc2start,
	commandEc2stop,
	commandEc2list,
	commandEc2type,
	commandEc2run,
	commandAttachEIP,
	commandDetachEIP,
}

func main() {
	app := cli.NewApp()
	app.Name = "rnzoo"
	app.Version = version + " (" + hash + ") " + goversion
	app.Usage = "useful commands for ec2."
	app.Author = "reiki4040"
	app.Email = ""
	app.Commands = Commands
	app.Flags = CliFlags
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>//+build linux

package main

import (
	"strings"
	"unsafe"

	"golang.org/x/sys/unix"
)

// canParseID indicates that drive information can be parsed on Linux.
const canParseID = true

// parseID parses an ID structure from raw drive identification bytes.
func parseID(b [512]byte) *ID {
	id := *(*unix.HDDriveID)(unsafe.Pointer(&b[0]))

	return &ID{
		Model:    byteStr(id.Model[:]),
		Serial:   byteStr(id.Serial_no[:]),
		Firmware: byteStr(id.Fw_rev[:]),
	}
}

// byteStr converts bytes to a string and trims any null characters.
func byteStr(b []byte) string {
	return strings.TrimSpace(string(b))
}
<commit_msg>cmd/blockctl: explicitly remove trailing NULL<commit_after>//+build linux

package main

import (
	"strings"
	"unsafe"

	"golang.org/x/sys/unix"
)

// canParseID indicates that drive information can be parsed on Linux.
const canParseID = true

// parseID parses an ID structure from raw drive identification bytes.
func parseID(b [512]byte) *ID {
	id := *(*unix.HDDriveID)(unsafe.Pointer(&b[0]))

	return &ID{
		Model:    byteStr(id.Model[:]),
		Serial:   byteStr(id.Serial_no[:]),
		Firmware: byteStr(id.Fw_rev[:]),
	}
}

// byteStr converts bytes to a string and trims any null characters.
func byteStr(b []byte) string {
	return strings.TrimSuffix(strings.TrimSpace(string(b)), "\x00")
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Frank Braun <frank@cryptogroup.net>
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

// ccprices prints current currency prices in ledger format.
package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"time"
)

const (
	euroAPI  = "http://api.fixer.io/latest"
	xauAPI   = "https://www.quandl.com/api/v3/datasets/LBMA/GOLD.json?limit=1"
	xagAPI   = "https://www.quandl.com/api/v3/datasets/LBMA/SILVER.json?limit=1"
	coinsAPI = "http://coinmarketcap.northpole.ro/api/v5/all.json"
)

var (
	// Quandl API key can be set via environment variable QUANDL_API_KEY
	quandl = os.Getenv("QUANDL_API_KEY")
	coins  = []string{"Bitcoin", "Decred", "Namecoin"}
)

type result struct {
	symbol string
	price  float64
}

func getEuroExchangeRates() (map[string]interface{}, error) {
	resp, err := http.Get(euroAPI)
	b, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn["rates"].(map[string]interface{}), nil
}

func getLBMAPrice(api string, dataIndex int) (float64, error) {
	if quandl != "" {
		api += "?api_key=" + quandl
	}
	resp, err := http.Get(api)
	b, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return 0, err
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return 0, err
	}
	data := jsn["dataset"].(map[string]interface{})["data"].([]interface{})
	var price float64
	if data[0].([]interface{})[dataIndex] != nil {
		// p.m. price is available
		price = data[0].([]interface{})[dataIndex].(float64)
	} else {
		// p.m. price is not available, use a.m. price instead
		price = data[0].([]interface{})[dataIndex-1].(float64)
	}
	return price, nil
}

func getCoinPrices() ([]interface{}, error) {
	resp, err := http.Get(coinsAPI)
	if err != nil {
		return nil, err
	}
	b, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn["markets"].([]interface{}), nil
}

func fatal(err error) {
	fmt.Fprintf(os.Stderr, "%s: error: %s\n", os.Args[0], err)
	os.Exit(1)
}

func main() {
	// get euro exchange rates
	rates, err := getEuroExchangeRates()
	if err != nil {
		fatal(err)
	}
	// get gold price
	xau, err := getLBMAPrice(xauAPI, 6)
	if err != nil {
		fatal(err)
	}
	// get silver price
	xag, err := getLBMAPrice(xagAPI, 3)
	if err != nil {
		fatal(err)
	}
	// get all coin prices
	all, err := getCoinPrices()
	if err != nil {
		fatal(err)
	}
	// construct map of coin names we are interested in
	names := make(map[string]struct{})
	for _, name := range coins {
		names[name] = struct{}{}
	}
	prices := make(map[string]*result)
	// iterate over all coin informations
	for _, info := range all {
		coin := info.(map[string]interface{})
		name := coin["name"].(string)
		_, ok := names[name]
		if ok {
			// we are interested in this coin -> store price and symbol
			f := coin["price"].(map[string]interface{})["eur"].(string)
			p, err := strconv.ParseFloat(f, 64)
			if err != nil {
				fatal(err)
			}
			prices[name] = &result{symbol: coin["symbol"].(string), price: p}
		}
	}
	// output all prices
	t := time.Now().Format("2006/01/02 15:04:05")
	fmt.Printf("P %s USD %11.6f EUR\n", t, 1/rates["USD"].(float64))
	fmt.Printf("P %s GBP %11.6f EUR\n", t, 1/rates["GBP"].(float64))
	fmt.Printf("P %s CHF %11.6f EUR\n", t, 1/rates["CHF"].(float64))
	fmt.Printf("P %s CZK %11.6f EUR\n", t, 1/rates["CZK"].(float64))
	fmt.Printf("P %s XAU %11.6f EUR\n", t, xau)
	fmt.Printf("P %s XAG %11.6f EUR\n", t, xag)
	for _, name := range coins {
		fmt.Printf("P %s %s %11.6f EUR\n", t, prices[name].symbol,
			prices[name].price)
	}
}
<commit_msg>ccprices: extend<commit_after>// Copyright (c) 2016 Frank Braun <frank@cryptogroup.net>
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

// ccprices prints current currency prices in ledger format.
package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"time"
)

const (
	euroAPI  = "http://api.fixer.io/latest"
	xauAPI   = "https://www.quandl.com/api/v3/datasets/LBMA/GOLD.json?limit=1"
	xagAPI   = "https://www.quandl.com/api/v3/datasets/LBMA/SILVER.json?limit=1"
	coinsAPI = "http://coinmarketcap.northpole.ro/api/v5/all.json"
)

var (
	// Quandl API key can be set via environment variable QUANDL_API_KEY
	quandl = os.Getenv("QUANDL_API_KEY")
	coins  = []string{"Bitcoin", "Decred", "Ethereum", "Ethereum Classic", "Monero", "Namecoin", "Zcash"}
)

type result struct {
	symbol string
	price  float64
}

func getEuroExchangeRates() (map[string]interface{}, error) {
	resp, err := http.Get(euroAPI)
	b, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn["rates"].(map[string]interface{}), nil
}

func getLBMAPrice(api string, dataIndex int) (float64, error) {
	if quandl != "" {
		api += "?api_key=" + quandl
	}
	resp, err := http.Get(api)
	b, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return 0, err
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return 0, err
	}
	data := jsn["dataset"].(map[string]interface{})["data"].([]interface{})
	var price float64
	if data[0].([]interface{})[dataIndex] != nil {
		// p.m. price is available
		price = data[0].([]interface{})[dataIndex].(float64)
	} else {
		// p.m. price is not available, use a.m. price instead
		price = data[0].([]interface{})[dataIndex-1].(float64)
	}
	return price, nil
}

func getCoinPrices() ([]interface{}, error) {
	resp, err := http.Get(coinsAPI)
	if err != nil {
		return nil, err
	}
	b, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn["markets"].([]interface{}), nil
}

func fatal(err error) {
	fmt.Fprintf(os.Stderr, "%s: error: %s\n", os.Args[0], err)
	os.Exit(1)
}

func main() {
	// get euro exchange rates
	rates, err := getEuroExchangeRates()
	if err != nil {
		fatal(err)
	}
	// get gold price
	xau, err := getLBMAPrice(xauAPI, 6)
	if err != nil {
		fatal(err)
	}
	// get silver price
	xag, err := getLBMAPrice(xagAPI, 3)
	if err != nil {
		fatal(err)
	}
	// get all coin prices
	all, err := getCoinPrices()
	if err != nil {
		fatal(err)
	}
	// construct map of coin names we are interested in
	names := make(map[string]struct{})
	for _, name := range coins {
		names[name] = struct{}{}
	}
	prices := make(map[string]*result)
	// iterate over all coin informations
	for _, info := range all {
		coin := info.(map[string]interface{})
		name := coin["name"].(string)
		_, ok := names[name]
		if ok {
			// we are interested in this coin -> store price and symbol
			f := coin["price"].(map[string]interface{})["eur"].(string)
			p, err := strconv.ParseFloat(f, 64)
			if err != nil {
				fatal(err)
			}
			prices[name] = &result{symbol: coin["symbol"].(string), price: p}
		}
	}
	// output all prices
	t := time.Now().Format("2006/01/02 15:04:05")
	fmt.Printf("P %s USD %11.6f EUR\n", t, 1/rates["USD"].(float64))
	fmt.Printf("P %s GBP %11.6f EUR\n", t, 1/rates["GBP"].(float64))
	fmt.Printf("P %s CHF %11.6f EUR\n", t, 1/rates["CHF"].(float64))
	fmt.Printf("P %s CZK %11.6f EUR\n", t, 1/rates["CZK"].(float64))
	fmt.Printf("P %s THB %11.6f EUR\n", t, 1/rates["THB"].(float64))
	fmt.Printf("P %s XAU %11.6f EUR\n", t, xau)
	fmt.Printf("P %s XAG %11.6f EUR\n", t, xag)
	for _, name := range coins {
		fmt.Printf("P %s %s %11.6f EUR\n", t, prices[name].symbol,
			prices[name].price)
	}
}
<|endoftext|>


<commit_before>// +build windows darwin

package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/cloudfoundry-incubator/garden/server"
	"github.com/concourse/atc"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/vito/houdini"
)

func (cmd *WorkerCommand) houdiniRunner(logger lager.Logger, platform string) (atc.Worker, ifrit.Runner, error) {
	depotDir := filepath.Join(cmd.WorkDir, "containers")

	err = os.MkdirAll(depotDir, 0755)
	if err != nil {
		return atc.Worker{}, nil, fmt.Errorf("failed to create depot dir: %s", err)
	}

	backend := houdini.NewBackend(depotDir)

	server := server.New(
		"tcp",
		cmd.bindAddr(),
		0,
		backend,
		logger,
	)

	worker := atc.Worker{
		Platform: "windows",
		Tags:     cmd.Tags,
	}

	worker.Name, err = cmd.workerName()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	return worker, gardenServerRunner{logger, server}, nil
}
<commit_msg>fix decl<commit_after>// +build windows darwin

package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/cloudfoundry-incubator/garden/server"
	"github.com/concourse/atc"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/vito/houdini"
)

func (cmd *WorkerCommand) houdiniRunner(logger lager.Logger, platform string) (atc.Worker, ifrit.Runner, error) {
	depotDir := filepath.Join(cmd.WorkDir, "containers")

	err := os.MkdirAll(depotDir, 0755)
	if err != nil {
		return atc.Worker{}, nil, fmt.Errorf("failed to create depot dir: %s", err)
	}

	backend := houdini.NewBackend(depotDir)

	server := server.New(
		"tcp",
		cmd.bindAddr(),
		0,
		backend,
		logger,
	)

	worker := atc.Worker{
		Platform: "windows",
		Tags:     cmd.Tags,
	}

	worker.Name, err = cmd.workerName()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	return worker, gardenServerRunner{logger, server}, nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).

package main

import (
	"bytes"
	"crypto/tls"
	"database/sql"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"log"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strconv"
	"sync"
	"time"

	"github.com/golang/groupcache/lru"
	"github.com/juju/ratelimit"
	"github.com/syncthing/syncthing/lib/protocol"
	"golang.org/x/net/context"
)

type querysrv struct {
	addr     string
	db       *sql.DB
	prep     map[string]*sql.Stmt
	limiter  *safeCache
	cert     tls.Certificate
	listener net.Listener
}

type announcement struct {
	Seen      time.Time `json:"seen"`
	Addresses []string  `json:"addresses"`
}

type safeCache struct {
	*lru.Cache
	mut sync.Mutex
}

func (s *safeCache) Get(key string) (val interface{}, ok bool) {
	s.mut.Lock()
	val, ok = s.Cache.Get(key)
	s.mut.Unlock()
	return
}

func (s *safeCache) Add(key string, val interface{}) {
	s.mut.Lock()
	s.Cache.Add(key, val)
	s.mut.Unlock()
}

type requestID int64

func (i requestID) String() string {
	return fmt.Sprintf("%016x", int64(i))
}

func negCacheFor(lastSeen time.Time) int {
	since := time.Since(lastSeen).Seconds()
	if since >= maxDeviceAge {
		return maxNegCache
	}
	if since < 0 {
		// That's weird
		return minNegCache
	}

	// Return a value linearly scaled from minNegCache (at zero seconds ago)
	// to maxNegCache (at maxDeviceAge seconds ago).
	r := since / maxDeviceAge
	return int(minNegCache + r*(maxNegCache-minNegCache))
}

func (s *querysrv) Serve() {
	s.limiter = &safeCache{
		Cache: lru.New(lruSize),
	}

	if useHTTP {
		listener, err := net.Listen("tcp", s.addr)
		if err != nil {
			log.Println("Listen:", err)
			return
		}
		s.listener = listener
	} else {
		tlsCfg := &tls.Config{
			Certificates:           []tls.Certificate{s.cert},
			ClientAuth:             tls.RequestClientCert,
			SessionTicketsDisabled: true,
			MinVersion:             tls.VersionTLS12,
			CipherSuites: []uint16{
				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
				tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
				tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
			},
		}

		tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
		if err != nil {
			log.Println("Listen:", err)
			return
		}
		s.listener = tlsListener
	}

	http.HandleFunc("/v2/", s.handler)
	http.HandleFunc("/ping", handlePing)

	srv := &http.Server{
		ReadTimeout:    5 * time.Second,
		WriteTimeout:   5 * time.Second,
		MaxHeaderBytes: 1 << 10,
	}

	if err := srv.Serve(s.listener); err != nil {
		log.Println("Serve:", err)
	}
}

var topCtx = context.Background()

func (s *querysrv) handler(w http.ResponseWriter, req *http.Request) {
	reqID := requestID(rand.Int63())
	ctx := context.WithValue(topCtx, "id", reqID)

	if debug {
		log.Println(reqID, req.Method, req.URL)
	}

	t0 := time.Now()
	defer func() {
		diff := time.Since(t0)
		var comment string
		if diff > time.Second {
			comment = "(very slow request)"
		} else if diff > 100*time.Millisecond {
			comment = "(slow request)"
		}
		if comment != "" || debug {
			log.Println(reqID, req.Method, req.URL, "completed in", diff, comment)
		}
	}()

	var remoteIP net.IP
	if useHTTP {
		remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
	} else {
		addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
		if err != nil {
			log.Println("remoteAddr:", err)
			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
			return
		}
		remoteIP = addr.IP
	}

	if s.limit(remoteIP) {
		if debug {
			log.Println(remoteIP, "is limited")
		}
		w.Header().Set("Retry-After", "60")
		http.Error(w, "Too Many Requests", 429)
		return
	}

	switch req.Method {
	case "GET":
		s.handleGET(ctx, w, req)
	case "POST":
		s.handlePOST(ctx, remoteIP, w, req)
	default:
		globalStats.Error()
		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
	}
}

func (s *querysrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
	reqID := ctx.Value("id").(requestID)

	deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
	if err != nil {
		if debug {
			log.Println(reqID, "bad device param")
		}
		globalStats.Error()
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}

	var ann announcement

	ann.Seen, err = s.getDeviceSeen(deviceID)
	negCache := strconv.Itoa(negCacheFor(ann.Seen))
	w.Header().Set("Retry-After", negCache)
	w.Header().Set("Cache-Control", "public, max-age="+negCache)

	if err != nil {
		// The device is not in the database.
		globalStats.Query()
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	t0 := time.Now()
	ann.Addresses, err = s.getAddresses(ctx, deviceID)
	if err != nil {
		log.Println(reqID, "getAddresses:", err)
		globalStats.Error()
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	if debug {
		log.Println(reqID, "getAddresses in", time.Since(t0))
	}

	globalStats.Query()

	if len(ann.Addresses) == 0 {
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	globalStats.Answer()

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(ann)
}

func (s *querysrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
	reqID := ctx.Value("id").(requestID)

	rawCert := certificateBytes(req)
	if rawCert == nil {
		if debug {
			log.Println(reqID, "no certificates")
		}
		globalStats.Error()
		http.Error(w, "Forbidden", http.StatusForbidden)
		return
	}

	var ann announcement
	if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
		if debug {
			log.Println(reqID, "decode:", err)
		}
		globalStats.Error()
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}

	deviceID := protocol.NewDeviceID(rawCert)

	// handleAnnounce returns *two* errors. The first indicates a problem with
	// something the client posted to us. We should return a 400 Bad Request
	// and not worry about it. The second indicates that the request was fine,
	// but something internal messed up. We should log it and respond with a
	// more apologetic 500 Internal Server Error.
	userErr, internalErr := s.handleAnnounce(ctx, remoteIP, deviceID, ann.Addresses)
	if userErr != nil {
		if debug {
			log.Println(reqID, "handleAnnounce:", userErr)
		}
		globalStats.Error()
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}
	if internalErr != nil {
		log.Println(reqID, "handleAnnounce:", internalErr)
		globalStats.Error()
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	globalStats.Announce()

	// TODO: Slowly increase this for stable clients
	w.Header().Set("Reannounce-After", "1800")

	// We could return the lookup result here, but it's kind of unnecessarily
	// expensive to go query the database again so we let the client decide to
	// do a lookup if they really care.
	w.WriteHeader(http.StatusNoContent)
}

func (s *querysrv) Stop() {
	s.listener.Close()
}

func (s *querysrv) handleAnnounce(ctx context.Context, remote net.IP, deviceID protocol.DeviceID, addresses []string) (userErr, internalErr error) {
	reqID := ctx.Value("id").(requestID)

	tx, err := s.db.Begin()
	if err != nil {
		internalErr = err
		return
	}

	defer func() {
		// Since we return from a bunch of different places, we handle
		// rollback in the defer.
		if internalErr != nil || userErr != nil {
			tx.Rollback()
		}
	}()

	for _, annAddr := range addresses {
		uri, err := url.Parse(annAddr)
		if err != nil {
			userErr = err
			return
		}

		host, port, err := net.SplitHostPort(uri.Host)
		if err != nil {
			userErr = err
			return
		}

		ip := net.ParseIP(host)
		if host == "" || ip.IsUnspecified() {
			host = remote.String()
		}

		uri.Host = net.JoinHostPort(host, port)

		if err := s.updateAddress(ctx, tx, deviceID, uri.String()); err != nil {
			internalErr = err
			return
		}
	}

	if err := s.updateDevice(ctx, tx, deviceID); err != nil {
		internalErr = err
		return
	}

	t0 := time.Now()
	internalErr = tx.Commit()
	if debug {
		log.Println(reqID, "commit in", time.Since(t0))
	}
	return
}

func (s *querysrv) limit(remote net.IP) bool {
	key := remote.String()

	bkt, ok := s.limiter.Get(key)
	if ok {
		bkt := bkt.(*ratelimit.Bucket)
		if bkt.TakeAvailable(1) != 1 {
			// Rate limit exceeded; ignore packet
			return true
		}
	} else {
		// One packet per ten seconds average rate, burst ten packets
		s.limiter.Add(key, ratelimit.NewBucket(10*time.Second/time.Duration(limitAvg), int64(limitBurst)))
	}

	return false
}

func (s *querysrv) updateDevice(ctx context.Context, tx *sql.Tx, device protocol.DeviceID) error {
	reqID := ctx.Value("id").(requestID)
	t0 := time.Now()
	res, err := tx.Stmt(s.prep["updateDevice"]).Exec(device.String())
	if err != nil {
		return err
	}
	if debug {
		log.Println(reqID, "updateDevice in", time.Since(t0))
	}

	if rows, _ := res.RowsAffected(); rows == 0 {
		t0 = time.Now()
		_, err := tx.Stmt(s.prep["insertDevice"]).Exec(device.String())
		if err != nil {
			return err
		}
		if debug {
			log.Println(reqID, "insertDevice in", time.Since(t0))
		}
	}

	return nil
}

func (s *querysrv) updateAddress(ctx context.Context, tx *sql.Tx, device protocol.DeviceID, uri string) error {
	res, err := tx.Stmt(s.prep["updateAddress"]).Exec(device.String(), uri)
	if err != nil {
		return err
	}

	if rows, _ := res.RowsAffected(); rows == 0 {
		_, err := tx.Stmt(s.prep["insertAddress"]).Exec(device.String(), uri)
		if err != nil {
			return err
		}
	}

	return nil
}

func (s *querysrv) getAddresses(ctx context.Context, device protocol.DeviceID) ([]string, error) {
	rows, err := s.prep["selectAddress"].Query(device.String())
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var res []string
	for rows.Next() {
		var addr string

		err := rows.Scan(&addr)
		if err != nil {
			log.Println("Scan:", err)
			continue
		}
		res = append(res, addr)
	}

	return res, nil
}

func (s *querysrv) getDeviceSeen(device protocol.DeviceID) (time.Time, error) {
	row := s.prep["selectDevice"].QueryRow(device.String())
	var seen time.Time
	if err := row.Scan(&seen); err != nil {
		return time.Time{}, err
	}
	return seen, nil
}

func handlePing(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(204)
}

func certificateBytes(req *http.Request) []byte {
	if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
		return req.TLS.PeerCertificates[0].Raw
	}

	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
		bs := []byte(hdr)
		// The certificate is in PEM format but with spaces for newlines. We
		// need to reinstate the newlines for the PEM decoder. But we need to
		// leave the spaces in the BEGIN and END lines - the first and last
		// space - alone.
		firstSpace := bytes.Index(bs, []byte(" "))
		lastSpace := bytes.LastIndex(bs, []byte(" "))
		for i := firstSpace + 1; i < lastSpace; i++ {
			if bs[i] == ' ' {
				bs[i] = '\n'
			}
		}
		block, _ := pem.Decode(bs)
		if block == nil {
			// Decoding failed
			return nil
		}
		return block.Bytes
	}

	return nil
}
<commit_msg>cmd/discosrv: Respect the listen address scheme (fixes #3346)<commit_after>// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).

package main

import (
	"bytes"
	"crypto/tls"
	"database/sql"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"log"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strconv"
	"sync"
	"time"

	"github.com/golang/groupcache/lru"
	"github.com/juju/ratelimit"
	"github.com/syncthing/syncthing/lib/protocol"
	"golang.org/x/net/context"
)

type querysrv struct {
	addr     string
	db       *sql.DB
	prep     map[string]*sql.Stmt
	limiter  *safeCache
	cert     tls.Certificate
	listener net.Listener
}

type announcement struct {
	Seen      time.Time `json:"seen"`
	Addresses []string  `json:"addresses"`
}

type safeCache struct {
	*lru.Cache
	mut sync.Mutex
}

func (s *safeCache) Get(key string) (val interface{}, ok bool) {
	s.mut.Lock()
	val, ok = s.Cache.Get(key)
	s.mut.Unlock()
	return
}

func (s *safeCache) Add(key string, val interface{}) {
	s.mut.Lock()
	s.Cache.Add(key, val)
	s.mut.Unlock()
}

type requestID int64

func (i requestID) String() string {
	return fmt.Sprintf("%016x", int64(i))
}

func negCacheFor(lastSeen time.Time) int {
	since := time.Since(lastSeen).Seconds()
	if since >= maxDeviceAge {
		return maxNegCache
	}
	if since < 0 {
		// That's weird
		return minNegCache
	}

	// Return a value linearly scaled from minNegCache (at zero seconds ago)
	// to maxNegCache (at maxDeviceAge seconds ago).
	r := since / maxDeviceAge
	return int(minNegCache + r*(maxNegCache-minNegCache))
}

func (s *querysrv) Serve() {
	s.limiter = &safeCache{
		Cache: lru.New(lruSize),
	}

	if useHTTP {
		listener, err := net.Listen("tcp", s.addr)
		if err != nil {
			log.Println("Listen:", err)
			return
		}
		s.listener = listener
	} else {
		tlsCfg := &tls.Config{
			Certificates:           []tls.Certificate{s.cert},
			ClientAuth:             tls.RequestClientCert,
			SessionTicketsDisabled: true,
			MinVersion:             tls.VersionTLS12,
			CipherSuites: []uint16{
				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
				tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
				tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
			},
		}

		tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
		if err != nil {
			log.Println("Listen:", err)
			return
		}
		s.listener = tlsListener
	}

	http.HandleFunc("/v2/", s.handler)
	http.HandleFunc("/ping", handlePing)

	srv := &http.Server{
		ReadTimeout:    5 * time.Second,
		WriteTimeout:   5 * time.Second,
		MaxHeaderBytes: 1 << 10,
	}

	if err := srv.Serve(s.listener); err != nil {
		log.Println("Serve:", err)
	}
}

var topCtx = context.Background()

func (s *querysrv) handler(w http.ResponseWriter, req *http.Request) {
	reqID := requestID(rand.Int63())
	ctx := context.WithValue(topCtx, "id", reqID)

	if debug {
		log.Println(reqID, req.Method, req.URL)
	}

	t0 := time.Now()
	defer func() {
		diff := time.Since(t0)
		var comment string
		if diff > time.Second {
			comment = "(very slow request)"
		} else if diff > 100*time.Millisecond {
			comment = "(slow request)"
		}
		if comment != "" || debug {
			log.Println(reqID, req.Method, req.URL, "completed in", diff, comment)
		}
	}()

	var remoteIP net.IP
	if useHTTP {
		remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
	} else {
		addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
		if err != nil {
			log.Println("remoteAddr:", err)
			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
			return
		}
		remoteIP = addr.IP
	}

	if s.limit(remoteIP) {
		if debug {
			log.Println(remoteIP, "is limited")
		}
		w.Header().Set("Retry-After", "60")
		http.Error(w, "Too Many Requests", 429)
		return
	}

	switch req.Method {
	case "GET":
		s.handleGET(ctx, w, req)
	case "POST":
		s.handlePOST(ctx, remoteIP, w, req)
	default:
		globalStats.Error()
		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
	}
}

func (s *querysrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
	reqID := ctx.Value("id").(requestID)

	deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
	if err != nil {
		if debug {
			log.Println(reqID, "bad device param")
		}
		globalStats.Error()
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}

	var ann announcement

	ann.Seen, err = s.getDeviceSeen(deviceID)
	negCache := strconv.Itoa(negCacheFor(ann.Seen))
	w.Header().Set("Retry-After", negCache)
	w.Header().Set("Cache-Control", "public, max-age="+negCache)

	if err != nil {
		// The device is not in the database.
		globalStats.Query()
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	t0 := time.Now()
	ann.Addresses, err = s.getAddresses(ctx, deviceID)
	if err != nil {
		log.Println(reqID, "getAddresses:", err)
		globalStats.Error()
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	if debug {
		log.Println(reqID, "getAddresses in", time.Since(t0))
	}

	globalStats.Query()

	if len(ann.Addresses) == 0 {
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	globalStats.Answer()

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(ann)
}

func (s *querysrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
	reqID := ctx.Value("id").(requestID)

	rawCert := certificateBytes(req)
	if rawCert == nil {
		if debug {
			log.Println(reqID, "no certificates")
		}
		globalStats.Error()
		http.Error(w, "Forbidden", http.StatusForbidden)
		return
	}

	var ann announcement
	if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
		if debug {
			log.Println(reqID, "decode:", err)
		}
		globalStats.Error()
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}

	deviceID := protocol.NewDeviceID(rawCert)

	// handleAnnounce returns *two* errors. The first indicates a problem with
	// something the client posted to us. We should return a 400 Bad Request
	// and not worry about it. The second indicates that the request was fine,
	// but something internal messed up. We should log it and respond with a
	// more apologetic 500 Internal Server Error.
	userErr, internalErr := s.handleAnnounce(ctx, remoteIP, deviceID, ann.Addresses)
	if userErr != nil {
		if debug {
			log.Println(reqID, "handleAnnounce:", userErr)
		}
		globalStats.Error()
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}
	if internalErr != nil {
		log.Println(reqID, "handleAnnounce:", internalErr)
		globalStats.Error()
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	globalStats.Announce()

	// TODO: Slowly increase this for stable clients
	w.Header().Set("Reannounce-After", "1800")

	// We could return the lookup result here, but it's kind of unnecessarily
	// expensive to go query the database again so we let the client decide to
	// do a lookup if they really care.
	w.WriteHeader(http.StatusNoContent)
}

func (s *querysrv) Stop() {
	s.listener.Close()
}

func (s *querysrv) handleAnnounce(ctx context.Context, remote net.IP, deviceID protocol.DeviceID, addresses []string) (userErr, internalErr error) {
	reqID := ctx.Value("id").(requestID)

	tx, err := s.db.Begin()
	if err != nil {
		internalErr = err
		return
	}

	defer func() {
		// Since we return from a bunch of different places, we handle
		// rollback in the defer.
		if internalErr != nil || userErr != nil {
			tx.Rollback()
		}
	}()

	for _, annAddr := range addresses {
		uri, err := url.Parse(annAddr)
		if err != nil {
			userErr = err
			return
		}

		host, port, err := net.SplitHostPort(uri.Host)
		if err != nil {
			userErr = err
			return
		}

		ip := net.ParseIP(host)
		if host == "" || ip.IsUnspecified() {
			// Do not use IPv6 remote address if requested scheme is tcp4
			if uri.Scheme == "tcp4" && remote.To4() == nil {
				continue
			}

			// Do not use IPv4 remote address if requested scheme is tcp6
			if uri.Scheme == "tcp6" && remote.To4() != nil {
				continue
			}

			host = remote.String()
		}

		uri.Host = net.JoinHostPort(host, port)

		if err := s.updateAddress(ctx, tx, deviceID, uri.String()); err != nil {
			internalErr = err
			return
		}
	}

	if err := s.updateDevice(ctx, tx, deviceID); err != nil {
		internalErr = err
		return
	}

	t0 := time.Now()
	internalErr = tx.Commit()
	if debug {
		log.Println(reqID, "commit in", time.Since(t0))
	}
	return
}

func (s *querysrv) limit(remote net.IP) bool {
	key := remote.String()

	bkt, ok := s.limiter.Get(key)
	if ok {
		bkt := bkt.(*ratelimit.Bucket)
		if bkt.TakeAvailable(1) != 1 {
			// Rate limit exceeded; ignore packet
			return true
		}
	} else {
		// One packet per ten seconds average rate, burst ten packets
		s.limiter.Add(key, ratelimit.NewBucket(10*time.Second/time.Duration(limitAvg), int64(limitBurst)))
	}

	return false
}

func (s *querysrv) updateDevice(ctx context.Context, tx *sql.Tx, device protocol.DeviceID) error {
	reqID := ctx.Value("id").(requestID)
	t0 := time.Now()
	res, err := tx.Stmt(s.prep["updateDevice"]).Exec(device.String())
	if err != nil {
		return err
	}
	if debug {
		log.Println(reqID, "updateDevice in", time.Since(t0))
	}

	if rows, _ := res.RowsAffected(); rows == 0 {
		t0 = time.Now()
		_, err := tx.Stmt(s.prep["insertDevice"]).Exec(device.String())
		if err != nil {
			return err
		}
		if debug {
			log.Println(reqID, "insertDevice in", time.Since(t0))
		}
	}

	return nil
}

func (s *querysrv) updateAddress(ctx context.Context, tx *sql.Tx, device protocol.DeviceID, uri string) error {
	res, err := tx.Stmt(s.prep["updateAddress"]).Exec(device.String(), uri)
	if err != nil {
		return err
	}

	if rows, _ := res.RowsAffected(); rows == 0 {
		_, err := tx.Stmt(s.prep["insertAddress"]).Exec(device.String(), uri)
		if err != nil {
			return err
		}
	}

	return nil
}

func (s *querysrv) getAddresses(ctx context.Context, device protocol.DeviceID) ([]string, error) {
	rows, err := s.prep["selectAddress"].Query(device.String())
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var res []string
	for rows.Next() {
		var addr string

		err := rows.Scan(&addr)
		if err != nil {
			log.Println("Scan:", err)
			continue
		}
		res = append(res, addr)
	}

	return res, nil
}

func (s *querysrv) getDeviceSeen(device protocol.DeviceID) (time.Time, error) {
	row := s.prep["selectDevice"].QueryRow(device.String())
	var seen time.Time
	if err := row.Scan(&seen); err != nil {
		return time.Time{}, err
	}
	return seen, nil
}

func handlePing(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(204)
}

func certificateBytes(req *http.Request) []byte {
	if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
		return req.TLS.PeerCertificates[0].Raw
	}

	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
		bs := []byte(hdr)
		// The certificate is in PEM format but with spaces for newlines. We
		// need to reinstate the newlines for the PEM decoder. But we need to
		// leave the spaces in the BEGIN and END lines - the first and last
		// space - alone.
		firstSpace := bytes.Index(bs, []byte(" "))
		lastSpace := bytes.LastIndex(bs, []byte(" "))
		for i := firstSpace + 1; i < lastSpace; i++ {
			if bs[i] == ' ' {
				bs[i] = '\n'
			}
		}
		block, _ := pem.Decode(bs)
		if block == nil {
			// Decoding failed
			return nil
		}
		return block.Bytes
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"k8s.io/api/networking/v1beta1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/ingress-gce/pkg/annotations"
	"k8s.io/ingress-gce/pkg/e2e"
	"k8s.io/ingress-gce/pkg/fuzz"
	"testing"
)

func TestILBBasic(t *testing.T) {
	t.Parallel()

	port80 := intstr.FromInt(80)

	for _, tc := range []struct {
		desc string
		ing  *v1beta1.Ingress

		numForwardingRules int
		numBackendServices int
	}{
		{
			desc: "http ILB default backend",
			ing: fuzz.NewIngressBuilder("", "ingress-1", "").
				DefaultBackend("service-1", port80).
				ConfigureForILB().
				Build(),
			numForwardingRules: 1,
			numBackendServices: 1,
		},
		{
			desc: "http ILB one path",
			ing: fuzz.NewIngressBuilder("", "ingress-1", "").
				AddPath("test.com", "/", "service-1", port80).
				ConfigureForILB().
				Build(),
			numForwardingRules: 1,
			numBackendServices: 2,
		},
		{
			desc: "http ILB multiple paths",
			ing: fuzz.NewIngressBuilder("", "ingress-1", "").
				AddPath("test.com", "/foo", "service-1", port80).
				AddPath("test.com", "/bar", "service-1", port80).
				ConfigureForILB().
				Build(),
			numForwardingRules: 1,
			numBackendServices: 2,
		},
	} {
		tc := tc // Capture tc as we are running this in parallel.
		Framework.RunWithSandbox(tc.desc, t, func(t *testing.T, s *e2e.Sandbox) {
			t.Parallel()

			t.Logf("Ingress = %s", tc.ing.String())

			negAnnotation := annotations.NegAnnotation{Ingress: true}
			annotation := map[string]string{annotations.NEGAnnotationKey: negAnnotation.String()}

			_, err := e2e.CreateEchoService(s, "service-1", annotation)
			if err != nil {
				t.Fatalf("error creating echo service: %v", err)
			}
			t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1")

			if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(tc.ing); err != nil {
				t.Fatalf("error creating Ingress spec: %v", err)
			}
			t.Logf("Ingress created (%s/%s)", s.Namespace, tc.ing.Name)

			ing, err := e2e.WaitForIngress(s, tc.ing, nil)
			if err != nil {
				t.Fatalf("error waiting for Ingress to stabilize: %v", err)
			}
			t.Logf("GCLB resources createdd (%s/%s)", s.Namespace, tc.ing.Name)

			// Perform whitebox testing.
			if len(ing.Status.LoadBalancer.Ingress) < 1 {
				t.Fatalf("Ingress does not have an IP: %+v", ing.Status)
			}

			vip := ing.Status.LoadBalancer.Ingress[0].IP
			t.Logf("Ingress %s/%s VIP = %s", s.Namespace, tc.ing.Name, vip)
			if !e2e.IsRfc1918Addr(vip) {
				t.Fatalf("got %v, want RFC1918 address, ing: %v", vip, ing)
			}

			// TODO(shance): update gcp.go for regional resources so that we can check GC here
		})
	}
}
<commit_msg>Remove word 'Basic' from ILB e2e test<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"k8s.io/api/networking/v1beta1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/ingress-gce/pkg/annotations"
	"k8s.io/ingress-gce/pkg/e2e"
	"k8s.io/ingress-gce/pkg/fuzz"
	"testing"
)

func TestILB(t *testing.T) {
	t.Parallel()

	port80 := intstr.FromInt(80)

	for _, tc := range []struct {
		desc string
		ing  *v1beta1.Ingress

		numForwardingRules int
		numBackendServices int
	}{
		{
			desc: "http ILB default backend",
			ing: fuzz.NewIngressBuilder("", "ingress-1", "").
				DefaultBackend("service-1", port80).
				ConfigureForILB().
				Build(),
			numForwardingRules: 1,
			numBackendServices: 1,
		},
		{
			desc: "http ILB one path",
			ing: fuzz.NewIngressBuilder("", "ingress-1", "").
				AddPath("test.com", "/", "service-1", port80).
				ConfigureForILB().
				Build(),
			numForwardingRules: 1,
			numBackendServices: 2,
		},
		{
			desc: "http ILB multiple paths",
			ing: fuzz.NewIngressBuilder("", "ingress-1", "").
				AddPath("test.com", "/foo", "service-1", port80).
				AddPath("test.com", "/bar", "service-1", port80).
				ConfigureForILB().
				Build(),
			numForwardingRules: 1,
			numBackendServices: 2,
		},
	} {
		tc := tc // Capture tc as we are running this in parallel.
		Framework.RunWithSandbox(tc.desc, t, func(t *testing.T, s *e2e.Sandbox) {
			t.Parallel()

			t.Logf("Ingress = %s", tc.ing.String())

			negAnnotation := annotations.NegAnnotation{Ingress: true}
			annotation := map[string]string{annotations.NEGAnnotationKey: negAnnotation.String()}

			_, err := e2e.CreateEchoService(s, "service-1", annotation)
			if err != nil {
				t.Fatalf("error creating echo service: %v", err)
			}
			t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1")

			if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(tc.ing); err != nil {
				t.Fatalf("error creating Ingress spec: %v", err)
			}
			t.Logf("Ingress created (%s/%s)", s.Namespace, tc.ing.Name)

			ing, err := e2e.WaitForIngress(s, tc.ing, nil)
			if err != nil {
				t.Fatalf("error waiting for Ingress to stabilize: %v", err)
			}
			t.Logf("GCLB resources createdd (%s/%s)", s.Namespace, tc.ing.Name)

			// Perform whitebox testing.
			if len(ing.Status.LoadBalancer.Ingress) < 1 {
				t.Fatalf("Ingress does not have an IP: %+v", ing.Status)
			}

			vip := ing.Status.LoadBalancer.Ingress[0].IP
			t.Logf("Ingress %s/%s VIP = %s", s.Namespace, tc.ing.Name, vip)
			if !e2e.IsRfc1918Addr(vip) {
				t.Fatalf("got %v, want RFC1918 address, ing: %v", vip, ing)
			}

			// TODO(shance): update gcp.go for regional resources so that we can check GC here
		})
	}
}
<|endoftext|>