<commit_before>package drax

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/hashicorp/raft"
)

func TestSetPeers(t *testing.T) {
	home, err := ioutil.TempDir("", "test-peers")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(home)
	_, trans := raft.NewInmemTransport()
	p := newPeerStore(home, trans)
	peers, err := p.Peers()
	if err != nil {
		t.Fatal(err)
	}
	if len(peers) != 0 {
		t.Fatalf("expected peers to be empty but got %v", peers)
	}

	peer := "1.2.3.4:1234"
	err = p.SetPeers([]string{peer})
	if err != nil {
		t.Fatal(err)
	}

	peers, err = p.Peers()
	if err != nil {
		t.Fatal(err)
	}
	if len(peers) != 1 {
		t.Fatalf("expected 1 peer, got %v", peers)
	}
	if peers[0] != peer {
		t.Fatalf("expected peer %s, got %s", peer, peer[0])
	}
}
<commit_msg>Fix go vet error<commit_after>package drax

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/hashicorp/raft"
)

func TestSetPeers(t *testing.T) {
	home, err := ioutil.TempDir("", "test-peers")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(home)
	_, trans := raft.NewInmemTransport()
	p := newPeerStore(home, trans)
	peers, err := p.Peers()
	if err != nil {
		t.Fatal(err)
	}
	if len(peers) != 0 {
		t.Fatalf("expected peers to be empty but got %v", peers)
	}

	peer := "1.2.3.4:1234"
	err = p.SetPeers([]string{peer})
	if err != nil {
		t.Fatal(err)
	}

	peers, err = p.Peers()
	if err != nil {
		t.Fatal(err)
	}
	if len(peers) != 1 {
		t.Fatalf("expected 1 peer, got %v", peers)
	}
	if peers[0] != peer {
		t.Fatalf("expected peer %s, got %s", peer, peers[0])
	}
}
<|endoftext|>


<commit_before>package connmgr

//
//	连接管理模型
//	-	每条连接起两个GO程，一个读，一个写
//	-	连接管理负责Event和Message的汇集和分发
//	-	会有一个或多个写分发GO程，将输出数据包分发到对应连接的写GO程
//
//	事件流程
//	-	连接的读循环将事件抛入连接管理
//	-	处理程序从连接管理读入事件，进行处理，然后再PushEvent或PushMessage
//	-	写分发GO程将需要分发的数据包分发到对应连接的写GO程
//	-	连接写GO程将数据包发送到客户端
//

// 读处理循环，可被调用于GO程
type Reader interface {
	ReadLoop(conn *Conn, stop <-chan bool) error // 处理连接读循环
}

// 写处理循环，可被调用于GO程
type Writer interface {
	WriteLoop(conn *Conn, stop <-chan bool) error // 处理连接写循环
}

// 写队列分发循环，可被调用于GO程
type Dispatcher interface {
	DispatchLoop(queue <-chan *Event) error // 分发输出数据包
}
<commit_msg>add connmgr.Acceptor and connmgr.Handler<commit_after>package connmgr

import (
	"net"
)

//
//	连接管理模型
//	-	每条连接起两个GO程，一个读，一个写
//	-	连接管理负责Event和Message的汇集和分发
//	-	会有一个或多个写分发GO程，将输出数据包分发到对应连接的写GO程
//
//	事件流程
//	-	连接的读循环将事件抛入连接管理
//	-	处理程序从连接管理读入事件，进行处理，然后再PushEvent或PushMessage
//	-	写分发GO程将需要分发的数据包分发到对应连接的写GO程
//	-	连接写GO程将数据包发送到客户端
//

// 读处理循环，可被调用于GO程
type Reader interface {
	ReadLoop(conn *Conn, stop <-chan bool) error // 处理连接读循环
}

// 写处理循环，可被调用于GO程
type Writer interface {
	WriteLoop(conn *Conn, stop <-chan bool) error // 处理连接写循环
}

// 连接处理入口
type Handler interface {
	HandleConn(conn *Conn) error // 处理新连接
}

// 写队列分发循环，可被调用于GO程
type Dispatcher interface {
	DispatchLoop(queue <-chan *Event) error // 分发输出数据包
}

// 接收循环
type Acceptor interface {
	AcceptLoop(listener *net.TCPListener) error // 处理Accept循环
}
<|endoftext|>


<commit_before>package gps

import (
	"fmt"
	"sort"

	"github.com/Masterminds/semver"
)

var (
	none = noneConstraint{}
	any  = anyConstraint{}
)

// A Constraint provides structured limitations on the versions that are
// admissible for a given project.
//
// As with Version, it has a private method because the gps's internal
// implementation of the problem is complete, and the system relies on type
// magic to operate.
type Constraint interface {
	fmt.Stringer
	// Matches indicates if the provided Version is allowed by the Constraint.
	Matches(Version) bool
	// MatchesAny indicates if the intersection of the Constraint with the
	// provided Constraint would yield a Constraint that could allow *any*
	// Version.
	MatchesAny(Constraint) bool
	// Intersect computes the intersection of the Constraint with the provided
	// Constraint.
	Intersect(Constraint) Constraint
	_private()
}

func (semverConstraint) _private() {}
func (anyConstraint) _private()    {}
func (noneConstraint) _private()   {}

// NewSemverConstraint attempts to construct a semver Constraint object from the
// input string.
//
// If the input string cannot be made into a valid semver Constraint, an error
// is returned.
func NewSemverConstraint(body string) (Constraint, error) {
	c, err := semver.NewConstraint(body)
	if err != nil {
		return nil, err
	}
	return semverConstraint{c: c}, nil
}

type semverConstraint struct {
	c semver.Constraint
}

func (c semverConstraint) String() string {
	return c.c.String()
}

func (c semverConstraint) Matches(v Version) bool {
	switch tv := v.(type) {
	case versionTypeUnion:
		for _, elem := range tv {
			if c.Matches(elem) {
				return true
			}
		}
	case semVersion:
		return c.c.Matches(tv.sv) == nil
	case versionPair:
		if tv2, ok := tv.v.(semVersion); ok {
			return c.c.Matches(tv2.sv) == nil
		}
	}

	return false
}

func (c semverConstraint) MatchesAny(c2 Constraint) bool {
	return c.Intersect(c2) != none
}

func (c semverConstraint) Intersect(c2 Constraint) Constraint {
	switch tc := c2.(type) {
	case anyConstraint:
		return c
	case versionTypeUnion:
		for _, elem := range tc {
			if rc := c.Intersect(elem); rc != none {
				return rc
			}
		}
	case semverConstraint:
		rc := c.c.Intersect(tc.c)
		if !semver.IsNone(rc) {
			return semverConstraint{c: rc}
		}
	case semVersion:
		rc := c.c.Intersect(tc.sv)
		if !semver.IsNone(rc) {
			// If single version intersected with constraint, we know the result
			// must be the single version, so just return it back out
			return c2
		}
	case versionPair:
		if tc2, ok := tc.v.(semVersion); ok {
			rc := c.c.Intersect(tc2.sv)
			if !semver.IsNone(rc) {
				// same reasoning as previous case
				return c2
			}
		}
	}

	return none
}

// IsAny indicates if the provided constraint is the wildcard "Any" constraint.
func IsAny(c Constraint) bool {
	_, ok := c.(anyConstraint)
	return ok
}

// Any returns a constraint that will match anything.
func Any() Constraint {
	return anyConstraint{}
}

// anyConstraint is an unbounded constraint - it matches all other types of
// constraints. It mirrors the behavior of the semver package's any type.
type anyConstraint struct{}

func (anyConstraint) String() string {
	return "*"
}

func (anyConstraint) Matches(Version) bool {
	return true
}

func (anyConstraint) MatchesAny(Constraint) bool {
	return true
}

func (anyConstraint) Intersect(c Constraint) Constraint {
	return c
}

// noneConstraint is the empty set - it matches no versions. It mirrors the
// behavior of the semver package's none type.
type noneConstraint struct{}

func (noneConstraint) String() string {
	return ""
}

func (noneConstraint) Matches(Version) bool {
	return false
}

func (noneConstraint) MatchesAny(Constraint) bool {
	return false
}

func (noneConstraint) Intersect(Constraint) Constraint {
	return none
}

// A ProjectConstraint combines a ProjectIdentifier with a Constraint. It
// indicates that, if packages contained in the ProjectIdentifier enter the
// depgraph, they must do so at a version that is allowed by the Constraint.
type ProjectConstraint struct {
	Ident      ProjectIdentifier
	Constraint Constraint
}

type workingConstraint struct {
	Ident                     ProjectIdentifier
	Constraint                Constraint
	overrNet, overrConstraint bool
}

type ProjectConstraints map[ProjectRoot]ProjectProperties

func pcSliceToMap(l []ProjectConstraint, r ...[]ProjectConstraint) ProjectConstraints {
	final := make(ProjectConstraints)

	for _, pc := range l {
		final[pc.Ident.ProjectRoot] = ProjectProperties{
			NetworkName: pc.Ident.netName(),
			Constraint:  pc.Constraint,
		}
	}

	for _, pcs := range r {
		for _, pc := range pcs {
			if pp, exists := final[pc.Ident.ProjectRoot]; exists {
				// Technically this should be done through a bridge for
				// cross-version-type matching...but this is a one off for root and
				// that's just ridiculous for this.
				pp.Constraint = pp.Constraint.Intersect(pc.Constraint)
				final[pc.Ident.ProjectRoot] = pp
			} else {
				final[pc.Ident.ProjectRoot] = ProjectProperties{
					NetworkName: pc.Ident.netName(),
					Constraint:  pc.Constraint,
				}
			}
		}
	}

	return final
}

func (m ProjectConstraints) asSortedSlice() []ProjectConstraint {
	pcs := make([]ProjectConstraint, len(m))

	k := 0
	for pr, pp := range m {
		pcs[k] = ProjectConstraint{
			Ident: ProjectIdentifier{
				ProjectRoot: pr,
				NetworkName: pp.NetworkName,
			},
			Constraint: pp.Constraint,
		}
		k++
	}

	sort.Stable(sortedConstraints(pcs))
	return pcs
}

// overrideAll treats the ProjectConstraints map as an override map, and applies
// overridden values to the input.
//
// A slice of workingConstraint is returned, allowing differentiation between
// values that were or were not overridden.
func (m ProjectConstraints) overrideAll(in []ProjectConstraint) (out []workingConstraint) {
	out = make([]workingConstraint, len(in))
	k := 0
	for _, pc := range in {
		out[k] = m.override(pc)
		k++
	}

	return
}

// override replaces a single ProjectConstraint with a workingConstraint,
// overriding its values if a corresponding entry exists in the
// ProjectConstraints map.
func (m ProjectConstraints) override(pc ProjectConstraint) workingConstraint {
	wc := workingConstraint{
		Ident:      pc.Ident.normalize(), // necessary to normalize?
		Constraint: pc.Constraint,
	}

	if pp, has := m[pc.Ident.ProjectRoot]; has {
		// The rule for overrides is that *any* non-zero value for the prop
		// should be considered an override, even if it's equal to what's
		// already there.
		if pp.Constraint != nil {
			wc.Constraint = pp.Constraint
			wc.overrConstraint = true
		}

		if pp.NetworkName != "" {
			wc.Ident.NetworkName = pp.NetworkName
			wc.overrNet = true
		}

	}

	return wc
}

type sortedConstraints []ProjectConstraint

func (s sortedConstraints) Len() int {
	return len(s)
}

func (s sortedConstraints) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s sortedConstraints) Less(i, j int) bool {
	return s[i].Ident.less(s[j].Ident)
}
<commit_msg>Simplify new semver constraints if possible<commit_after>package gps

import (
	"fmt"
	"sort"

	"github.com/Masterminds/semver"
)

var (
	none = noneConstraint{}
	any  = anyConstraint{}
)

// A Constraint provides structured limitations on the versions that are
// admissible for a given project.
//
// As with Version, it has a private method because the gps's internal
// implementation of the problem is complete, and the system relies on type
// magic to operate.
type Constraint interface {
	fmt.Stringer
	// Matches indicates if the provided Version is allowed by the Constraint.
	Matches(Version) bool
	// MatchesAny indicates if the intersection of the Constraint with the
	// provided Constraint would yield a Constraint that could allow *any*
	// Version.
	MatchesAny(Constraint) bool
	// Intersect computes the intersection of the Constraint with the provided
	// Constraint.
	Intersect(Constraint) Constraint
	_private()
}

func (semverConstraint) _private() {}
func (anyConstraint) _private()    {}
func (noneConstraint) _private()   {}

// NewSemverConstraint attempts to construct a semver Constraint object from the
// input string.
//
// If the input string cannot be made into a valid semver Constraint, an error
// is returned.
func NewSemverConstraint(body string) (Constraint, error) {
	c, err := semver.NewConstraint(body)
	if err != nil {
		return nil, err
	}
	// If we got a simple semver.Version, simplify by returning our
	// corresponding type
	if sv, ok := c.(*semver.Version); ok {
		return semVersion{sv: sv}, nil
	}
	return semverConstraint{c: c}, nil
}

type semverConstraint struct {
	c semver.Constraint
}

func (c semverConstraint) String() string {
	return c.c.String()
}

func (c semverConstraint) Matches(v Version) bool {
	switch tv := v.(type) {
	case versionTypeUnion:
		for _, elem := range tv {
			if c.Matches(elem) {
				return true
			}
		}
	case semVersion:
		return c.c.Matches(tv.sv) == nil
	case versionPair:
		if tv2, ok := tv.v.(semVersion); ok {
			return c.c.Matches(tv2.sv) == nil
		}
	}

	return false
}

func (c semverConstraint) MatchesAny(c2 Constraint) bool {
	return c.Intersect(c2) != none
}

func (c semverConstraint) Intersect(c2 Constraint) Constraint {
	switch tc := c2.(type) {
	case anyConstraint:
		return c
	case versionTypeUnion:
		for _, elem := range tc {
			if rc := c.Intersect(elem); rc != none {
				return rc
			}
		}
	case semverConstraint:
		rc := c.c.Intersect(tc.c)
		if !semver.IsNone(rc) {
			return semverConstraint{c: rc}
		}
	case semVersion:
		rc := c.c.Intersect(tc.sv)
		if !semver.IsNone(rc) {
			// If single version intersected with constraint, we know the result
			// must be the single version, so just return it back out
			return c2
		}
	case versionPair:
		if tc2, ok := tc.v.(semVersion); ok {
			rc := c.c.Intersect(tc2.sv)
			if !semver.IsNone(rc) {
				// same reasoning as previous case
				return c2
			}
		}
	}

	return none
}

// IsAny indicates if the provided constraint is the wildcard "Any" constraint.
func IsAny(c Constraint) bool {
	_, ok := c.(anyConstraint)
	return ok
}

// Any returns a constraint that will match anything.
func Any() Constraint {
	return anyConstraint{}
}

// anyConstraint is an unbounded constraint - it matches all other types of
// constraints. It mirrors the behavior of the semver package's any type.
type anyConstraint struct{}

func (anyConstraint) String() string {
	return "*"
}

func (anyConstraint) Matches(Version) bool {
	return true
}

func (anyConstraint) MatchesAny(Constraint) bool {
	return true
}

func (anyConstraint) Intersect(c Constraint) Constraint {
	return c
}

// noneConstraint is the empty set - it matches no versions. It mirrors the
// behavior of the semver package's none type.
type noneConstraint struct{}

func (noneConstraint) String() string {
	return ""
}

func (noneConstraint) Matches(Version) bool {
	return false
}

func (noneConstraint) MatchesAny(Constraint) bool {
	return false
}

func (noneConstraint) Intersect(Constraint) Constraint {
	return none
}

// A ProjectConstraint combines a ProjectIdentifier with a Constraint. It
// indicates that, if packages contained in the ProjectIdentifier enter the
// depgraph, they must do so at a version that is allowed by the Constraint.
type ProjectConstraint struct {
	Ident      ProjectIdentifier
	Constraint Constraint
}

type workingConstraint struct {
	Ident                     ProjectIdentifier
	Constraint                Constraint
	overrNet, overrConstraint bool
}

type ProjectConstraints map[ProjectRoot]ProjectProperties

func pcSliceToMap(l []ProjectConstraint, r ...[]ProjectConstraint) ProjectConstraints {
	final := make(ProjectConstraints)

	for _, pc := range l {
		final[pc.Ident.ProjectRoot] = ProjectProperties{
			NetworkName: pc.Ident.netName(),
			Constraint:  pc.Constraint,
		}
	}

	for _, pcs := range r {
		for _, pc := range pcs {
			if pp, exists := final[pc.Ident.ProjectRoot]; exists {
				// Technically this should be done through a bridge for
				// cross-version-type matching...but this is a one off for root and
				// that's just ridiculous for this.
				pp.Constraint = pp.Constraint.Intersect(pc.Constraint)
				final[pc.Ident.ProjectRoot] = pp
			} else {
				final[pc.Ident.ProjectRoot] = ProjectProperties{
					NetworkName: pc.Ident.netName(),
					Constraint:  pc.Constraint,
				}
			}
		}
	}

	return final
}

func (m ProjectConstraints) asSortedSlice() []ProjectConstraint {
	pcs := make([]ProjectConstraint, len(m))

	k := 0
	for pr, pp := range m {
		pcs[k] = ProjectConstraint{
			Ident: ProjectIdentifier{
				ProjectRoot: pr,
				NetworkName: pp.NetworkName,
			},
			Constraint: pp.Constraint,
		}
		k++
	}

	sort.Stable(sortedConstraints(pcs))
	return pcs
}

// overrideAll treats the ProjectConstraints map as an override map, and applies
// overridden values to the input.
//
// A slice of workingConstraint is returned, allowing differentiation between
// values that were or were not overridden.
func (m ProjectConstraints) overrideAll(in []ProjectConstraint) (out []workingConstraint) {
	out = make([]workingConstraint, len(in))
	k := 0
	for _, pc := range in {
		out[k] = m.override(pc)
		k++
	}

	return
}

// override replaces a single ProjectConstraint with a workingConstraint,
// overriding its values if a corresponding entry exists in the
// ProjectConstraints map.
func (m ProjectConstraints) override(pc ProjectConstraint) workingConstraint {
	wc := workingConstraint{
		Ident:      pc.Ident.normalize(), // necessary to normalize?
		Constraint: pc.Constraint,
	}

	if pp, has := m[pc.Ident.ProjectRoot]; has {
		// The rule for overrides is that *any* non-zero value for the prop
		// should be considered an override, even if it's equal to what's
		// already there.
		if pp.Constraint != nil {
			wc.Constraint = pp.Constraint
			wc.overrConstraint = true
		}

		if pp.NetworkName != "" {
			wc.Ident.NetworkName = pp.NetworkName
			wc.overrNet = true
		}

	}

	return wc
}

type sortedConstraints []ProjectConstraint

func (s sortedConstraints) Len() int {
	return len(s)
}

func (s sortedConstraints) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s sortedConstraints) Less(i, j int) bool {
	return s[i].Ident.less(s[j].Ident)
}
<|endoftext|>


<commit_before>/*
Ranges parses ranges of the form -n,n,n-n,n- (or -).  See the manpage for
cut(1) for more info.

Example:
 TODO: Finish this

Bug: There really should be some sort of optimization for the filter.
*/
package ranges

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
)

/* ranges.go
 * library to parse range strings of the form -n,n,n-n,n- or -
 * by J. Stuart McMurray
 * Created 20141119
 * Last modified 20141119
 *
 * Copyright (c) 2014 J. Stuart McMurray <kd5pbo@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

/* Irange represents a range of input */
type IRange struct {
	Start int
	End   int
}

/* Has returns whether or not n is in the range specified by i */
func (i IRange) Has(n int) bool {
	return (n <= i.End) && (n >= i.Start)
}

/* Stringify IRange */
func (i IRange) String() string {
	return fmt.Sprintf("%v-%v", i.Start, i.End)
}

/* Filter for ranges and indices */
type Filter struct {
	All              bool     /* - */
	Upto             int      /* -n */
	UptoSpec         bool     /* If Upto has been specified */
	Singles          []int    /* n,n,n,n */
	Ranges           []IRange /* n-n,n-n */
	Andfollowing     int      /* n- */
	AndfollowingSpec bool     /* If Andfollowing has been specified */

	/* Printf-like functions to print Information and Debugging messages */
	Debug   func(f string, a ...interface{})
	Verbose func(f string, a ...interface{})
}

/* New makes a new Filter, with the Debug and Verbose functions set to d and v,
which may be nil to indicate a no-op. */
func New(v, d func(f string, a ...interface{})) Filter {
	/* Make a filter */
	f := Filter{}
	/* Make noops if needed */
	if nil == d {
		d = func(f string, a ...interface{}) {}
	}
	if nil == v {
		v = func(f string, a ...interface{}) {}
	}
	/* Set output functions */
	f.Debug = d
	f.Verbose = v

	return f
}

/* Stringify the filter to a one-line string */
func (f Filter) String() string {
	/* Simpler true/false */
	a := "F" /* All */
	if f.All {
		a = "T"
	}
	u := "F" /* UptoSpec */
	if f.UptoSpec {
		u = "T"
	}
	n := "F" /* AndfollowingSpec */
	if f.AndfollowingSpec {
		n = "T"
	}
	/* TODO: Finish this */
	return fmt.Sprintf("[-: %v][-n: %v(%v)][n- %v(%v)][n-n %v]"+
		"[n %v]",
		a,
		f.Upto, u,
		f.Andfollowing, n,
		f.Ranges,
		f.Singles)
}

/* Pretty-print the filter */
func (f Filter) PPrint() {
	/* TODO: Finish this */
	fmt.Printf("%v\n", f)
}

/* Update the filter with s, which contains a comma-separated list of ranges,
or indices.  It is a union operation.  It is an error to pass a string to
Update which doesn't contain at least one range. */
func (f *Filter) Update(s string) error {
	one := false /* Have at least one range */
	/* Trim Whitespace */
	s = strings.TrimSpace(s)
	/* Split on , */
	ss := strings.Split(s, ",")
	if 0 == len(ss) {
		return errors.New("no ranges")
	}
	f.Verbose("Processing %v", s)
	/* Process each string in ss */
	for _, r := range ss {
		/* Skip whitespace-only strings */
		r = strings.TrimSpace(r)
		if len(s) > 0 {
			/* Die on errors */
			if err := f.UpdateOne(r); err != nil {
				return errors.New(fmt.Sprintf("Error "+
					"processing %v: %v", r, err))
			}
			/* We got one */
			one = true
		}
	}
	/* If we didn't even get one update, throw an error */
	if !one {
		return errors.New("no ranges or indices found")
	}
	return nil
}

/* UpdateOne adds to the filter the number or range in s.  It is a union. */
func (f *Filter) UpdateOne(s string) error {
	/* Trim whitespace */
	s = strings.TrimSpace(s)
	/* Give up if string was only whitespace */
	if "" == s {
		f.Debug("Got whitespace.")
		return nil
	}
	f.Debug("Processing %v", s)
	/* Work out what sort of specification: -, -n, n-, n-n, n */
	switch {
	case "-" == s: /* - */
		f.All = true
		f.Verbose("Entire input selected")

	case strings.HasPrefix(s, "-"): /* -n */
		/* Extract the number */
		n, err := strconv.Atoi(strings.TrimLeft(s, "-"))
		if err != nil {
			return err
		}
		/* Don't change if it's a subset */
		if f.All || (f.UptoSpec && f.Upto >= n) {
			return nil
		}
		/* Update */
		f.Upto = n
		f.UptoSpec = true
		f.Verbose("Final range now %v", s)

	case strings.HasSuffix(s, "-"): /* n- */
		/* Extract the number */
		n, err := strconv.Atoi(strings.TrimLeft(s, "-"))
		if err != nil {
			return err
		}
		/* Don't change if it's a subset */
		if f.All || (f.AndfollowingSpec && f.Andfollowing <= n) {
			return nil
		}
		/* Update */
		f.Andfollowing = n
		f.AndfollowingSpec = true
		f.Verbose("Initial range now %v", s)

	case strings.ContainsRune(s, '-'): /* n-n */
		/* Extract the strings */
		ns := strings.SplitN(s, "-", 2)
		/* Make sure the split worked */
		if 2 != len(ns) { /* Should never happen */
			return errors.New("not enough numbers in range")
		}
		/* Extract the numbers */
		start, err := strconv.Atoi(ns[0])
		if err != nil {
			return err
		}
		end, err := strconv.Atoi(ns[1])
		if err != nil {
			return err
		}
		/* Check the obvious fields, optimize will do the rest */
		if f.All {
			return nil
		}
		if f.UptoSpec && f.Upto >= end {
			return nil
		}
		if f.AndfollowingSpec && f.Andfollowing <= start {
			return nil
		}
		/* Add it to the list */
		f.Ranges = append(f.Ranges, IRange{Start: start, End: end})
		f.Verbose("Added range %v", s)

	default: /* n, hopefully */
		/* Extract the number */
		n, err := strconv.Atoi(s)
		if err != nil {
			return err
		}
		/* Check the obvious fields */
		if f.All || (f.UptoSpec && n <= f.Upto) ||
			(f.AndfollowingSpec && n >= f.Andfollowing) {
			return nil
		}
		/* Add if it not there */
		if !f.Allows(n) {
			f.Singles = append(f.Singles, n)
		}
		f.Verbose("Added index %v")

	}
	f.Debug("filter: %v", f)
	return nil
}

/* AllowsOut returns whether or not f allows in, and an additional int that
describes which part of the filter allowed n. */
const (
	/* Return values for filter.AllowsOut */
	InRange  = iota /* In a range */
	IsIndex         /* Matched a single index */
	Above           /* Above or equal to Andfollowing */
	Below           /* Below or equal to Upto */
	AllMatch        /* All is set */
)

func (f Filter) AllowsOut(n int) (bool, int) {
	/* Check the obvious fields */
	switch {
	case f.All:
		return true, AllMatch
	case f.UptoSpec && n <= f.Upto:
		return true, Below
	case f.AndfollowingSpec && n >= f.Andfollowing:
		return true, Above
	case f.InRanges(n):
		return true, InRange
	}
	/* Check the individual indices */
	for _, i := range f.Singles {
		if n == i {
			return true, IsIndex
		}
	}
	/* If we're here, it's not allowed */
	return false, 0
}

/* Allows returns whether or not f allows n */
func (f Filter) Allows(n int) bool {
	a, _ := f.AllowsOut(n)
	return a
}

/* InRanges tests if in an index is in one of f's ranges */
func (f Filter) InRanges(i int) bool {
	for _, r := range f.Ranges {
		if i >= r.Start && i <= r.End {
			return true
		}
	}
	return false
}
<commit_msg>Cleaned up output, n- and -n get parsed properly<commit_after>/*
Ranges parses ranges of the form -n,n,n-n,n- (or -).  See the manpage for
cut(1) for more info.

Example:
 TODO: Finish this

Bug: There really should be some sort of optimization for the filter.
*/
package ranges

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
)

/* ranges.go
 * library to parse range strings of the form -n,n,n-n,n- or -
 * by J. Stuart McMurray
 * Created 20141119
 * Last modified 20141119
 *
 * Copyright (c) 2014 J. Stuart McMurray <kd5pbo@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

/* Irange represents a range of input */
type IRange struct {
	Start int
	End   int
}

/* Has returns whether or not n is in the range specified by i */
func (i IRange) Has(n int) bool {
	return (n <= i.End) && (n >= i.Start)
}

/* Stringify IRange */
func (i IRange) String() string {
	return fmt.Sprintf("%v-%v", i.Start, i.End)
}

/* Filter for ranges and indices */
type Filter struct {
	All              bool     /* - */
	Upto             int      /* -n */
	UptoSpec         bool     /* If Upto has been specified */
	Singles          []int    /* n,n,n,n */
	Ranges           []IRange /* n-n,n-n */
	Andfollowing     int      /* n- */
	AndfollowingSpec bool     /* If Andfollowing has been specified */

	/* Printf-like functions to print Information and Debugging messages */
	Debug   func(f string, a ...interface{})
	Verbose func(f string, a ...interface{})
}

/* New makes a new Filter, with the Debug and Verbose functions set to d and v,
which may be nil to indicate a no-op. */
func New(v, d func(f string, a ...interface{})) Filter {
	/* Make a filter */
	f := Filter{}
	/* Make noops if needed */
	if nil == d {
		d = func(f string, a ...interface{}) {}
	}
	if nil == v {
		v = func(f string, a ...interface{}) {}
	}
	/* Set output functions */
	f.Debug = d
	f.Verbose = v

	return f
}

/* Stringify the filter to a one-line string */
func (f Filter) String() string {
	/* Simpler true/false */
	a := "F" /* All */
	if f.All {
		a = "T"
	}
	u := "F" /* UptoSpec */
	if f.UptoSpec {
		u = "T"
	}
	n := "F" /* AndfollowingSpec */
	if f.AndfollowingSpec {
		n = "T"
	}
	/* TODO: Finish this */
	return fmt.Sprintf("[-: %v][-n: %v(%v)][n- %v(%v)][n-n %v]"+
		"[n %v]",
		a,
		f.Upto, u,
		f.Andfollowing, n,
		f.Ranges,
		f.Singles)
}

/* Pretty-print the filter */
func (f Filter) PPrint() {
	/* TODO: Finish this */
	fmt.Printf("%v\n", f)
}

/* Update the filter with s, which contains a comma-separated list of ranges,
or indices.  It is a union operation.  It is an error to pass a string to
Update which doesn't contain at least one range. */
func (f *Filter) Update(s string) error {
	one := false /* Have at least one range */
	/* Trim Whitespace */
	s = strings.TrimSpace(s)
	/* Split on , */
	ss := strings.Split(s, ",")
	if 0 == len(ss) {
		return errors.New("no ranges")
	}
	f.Verbose("Processing range(s): %v", s)
	/* Process each string in ss */
	for _, r := range ss {
		/* Skip whitespace-only strings */
		r = strings.TrimSpace(r)
		if len(s) > 0 {
			/* Die on errors */
			if err := f.UpdateOne(r); err != nil {
				return errors.New(fmt.Sprintf("Error "+
					"processing %v: %v", r, err))
			}
			/* We got one */
			one = true
		}
	}
	/* If we didn't even get one update, throw an error */
	if !one {
		return errors.New("no ranges or indices found")
	}
	return nil
}

/* UpdateOne adds to the filter the number or range in s.  It is a union. */
func (f *Filter) UpdateOne(s string) error {
	/* Trim whitespace */
	s = strings.TrimSpace(s)
	/* Give up if string was only whitespace */
	if "" == s {
		f.Debug("Got whitespace.")
		return nil
	}
	f.Debug("Processing range: %v", s)
	/* Work out what sort of specification: -, -n, n-, n-n, n */
	switch {
	case "-" == s: /* - */
		f.All = true
		f.Verbose("Entire input selected")

	case strings.HasPrefix(s, "-"): /* -n */
		/* Extract the number */
		n, err := strconv.Atoi(strings.TrimLeft(s, "-"))
		if err != nil {
			return err
		}
		/* Don't change if it's a subset */
		if f.All || (f.UptoSpec && f.Upto >= n) {
			return nil
		}
		/* Update */
		f.Upto = n
		f.UptoSpec = true
		f.Verbose("Initial range now %v", s)

	case strings.HasSuffix(s, "-"): /* n- */
		/* Extract the number */
		n, err := strconv.Atoi(strings.TrimRight(s, "-"))
		if err != nil {
			return err
		}
		/* Don't change if it's a subset */
		if f.All || (f.AndfollowingSpec && f.Andfollowing <= n) {
			return nil
		}
		/* Update */
		f.Andfollowing = n
		f.AndfollowingSpec = true
		f.Verbose("Final range now %v", s)

	case strings.ContainsRune(s, '-'): /* n-n */
		/* Extract the strings */
		ns := strings.SplitN(s, "-", 2)
		/* Make sure the split worked */
		if 2 != len(ns) { /* Should never happen */
			return errors.New("not enough numbers in range")
		}
		/* Extract the numbers */
		start, err := strconv.Atoi(ns[0])
		if err != nil {
			return err
		}
		end, err := strconv.Atoi(ns[1])
		if err != nil {
			return err
		}
		/* Check the obvious fields, optimize will do the rest */
		if f.All {
			return nil
		}
		if f.UptoSpec && f.Upto >= end {
			return nil
		}
		if f.AndfollowingSpec && f.Andfollowing <= start {
			return nil
		}
		/* Add it to the list */
		f.Ranges = append(f.Ranges, IRange{Start: start, End: end})
		f.Verbose("Added range %v", s)

	default: /* n, hopefully */
		/* Extract the number */
		n, err := strconv.Atoi(s)
		if err != nil {
			return err
		}
		/* Check the obvious fields */
		if f.All || (f.UptoSpec && n <= f.Upto) ||
			(f.AndfollowingSpec && n >= f.Andfollowing) {
			return nil
		}
		/* Add if it not there */
		if !f.Allows(n) {
			f.Singles = append(f.Singles, n)
		}
		f.Verbose("Added index %v")

	}
	f.Debug("filter: %v", f)
	return nil
}

/* AllowsOut returns whether or not f allows in, and an additional int that
describes which part of the filter allowed n. */
const (
	/* Return values for filter.AllowsOut */
	InRange  = iota /* In a range */
	IsIndex         /* Matched a single index */
	Above           /* Above or equal to Andfollowing */
	Below           /* Below or equal to Upto */
	AllMatch        /* All is set */
)

func (f Filter) AllowsOut(n int) (bool, int) {
	/* Check the obvious fields */
	switch {
	case f.All:
		return true, AllMatch
	case f.UptoSpec && n <= f.Upto:
		return true, Below
	case f.AndfollowingSpec && n >= f.Andfollowing:
		return true, Above
	case f.InRanges(n):
		return true, InRange
	}
	/* Check the individual indices */
	for _, i := range f.Singles {
		if n == i {
			return true, IsIndex
		}
	}
	/* If we're here, it's not allowed */
	return false, 0
}

/* Allows returns whether or not f allows n */
func (f Filter) Allows(n int) bool {
	a, _ := f.AllowsOut(n)
	return a
}

/* InRanges tests if in an index is in one of f's ranges */
func (f Filter) InRanges(i int) bool {
	for _, r := range f.Ranges {
		if i >= r.Start && i <= r.End {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package lz4

import (
	"io"

	"github.com/pierrec/lz4/v4/internal/lz4errors"
	"github.com/pierrec/lz4/v4/internal/lz4stream"
)

var readerStates = []aState{
	noState:     newState,
	errorState:  newState,
	newState:    readState,
	readState:   closedState,
	closedState: newState,
}

// NewReader returns a new LZ4 frame decoder.
func NewReader(r io.Reader) *Reader {
	zr := &Reader{frame: lz4stream.NewFrame()}
	zr.state.init(readerStates)
	_ = zr.Apply(defaultOnBlockDone)
	zr.Reset(r)
	return zr
}

// Reader allows reading an LZ4 stream.
type Reader struct {
	state   _State
	src     io.Reader        // source reader
	frame   *lz4stream.Frame // frame being read
	data    []byte           // pending data
	idx     int              // size of pending data
	handler func(int)
}

func (*Reader) private() {}

func (r *Reader) Apply(options ...Option) (err error) {
	defer r.state.check(&err)
	switch r.state.state {
	case newState:
	case errorState:
		return r.state.err
	default:
		return lz4errors.ErrOptionClosedOrError
	}
	for _, o := range options {
		if err = o(r); err != nil {
			return
		}
	}
	return
}

// Size returns the size of the underlying uncompressed data, if set in the stream.
func (r *Reader) Size() int {
	switch r.state.state {
	case readState, closedState:
		if r.frame.Descriptor.Flags.Size() {
			return int(r.frame.Descriptor.ContentSize)
		}
	}
	return 0
}

func (r *Reader) init() error {
	return r.frame.InitR(r.src)
}

func (r *Reader) Read(buf []byte) (n int, err error) {
	defer r.state.check(&err)
	switch r.state.state {
	case readState:
	case closedState, errorState:
		return 0, r.state.err
	case newState:
		// First initialization.
		if err = r.init(); r.state.next(err) {
			return
		}
		size := r.frame.Descriptor.Flags.BlockSizeIndex()
		r.data = size.Get()
	default:
		return 0, r.state.fail()
	}
	if len(buf) == 0 {
		return
	}

	var bn int
	if r.idx > 0 {
		// Some left over data, use it.
		goto fillbuf
	}
	// No uncompressed data yet.
	r.data = r.data[:cap(r.data)]
	for len(buf) >= len(r.data) {
		// Input buffer large enough and no pending data: uncompress directly into it.
		switch bn, err = r.frame.Blocks.Block.Uncompress(r.frame, r.src, buf); err {
		case nil:
			r.handler(bn)
			n += bn
			buf = buf[bn:]
		case io.EOF:
			goto close
		default:
			return
		}
	}
	if n > 0 {
		// Some data was read, done for now.
		return
	}
	// Read the next block.
	switch bn, err = r.frame.Blocks.Block.Uncompress(r.frame, r.src, r.data); err {
	case nil:
		r.handler(bn)
		r.data = r.data[:bn]
		goto fillbuf
	case io.EOF:
	default:
		return
	}
close:
	if er := r.frame.CloseR(r.src); er != nil {
		err = er
	}
	r.frame.Descriptor.Flags.BlockSizeIndex().Put(r.data)
	r.Reset(nil)
	return
fillbuf:
	bn = copy(buf, r.data[r.idx:])
	n += bn
	r.idx += bn
	if r.idx == len(r.data) {
		// All data read, get ready for the next Read.
		r.idx = 0
	}
	return
}

// Reset clears the state of the Reader r such that it is equivalent to its
// initial state from NewReader, but instead writing to writer.
// No access to reader is performed.
//
// w.Close must be called before Reset.
func (r *Reader) Reset(reader io.Reader) {
	r.frame.Reset(1)
	r.src = reader
	r.data = nil
	r.idx = 0
	r.state.reset()
}

// WriteTo efficiently uncompresses the data from the Reader underlying source to w.
func (r *Reader) WriteTo(w io.Writer) (n int64, err error) {
	switch r.state.state {
	case closedState, errorState:
		return 0, r.state.err
	case newState:
		if err = r.init(); r.state.next(err) {
			return
		}
	default:
		return 0, r.state.fail()
	}
	defer r.state.nextd(&err)

	var bn int
	block := r.frame.Blocks.Block
	size := r.frame.Descriptor.Flags.BlockSizeIndex()
	data := size.Get()
	defer size.Put(data)
	for {
		switch bn, err = block.Uncompress(r.frame, r.src, data); err {
		case nil:
		case io.EOF:
			err = r.frame.CloseR(r.src)
			return
		default:
			return
		}
		r.handler(bn)
		n += int64(bn)
		_, err = w.Write(data[:bn])
		if err != nil {
			return
		}
	}
}
<commit_msg>Reader: pool the local cache upon Reset<commit_after>package lz4

import (
	"io"

	"github.com/pierrec/lz4/v4/internal/lz4errors"
	"github.com/pierrec/lz4/v4/internal/lz4stream"
)

var readerStates = []aState{
	noState:     newState,
	errorState:  newState,
	newState:    readState,
	readState:   closedState,
	closedState: newState,
}

// NewReader returns a new LZ4 frame decoder.
func NewReader(r io.Reader) *Reader {
	zr := &Reader{frame: lz4stream.NewFrame()}
	zr.state.init(readerStates)
	_ = zr.Apply(defaultOnBlockDone)
	zr.Reset(r)
	return zr
}

// Reader allows reading an LZ4 stream.
type Reader struct {
	state   _State
	src     io.Reader        // source reader
	frame   *lz4stream.Frame // frame being read
	data    []byte           // pending data
	idx     int              // size of pending data
	handler func(int)
}

func (*Reader) private() {}

func (r *Reader) Apply(options ...Option) (err error) {
	defer r.state.check(&err)
	switch r.state.state {
	case newState:
	case errorState:
		return r.state.err
	default:
		return lz4errors.ErrOptionClosedOrError
	}
	for _, o := range options {
		if err = o(r); err != nil {
			return
		}
	}
	return
}

// Size returns the size of the underlying uncompressed data, if set in the stream.
func (r *Reader) Size() int {
	switch r.state.state {
	case readState, closedState:
		if r.frame.Descriptor.Flags.Size() {
			return int(r.frame.Descriptor.ContentSize)
		}
	}
	return 0
}

func (r *Reader) init() error {
	return r.frame.InitR(r.src)
}

func (r *Reader) Read(buf []byte) (n int, err error) {
	defer r.state.check(&err)
	switch r.state.state {
	case readState:
	case closedState, errorState:
		return 0, r.state.err
	case newState:
		// First initialization.
		if err = r.init(); r.state.next(err) {
			return
		}
		size := r.frame.Descriptor.Flags.BlockSizeIndex()
		r.data = size.Get()
	default:
		return 0, r.state.fail()
	}
	if len(buf) == 0 {
		return
	}

	var bn int
	if r.idx > 0 {
		// Some left over data, use it.
		goto fillbuf
	}
	// No uncompressed data yet.
	r.data = r.data[:cap(r.data)]
	for len(buf) >= len(r.data) {
		// Input buffer large enough and no pending data: uncompress directly into it.
		switch bn, err = r.frame.Blocks.Block.Uncompress(r.frame, r.src, buf); err {
		case nil:
			r.handler(bn)
			n += bn
			buf = buf[bn:]
		case io.EOF:
			goto close
		default:
			return
		}
	}
	if n > 0 {
		// Some data was read, done for now.
		return
	}
	// Read the next block.
	switch bn, err = r.frame.Blocks.Block.Uncompress(r.frame, r.src, r.data); err {
	case nil:
		r.handler(bn)
		r.data = r.data[:bn]
		goto fillbuf
	case io.EOF:
	default:
		return
	}
close:
	if er := r.frame.CloseR(r.src); er != nil {
		err = er
	}
	r.Reset(nil)
	return
fillbuf:
	bn = copy(buf, r.data[r.idx:])
	n += bn
	r.idx += bn
	if r.idx == len(r.data) {
		// All data read, get ready for the next Read.
		r.idx = 0
	}
	return
}

// Reset clears the state of the Reader r such that it is equivalent to its
// initial state from NewReader, but instead writing to writer.
// No access to reader is performed.
//
// w.Close must be called before Reset.
func (r *Reader) Reset(reader io.Reader) {
	size := r.frame.Descriptor.Flags.BlockSizeIndex()
	size.Put(r.data)
	r.frame.Reset(1)
	r.src = reader
	r.data = nil
	r.idx = 0
	r.state.reset()
}

// WriteTo efficiently uncompresses the data from the Reader underlying source to w.
func (r *Reader) WriteTo(w io.Writer) (n int64, err error) {
	switch r.state.state {
	case closedState, errorState:
		return 0, r.state.err
	case newState:
		if err = r.init(); r.state.next(err) {
			return
		}
	default:
		return 0, r.state.fail()
	}
	defer r.state.nextd(&err)

	var bn int
	block := r.frame.Blocks.Block
	size := r.frame.Descriptor.Flags.BlockSizeIndex()
	data := size.Get()
	defer size.Put(data)
	for {
		switch bn, err = block.Uncompress(r.frame, r.src, data); err {
		case nil:
		case io.EOF:
			err = r.frame.CloseR(r.src)
			return
		default:
			return
		}
		r.handler(bn)
		n += int64(bn)
		_, err = w.Write(data[:bn])
		if err != nil {
			return
		}
	}
}
<|endoftext|>


<commit_before>package gedcom

import (
	"errors"
	"io"
	"strconv"
)

type line struct {
	level  uint64
	xrefID string
	tag    string
	value  string
}

type Reader struct {
	t       *tokeniser
	options options

	peeked bool
	line   line
	err    error

	hadHeader, hadRecord bool
}

func NewReader(r io.Reader, opts ...Option) *Reader {
	var o options
	for _, opt := range opts {
		opt(&o)
	}
	return &Reader{
		t:       newTokeniser(r),
		options: o,
	}
}

func (r *Reader) readLine() {
	if r.err != nil {
		return
	}
	var t token
	t, r.err = r.t.GetToken()
	if r.err != nil {
		return
	}
	if t.typ != tokenLevel {
		r.err = ErrNotLevel
		return
	}
	r.line.level, r.err = strconv.ParseUint(t.data, 10, 64)
	if r.err != nil {
		return
	}
	t, r.err = r.t.GetToken()
	if r.err != nil {
		return
	}
	if t.typ == tokenXref {
		r.line.xrefID = t.data
		t, r.err = r.t.GetToken()
		if r.err != nil {
			return
		}
	} else {
		r.line.xrefID = ""
	}
	if t.typ != tokenTag {
		r.err = ErrNotTag
		return
	}
	r.line.tag = t.data
	t, r.err = r.t.GetToken()
	if r.err != nil {
		return
	}
	if t.typ == tokenEndLine {
		r.line.value = ""
		return
	}
	if t.typ != tokenLine && t.typ != tokenPointer {
		r.err = ErrNotLine
		return
	}
	r.line.value = t.data
}

func (r *Reader) Record() (Record, error) {
	if !r.peeked {
		r.readLine()
		r.peeked = true
	}
	if r.err != nil {
		return nil, r.err
	}
	if !r.hadHeader {
		if r.line.tag != "HEAD" {
			r.peeked = false
			return nil, ErrNoHeader
		}
		r.hadHeader = true
	} else if !r.hadRecord {
		switch r.line.tag {
		case "FAM", "INDI", "OBJE", "NOTE", "REPO", "SOUR", "SUBN":
			r.hadRecord = true
		case "TRLR":
			r.peeked = false
			return nil, ErrNoRecords
		}
	} else if r.line.tag == "TRLR" {
		r.peeked = false
		return &Trailer{}, nil
	}

	lines := make([]line, 1, 32)
	lines[0] = r.line
	var lastlevel uint64
	for {
		if r.err != nil {
			return nil, r.err
		}
		if r.line.level > lastlevel+1 {
			return nil, ErrInvalidLevel
		}
		lastlevel = r.line.level
		lines = append(lines, r.line)
		r.readLine()
		if r.line.level == 0 {
			plines := parseLines(lines)
			var record Record
			switch lines[0].tag {
			case "HEAD":
				record = &Header{}
			case "SUBM":
				record = &SubmissionRecord{}
			case "FAM":
				record = &Family{}
			case "INDI":
				record = &Individual{}
			case "OBJE":
				record = &MultimediaRecord{}
			case "NOTE":
				record = &NoteRecord{}
			case "REPO":
				record = &RepositoryRecord{}
			case "SOUR":
				record = &SourceRecord{}
			case "SUBN":
				record = &SubmitterRecord{}
			default:
				if lines[0].tag[0] == '_' {
					return plines, nil
				}
				return plines, ErrContext{"root", lines[0].tag, ErrUnknownTag}
			}
			err := record.parse(&plines, r.options)
			if err != nil {
				return nil, ErrContext{"root", lines[0].tag, err}
			}
			return record, nil
		}
	}

}

// Errors
var (
	ErrNoHeader  = errors.New("no header")
	ErrNoRecords = errors.New("no records")
	ErrNotLevel  = errors.New("not level token")
	ErrNotTag    = errors.New("not tag token")
	ErrNotLine   = errors.New("not line_value token")
)
<commit_msg>Passed the options to the tokeniser<commit_after>package gedcom

import (
	"errors"
	"io"
	"strconv"
)

type line struct {
	level  uint64
	xrefID string
	tag    string
	value  string
}

type Reader struct {
	t       *tokeniser
	options options

	peeked bool
	line   line
	err    error

	hadHeader, hadRecord bool
}

func NewReader(r io.Reader, opts ...Option) *Reader {
	var o options
	for _, opt := range opts {
		opt(&o)
	}
	return &Reader{
		t:       newTokeniser(r, o),
		options: o,
	}
}

func (r *Reader) readLine() {
	if r.err != nil {
		return
	}
	var t token
	t, r.err = r.t.GetToken()
	if r.err != nil {
		return
	}
	if t.typ != tokenLevel {
		r.err = ErrNotLevel
		return
	}
	r.line.level, r.err = strconv.ParseUint(t.data, 10, 64)
	if r.err != nil {
		return
	}
	t, r.err = r.t.GetToken()
	if r.err != nil {
		return
	}
	if t.typ == tokenXref {
		r.line.xrefID = t.data
		t, r.err = r.t.GetToken()
		if r.err != nil {
			return
		}
	} else {
		r.line.xrefID = ""
	}
	if t.typ != tokenTag {
		r.err = ErrNotTag
		return
	}
	r.line.tag = t.data
	t, r.err = r.t.GetToken()
	if r.err != nil {
		return
	}
	if t.typ == tokenEndLine {
		r.line.value = ""
		return
	}
	if t.typ != tokenLine && t.typ != tokenPointer {
		r.err = ErrNotLine
		return
	}
	r.line.value = t.data
}

func (r *Reader) Record() (Record, error) {
	if !r.peeked {
		r.readLine()
		r.peeked = true
	}
	if r.err != nil {
		return nil, r.err
	}
	if !r.hadHeader {
		if r.line.tag != "HEAD" {
			r.peeked = false
			return nil, ErrNoHeader
		}
		r.hadHeader = true
	} else if !r.hadRecord {
		switch r.line.tag {
		case "FAM", "INDI", "OBJE", "NOTE", "REPO", "SOUR", "SUBN":
			r.hadRecord = true
		case "TRLR":
			r.peeked = false
			return nil, ErrNoRecords
		}
	} else if r.line.tag == "TRLR" {
		r.peeked = false
		return &Trailer{}, nil
	}

	lines := make([]line, 1, 32)
	lines[0] = r.line
	var lastlevel uint64
	for {
		if r.err != nil {
			return nil, r.err
		}
		if r.line.level > lastlevel+1 {
			return nil, ErrInvalidLevel
		}
		lastlevel = r.line.level
		lines = append(lines, r.line)
		r.readLine()
		if r.line.level == 0 {
			plines := parseLines(lines)
			var record Record
			switch lines[0].tag {
			case "HEAD":
				record = &Header{}
			case "SUBM":
				record = &SubmissionRecord{}
			case "FAM":
				record = &Family{}
			case "INDI":
				record = &Individual{}
			case "OBJE":
				record = &MultimediaRecord{}
			case "NOTE":
				record = &NoteRecord{}
			case "REPO":
				record = &RepositoryRecord{}
			case "SOUR":
				record = &SourceRecord{}
			case "SUBN":
				record = &SubmitterRecord{}
			default:
				if lines[0].tag[0] == '_' {
					return plines, nil
				}
				return plines, ErrContext{"root", lines[0].tag, ErrUnknownTag}
			}
			err := record.parse(&plines, r.options)
			if err != nil {
				return nil, ErrContext{"root", lines[0].tag, err}
			}
			return record, nil
		}
	}

}

// Errors
var (
	ErrNoHeader  = errors.New("no header")
	ErrNoRecords = errors.New("no records")
	ErrNotLevel  = errors.New("not level token")
	ErrNotTag    = errors.New("not tag token")
	ErrNotLine   = errors.New("not line_value token")
)
<|endoftext|>


<commit_before>package disruptor

import (
	"runtime"
	"time"
)

type Reader struct {
	read     *Cursor
	written  *Cursor
	upstream Barrier
	consumer Consumer
	ready    bool
}

func NewReader(read, written *Cursor, upstream Barrier, consumer Consumer) *Reader {
	return &Reader{
		read:     read,
		written:  written,
		upstream: upstream,
		consumer: consumer,
		ready:    false,
	}
}

func (this *Reader) Start() {
	this.ready = true
	go this.receive()
}
func (this *Reader) Stop() {
	this.ready = false
}

func (this *Reader) receive() {
	previous := this.read.Load()
	idling, gating := 0, 0

	for {
		lower := previous + 1
		upper := this.upstream.Read(lower)

		if lower <= upper {
			this.consumer.Consume(lower, upper)
			this.read.Store(upper)
			previous = upper
		} else if upper = this.written.Load(); lower <= upper {
			time.Sleep(time.Microsecond)
			// Gating--TODO: wait strategy (provide gating count to wait strategy for phased backoff)
			gating++
			idling = 0
		} else if this.ready {
			time.Sleep(time.Millisecond)
			// Idling--TODO: wait strategy (provide idling count to wait strategy for phased backoff)
			idling++
			gating = 0
		} else {
			break
		}

		// sleeping increases the batch size which reduces number of writes required to store the sequence
		// reducing the number of writes allows the CPU to optimize the pipeline without prediction failures
		runtime.Gosched() // LockSupport.parkNanos(1L); http://bit.ly/1xiDINZ
	}
}
<commit_msg>Atomic operations.<commit_after>package disruptor

import (
	"runtime"
	"sync/atomic"
	"time"
)

type Reader struct {
	read     *Cursor
	written  *Cursor
	upstream Barrier
	consumer Consumer
	ready    int32
}

func NewReader(read, written *Cursor, upstream Barrier, consumer Consumer) *Reader {
	return &Reader{
		read:     read,
		written:  written,
		upstream: upstream,
		consumer: consumer,
	}
}

func (this *Reader) Start() {
	atomic.StoreInt32(&this.ready, 0)
	go this.receive()
}
func (this *Reader) Stop() {
	atomic.StoreInt32(&this.ready, 1)
}
func (this *Reader) isReady() bool {
	return atomic.LoadInt32(&this.ready) == 0
}

func (this *Reader) receive() {
	previous := this.read.Load()
	idling, gating := 0, 0

	for {
		lower := previous + 1
		upper := this.upstream.Read(lower)

		if lower <= upper {
			this.consumer.Consume(lower, upper)
			this.read.Store(upper)
			previous = upper
		} else if upper = this.written.Load(); lower <= upper {
			time.Sleep(time.Microsecond)
			// Gating--TODO: wait strategy (provide gating count to wait strategy for phased backoff)
			gating++
			idling = 0
		} else if this.isReady() {
			time.Sleep(time.Millisecond)
			// Idling--TODO: wait strategy (provide idling count to wait strategy for phased backoff)
			idling++
			gating = 0
		} else {
			break
		}

		// sleeping increases the batch size which reduces number of writes required to store the sequence
		// reducing the number of writes allows the CPU to optimize the pipeline without prediction failures
		runtime.Gosched() // LockSupport.parkNanos(1L); http://bit.ly/1xiDINZ
	}
}
<|endoftext|>


<commit_before>package gedis

import (
	"fmt"
	"io"
)

// Reads an int64 from the Reader
func ReadNumber(r Reader) (n int64, err error) {
	b := make([]byte, 1)

	var sign int64 = 1

	_, err = r.Read(b)
	if err != nil {
		return
	}
	if b[0] == '-' {
		sign = -1
		b[0] = '0'
	}

	for {
		if b[0] >= '0' && b[0] <= '9' {
			n = n*10 + int64(b[0]-'0')
		} else if b[0] == '\r' {
			_, err = r.Read(b)
			if b[0] == '\n' {
				break
			} else {
				return 0, NewParseError("Invalid EOF")
			}
		} else {
			return 0, NewParseError("Invalid character")
		}

		_, err = r.Read(b)
		if err == io.EOF {
			break
		} else if err != nil {
			return
		}
	}

	return sign * n, nil
}

func readLine(r Reader) (line string, err error) {
	bs := make([]byte, 1024)
	l := 0

	b := make([]byte, 1)

	for {
		_, err = r.Read(b)

		if err == io.EOF {
			err = nil
			break
		} else if err != nil {
			return "", err
		} else if b[0] == '\r' {
			_, err = r.Read(b)
			if err != nil {
				return "", err
			}
			if b[0] == '\n' {
				break
			} else {
				bs[l] = '\r'
				l++
			}
		}

		bs[l] = b[0]
		l++
	}

	line = string(bs[:l])

	return line, err
}

func readBulk(r Reader) (interface{}, error) {
	var bs []byte

	numBytes, err := ReadNumber(r)
	if err != nil {
		return nil, err
	}

	if numBytes == -1 {
		return nil, nil
	}

	bs = make([]byte, numBytes)

	bytesRead, err := r.Read(bs)

	if err != nil {
		return nil, err
	} else if int64(bytesRead) != numBytes {
		return nil, NewParseError("Invalid byte count read")
	}

	// Must read following two bytes for \r\n
	crlf := make([]byte, 2)
	r.Read(crlf)

	if crlf[0] != '\r' || crlf[1] != '\n' {
		return nil, NewParseError("Invalid EOF")
	}

	return string(bs), nil
}

func Read(r Reader) (ret interface{}, err error) {
	kind := make([]byte, 1)

	_, err = r.Read(kind)
	if err != nil {
		return
	}

	switch kind[0] {
	case '+':
		ret, err = readLine(r)
		if err == nil {
			if bs, ok := ret.(string); ok {
				return Status(bs), nil
			} else {
				return nil, fmt.Errorf("Cannot convert to string: %#v", ret)
			}
		} else {
			return nil, err
		}
	case '-':
		ret, err = readLine(r)

		if err == nil {
			if bs, ok := ret.(string); ok {
				err = fmt.Errorf(string(bs))
			} else {
				err = fmt.Errorf("Cannot convert to []byte: %#v", ret)
			}
			ret = nil
		}
	case ':':
		ret, err = ReadNumber(r)
	case '$':
		ret, err = readBulk(r)
		if err == nil {
			if ret == nil {
				return nil, nil
			}
			if bs, ok := ret.(string); ok {
				return bs, nil
			} else {
				return nil, fmt.Errorf("Cannot convert to string: %#v", ret)
			}
		} else {
			return nil, err
		}
	case '*':
		n, err := ReadNumber(r)
		if err != nil {
			return nil, err
		}

		if n == -1 {
			return nil, nil
		}

		res := make([]interface{}, n)

		for i := int64(0); i < n; i++ {
			ret, err := Read(r)
			if err == nil {
				res[i] = ret
			} else {
				res[i] = err
			}
		}

		ret = res
	default:
		return nil, fmt.Errorf("Unexpected character %#v", kind[0])
	}

	return
}
<commit_msg>Extend the buffer if the received message is greater than the buffer's cap (hardcoded to 1024)<commit_after>package gedis

import (
	"fmt"
	"io"
)

// Reads an int64 from the Reader
func ReadNumber(r Reader) (n int64, err error) {
	b := make([]byte, 1)

	var sign int64 = 1

	_, err = r.Read(b)
	if err != nil {
		return
	}
	if b[0] == '-' {
		sign = -1
		b[0] = '0'
	}

	for {
		if b[0] >= '0' && b[0] <= '9' {
			n = n*10 + int64(b[0]-'0')
		} else if b[0] == '\r' {
			_, err = r.Read(b)
			if b[0] == '\n' {
				break
			} else {
				return 0, NewParseError("Invalid EOF")
			}
		} else {
			return 0, NewParseError("Invalid character")
		}

		_, err = r.Read(b)
		if err == io.EOF {
			break
		} else if err != nil {
			return
		}
	}

	return sign * n, nil
}

func readLine(r Reader) (line string, err error) {
	bs := make([]byte, 1024)
	l := 0

	b := make([]byte, 1)

	for {
		_, err = r.Read(b)

		if err == io.EOF {
			err = nil
			break
		} else if err != nil {
			return "", err
		} else if b[0] == '\r' {
			_, err = r.Read(b)
			if err != nil {
				return "", err
			}
			if b[0] == '\n' {
				break
			} else {
				bs[l] = '\r'
				l++
			}
		}

		if l == cap(bs) {
			// Increase the buffer by 1.5
			total := len(bs)
			newSize := total*3/2 + 1
			newBs := make([]byte, newSize)
			copy(newBs, bs)
			bs = newBs
		}

		bs[l] = b[0]
		l++
	}

	line = string(bs[:l])

	return line, err
}

func readBulk(r Reader) (interface{}, error) {
	var bs []byte

	numBytes, err := ReadNumber(r)
	if err != nil {
		return nil, err
	}

	if numBytes == -1 {
		return nil, nil
	}

	bs = make([]byte, numBytes)

	bytesRead, err := r.Read(bs)

	if err != nil {
		return nil, err
	} else if int64(bytesRead) != numBytes {
		return nil, NewParseError("Invalid byte count read")
	}

	// Must read following two bytes for \r\n
	crlf := make([]byte, 2)
	r.Read(crlf)

	if crlf[0] != '\r' || crlf[1] != '\n' {
		return nil, NewParseError("Invalid EOF")
	}

	return string(bs), nil
}

func Read(r Reader) (ret interface{}, err error) {
	kind := make([]byte, 1)

	_, err = r.Read(kind)
	if err != nil {
		return
	}

	switch kind[0] {
	case '+':
		ret, err = readLine(r)
		if err == nil {
			if bs, ok := ret.(string); ok {
				return Status(bs), nil
			} else {
				return nil, fmt.Errorf("Cannot convert to string: %#v", ret)
			}
		} else {
			return nil, err
		}
	case '-':
		ret, err = readLine(r)

		if err == nil {
			if bs, ok := ret.(string); ok {
				err = fmt.Errorf(string(bs))
			} else {
				err = fmt.Errorf("Cannot convert to []byte: %#v", ret)
			}
			ret = nil
		}
	case ':':
		ret, err = ReadNumber(r)
	case '$':
		ret, err = readBulk(r)
		if err == nil {
			if ret == nil {
				return nil, nil
			}
			if bs, ok := ret.(string); ok {
				return bs, nil
			} else {
				return nil, fmt.Errorf("Cannot convert to string: %#v", ret)
			}
		} else {
			return nil, err
		}
	case '*':
		n, err := ReadNumber(r)
		if err != nil {
			return nil, err
		}

		if n == -1 {
			return nil, nil
		}

		res := make([]interface{}, n)

		for i := int64(0); i < n; i++ {
			ret, err := Read(r)
			if err == nil {
				res[i] = ret
			} else {
				res[i] = err
			}
		}

		ret = res
	default:
		return nil, fmt.Errorf("Unexpected character %#v", kind[0])
	}

	return
}
<|endoftext|>


<commit_before>// Package geoip2 provides a wrapper around the maxminddb package for
// easy use with the MaxMind GeoIP2 and GeoLite2 databases. The records for
// the IP address is returned from this package as well-formed structures
// that match the internal layout of data from MaxMind.
package geoip2

import (
	"github.com/oxtoacart/maxminddb-golang"
	"net"
)

// The City structure corresponds to the data in the GeoIP2/GeoLite2 City
// databases.
type City struct {
	City struct {
		GeoNameID int `maxminddb:"geoname_id"`
		Names     map[string]string
	}
	Continent struct {
		Code      string
		GeoNameID int `maxminddb:"geoname_id"`
		Names     map[string]string
	}
	Country struct {
		GeoNameID int    `maxminddb:"geoname_id"`
		IsoCode   string `maxminddb:"iso_code"`
		Names     map[string]string
	}
	Location struct {
		Latitude  float64
		Longitude float64
		MetroCode int    `maxminddb:"metro_code"`
		TimeZone  string `maxminddb:"time_zone"`
	}
	Postal struct {
		Code string
	}
	RegisteredCountry struct {
		GeoNameID int    `maxminddb:"geoname_id"`
		IsoCode   string `maxminddb:"iso_code"`
		Names     map[string]string
	} `maxminddb:"registered_country"`
	RepresentedCountry struct {
		GeoNameID int    `maxminddb:"geoname_id"`
		IsoCode   string `maxminddb:"iso_code"`
		Names     map[string]string
		Type      string
	} `maxminddb:"represented_country"`
	Subdivisions []struct {
		GeoNameID int    `maxminddb:"geoname_id"`
		IsoCode   string `maxminddb:"iso_code"`
		Names     map[string]string
	}
	Traits struct {
		IsAnonymousProxy    bool `maxminddb:"is_anonymous_proxy"`
		IsSatelliteProvider bool `maxminddb:"is_satellite_provider"`
	}
}

// The Country structure corresponds to the data in the GeoIP2/GeoLite2
// Country databases.
type Country struct {
	Continent struct {
		Code      string
		GeoNameID int `maxminddb:"geoname_id"`
		Names     map[string]string
	}
	Country struct {
		GeoNameID int    `maxminddb:"geoname_id"`
		IsoCode   string `maxminddb:"iso_code"`
		Names     map[string]string
	}
	RegisteredCountry struct {
		GeoNameID int    `maxminddb:"geoname_id"`
		IsoCode   string `maxminddb:"iso_code"`
		Names     map[string]string
	} `maxminddb:"registered_country"`
	RepresentedCountry struct {
		GeoNameID int    `maxminddb:"geoname_id"`
		IsoCode   string `maxminddb:"iso_code"`
		Names     map[string]string
		Type      string
	} `maxminddb:"represented_country"`
	Traits struct {
		IsAnonymousProxy    bool `maxminddb:"is_anonymous_proxy"`
		IsSatelliteProvider bool `maxminddb:"is_satellite_provider"`
	}
}

// Reader holds the maxminddb.Reader structure. It should be created
// using the Open function.
type Reader struct {
	mmdbReader *maxminddb.Reader
}

// Open takes a string path to a file and returns a Reader structure or an
// error. The database file opened using a memory map. Use the Close method
// on the Reader object to return the resources to the system.
func Open(file string) (*Reader, error) {
	reader, err := maxminddb.Open(file)
	return &Reader{mmdbReader: reader}, err
}

// OpenBytes takes a string path to a file and returns a Reader structure or an
// error. The database file opened using a memory map. Use the Close method
// on the Reader object to return the resources to the system.
func OpenBytes(bytes []byte) (*Reader, error) {
	reader, err := maxminddb.OpenBytes(bytes)
	return &Reader{mmdbReader: reader}, err
}

// City takes an IP address as a net.IP struct and returns a City struct
// and/or an error. Although this can be used with other databases, this
// method generally should be used with the GeoIP2 or GeoLite2 City databases.
func (r *Reader) City(ipAddress net.IP) (*City, error) {
	var city City
	err := r.mmdbReader.Unmarshal(ipAddress, &city)
	return &city, err
}

// Country takes an IP address as a net.IP struct and returns a Country struct
// and/or an error. Although this can be used with other databases, this
// method generally should be used with the GeoIP2 or GeoLite2 Country
// databases.
func (r *Reader) Country(ipAddress net.IP) (*Country, error) {
	var country Country
	err := r.mmdbReader.Unmarshal(ipAddress, &country)
	return &country, err
}

// Close unmaps the database file from virtual memory and returns the
// resources to the system.
func (r *Reader) Close() {
	r.mmdbReader.Close()
}
<commit_msg>Renamed constructor<commit_after>// Package geoip2 provides a wrapper around the maxminddb package for
// easy use with the MaxMind GeoIP2 and GeoLite2 databases. The records for
// the IP address is returned from this package as well-formed structures
// that match the internal layout of data from MaxMind.
package geoip2

import (
	"github.com/oschwald/maxminddb-golang"
	"net"
)

// The City structure corresponds to the data in the GeoIP2/GeoLite2 City
// databases.
type City struct {
	City struct {
		GeoNameID int `maxminddb:"geoname_id"`
		Names     map[string]string
	}
	Continent struct {
		Code      string
		GeoNameID int `maxminddb:"geoname_id"`
		Names     map[string]string
	}
	Country struct {
		GeoNameID int    `maxminddb:"geoname_id"`
		IsoCode   string `maxminddb:"iso_code"`
		Names     map[string]string
	}
	Location struct {
		Latitude  float64
		Longitude float64
		MetroCode int    `maxminddb:"metro_code"`
		TimeZone  string `maxminddb:"time_zone"`
	}
	Postal struct {
		Code string
	}
	RegisteredCountry struct {
		GeoNameID int    `maxminddb:"geoname_id"`
		IsoCode   string `maxminddb:"iso_code"`
		Names     map[string]string
	} `maxminddb:"registered_country"`
	RepresentedCountry struct {
		GeoNameID int    `maxminddb:"geoname_id"`
		IsoCode   string `maxminddb:"iso_code"`
		Names     map[string]string
		Type      string
	} `maxminddb:"represented_country"`
	Subdivisions []struct {
		GeoNameID int    `maxminddb:"geoname_id"`
		IsoCode   string `maxminddb:"iso_code"`
		Names     map[string]string
	}
	Traits struct {
		IsAnonymousProxy    bool `maxminddb:"is_anonymous_proxy"`
		IsSatelliteProvider bool `maxminddb:"is_satellite_provider"`
	}
}

// The Country structure corresponds to the data in the GeoIP2/GeoLite2
// Country databases.
type Country struct {
	Continent struct {
		Code      string
		GeoNameID int `maxminddb:"geoname_id"`
		Names     map[string]string
	}
	Country struct {
		GeoNameID int    `maxminddb:"geoname_id"`
		IsoCode   string `maxminddb:"iso_code"`
		Names     map[string]string
	}
	RegisteredCountry struct {
		GeoNameID int    `maxminddb:"geoname_id"`
		IsoCode   string `maxminddb:"iso_code"`
		Names     map[string]string
	} `maxminddb:"registered_country"`
	RepresentedCountry struct {
		GeoNameID int    `maxminddb:"geoname_id"`
		IsoCode   string `maxminddb:"iso_code"`
		Names     map[string]string
		Type      string
	} `maxminddb:"represented_country"`
	Traits struct {
		IsAnonymousProxy    bool `maxminddb:"is_anonymous_proxy"`
		IsSatelliteProvider bool `maxminddb:"is_satellite_provider"`
	}
}

// Reader holds the maxminddb.Reader structure. It should be created
// using the Open function.
type Reader struct {
	mmdbReader *maxminddb.Reader
}

// Open takes a string path to a file and returns a Reader structure or an
// error. The database file opened using a memory map. Use the Close method
// on the Reader object to return the resources to the system.
func Open(file string) (*Reader, error) {
	reader, err := maxminddb.Open(file)
	return &Reader{mmdbReader: reader}, err
}

// FromBytes takes a string path to a file and returns a Reader structure or an
// error. The database file opened using a memory map. Use the Close method
// on the Reader object to return the resources to the system.
func FromBytes(bytes []byte) (*Reader, error) {
	reader, err := maxminddb.FromBytes(bytes)
	return &Reader{mmdbReader: reader}, err
}

// City takes an IP address as a net.IP struct and returns a City struct
// and/or an error. Although this can be used with other databases, this
// method generally should be used with the GeoIP2 or GeoLite2 City databases.
func (r *Reader) City(ipAddress net.IP) (*City, error) {
	var city City
	err := r.mmdbReader.Unmarshal(ipAddress, &city)
	return &city, err
}

// Country takes an IP address as a net.IP struct and returns a Country struct
// and/or an error. Although this can be used with other databases, this
// method generally should be used with the GeoIP2 or GeoLite2 Country
// databases.
func (r *Reader) Country(ipAddress net.IP) (*Country, error) {
	var country Country
	err := r.mmdbReader.Unmarshal(ipAddress, &country)
	return &country, err
}

// Close unmaps the database file from virtual memory and returns the
// resources to the system.
func (r *Reader) Close() {
	r.mmdbReader.Close()
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"golang.org/x/net/context"

	"encoding/json"

	"github.com/Sirupsen/logrus"
	"github.com/gin-gonic/gin"
	"github.com/iron-io/functions/api/models"
	"github.com/iron-io/functions/api/runner"
	titancommon "github.com/iron-io/titan/common"
	"github.com/satori/go.uuid"
)

func handleSpecial(c *gin.Context) {
	ctx := c.MustGet("ctx").(context.Context)
	log := titancommon.Logger(ctx)

	err := Api.UseSpecialHandlers(c)
	if err != nil {
		log.WithError(err).Errorln("Error using special handler!")
		// todo: what do we do here? Should probably return a 500 or something
	}
}

func handleRunner(c *gin.Context) {
	if strings.HasPrefix(c.Request.URL.Path, "/v1") {
		c.Status(http.StatusNotFound)
		return
	}

	ctx := c.MustGet("ctx").(context.Context)
	log := titancommon.Logger(ctx)

	reqID := uuid.NewV5(uuid.Nil, fmt.Sprintf("%s%s%d", c.Request.RemoteAddr, c.Request.URL.Path, time.Now().Unix())).String()
	c.Set("reqID", reqID) // todo: put this in the ctx instead of gin's

	log = log.WithFields(logrus.Fields{"request_id": reqID})

	var err error

	var payload []byte
	if c.Request.Method == "POST" || c.Request.Method == "PUT" {
		payload, err = ioutil.ReadAll(c.Request.Body)
	} else if c.Request.Method == "GET" {
		qPL := c.Request.URL.Query()["payload"]
		if len(qPL) > 0 {
			payload = []byte(qPL[0])
		}
	}

	if len(payload) > 0 {
		var emptyJSON map[string]interface{}
		if err := json.Unmarshal(payload, &emptyJSON); err != nil {
			log.WithError(err).Error(models.ErrInvalidJSON)
			c.JSON(http.StatusBadRequest, simpleError(models.ErrInvalidJSON))
			return
		}
	}

	log.WithField("payload", string(payload)).Debug("Got payload")

	appName := c.Param("app")
	if appName == "" {
		// check context, app can be added via special handlers
		a, ok := c.Get("app")
		if ok {
			appName = a.(string)
		}
	}
	// if still no appName, we gotta exit
	if appName == "" {
		log.WithError(err).Error(models.ErrAppsNotFound)
		c.JSON(http.StatusBadRequest, simpleError(models.ErrAppsNotFound))
		return
	}
	route := c.Param("route")
	if route == "" {
		route = c.Request.URL.Path
	}

	log.WithFields(logrus.Fields{"app": appName, "path": route}).Debug("Finding route on datastore")

	app, err := Api.Datastore.GetApp(appName)
	if err != nil || app == nil {
		log.WithError(err).Error(models.ErrAppsNotFound)
		c.JSON(http.StatusNotFound, simpleError(models.ErrAppsNotFound))
		return
	}

	routes, err := Api.Datastore.GetRoutesByApp(appName, &models.RouteFilter{})
	if err != nil {
		log.WithError(err).Error(models.ErrRoutesList)
		c.JSON(http.StatusInternalServerError, simpleError(models.ErrRoutesList))
		return
	}

	log.WithField("routes", routes).Debug("Got routes from datastore")
	for _, el := range routes {
		if params, match := matchRoute(el.Path, route); match {

			var stdout bytes.Buffer // TODO: should limit the size of this, error if gets too big. akin to: https://golang.org/pkg/io/#LimitReader
			stderr := runner.NewFuncLogger(appName, route, el.Image, reqID)

			envVars := map[string]string{
				"METHOD":      c.Request.Method,
				"ROUTE":       el.Path,
				"PAYLOAD":     string(payload),
				"REQUEST_URL": c.Request.URL.String(),
			}

			// app config
			for k, v := range app.Config {
				envVars["CONFIG_"+strings.ToUpper(k)] = v
			}

			// route config
			for k, v := range el.Config {
				envVars["CONFIG_"+strings.ToUpper(k)] = v
			}

			// params
			for _, param := range params {
				envVars["PARAM_"+strings.ToUpper(param.Key)] = param.Value
			}

			// headers
			for header, value := range c.Request.Header {
				envVars["HEADER_"+strings.ToUpper(header)] = strings.Join(value, " ")
			}

			cfg := &runner.Config{
				Image:   el.Image,
				Timeout: 30 * time.Second,
				ID:      reqID,
				AppName: appName,
				Stdout:  &stdout,
				Stderr:  stderr,
				Env:     envVars,
			}

			if result, err := Api.Runner.Run(c, cfg); err != nil {
				log.WithError(err).Error(models.ErrRunnerRunRoute)
				c.JSON(http.StatusInternalServerError, simpleError(models.ErrRunnerRunRoute))
			} else {
				for k, v := range el.Headers {
					c.Header(k, v[0])
				}

				if result.Status() == "success" {
					c.Data(http.StatusOK, "", stdout.Bytes())
				} else {
					// log.WithFields(logrus.Fields{"app": appName, "route": el, "req_id": reqID}).Debug(stderr.String())
					c.AbortWithStatus(http.StatusInternalServerError)
				}
			}
			return
		}
	}

	log.WithError(err).Error(models.ErrRunnerRouteNotFound)
	c.JSON(http.StatusNotFound, simpleError(models.ErrRunnerRouteNotFound))
}

var fakeHandler = func(http.ResponseWriter, *http.Request, Params) {}

func matchRoute(baseRoute, route string) (Params, bool) {
	tree := &node{}
	tree.addRoute(baseRoute, fakeHandler)
	handler, p, _ := tree.getValue(route)
	if handler == nil {
		return nil, false
	}

	return p, true
}
<commit_msg>runner metrics<commit_after>package server

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"golang.org/x/net/context"

	"encoding/json"

	"github.com/Sirupsen/logrus"
	"github.com/gin-gonic/gin"
	"github.com/iron-io/functions/api/models"
	"github.com/iron-io/functions/api/runner"
	titancommon "github.com/iron-io/titan/common"
	"github.com/satori/go.uuid"
)

func handleSpecial(c *gin.Context) {
	ctx := c.MustGet("ctx").(context.Context)
	log := titancommon.Logger(ctx)

	err := Api.UseSpecialHandlers(c)
	if err != nil {
		log.WithError(err).Errorln("Error using special handler!")
		// todo: what do we do here? Should probably return a 500 or something
	}
}

func handleRunner(c *gin.Context) {
	if strings.HasPrefix(c.Request.URL.Path, "/v1") {
		c.Status(http.StatusNotFound)
		return
	}

	ctx := c.MustGet("ctx").(context.Context)
	log := titancommon.Logger(ctx)

	reqID := uuid.NewV5(uuid.Nil, fmt.Sprintf("%s%s%d", c.Request.RemoteAddr, c.Request.URL.Path, time.Now().Unix())).String()
	c.Set("reqID", reqID) // todo: put this in the ctx instead of gin's

	log = log.WithFields(logrus.Fields{"request_id": reqID})
	// Request count metric
	log.WithFields(logrus.Fields{"metric": "function-request", "type": "count", "value": 1}).Info()

	var err error

	var payload []byte
	if c.Request.Method == "POST" || c.Request.Method == "PUT" {
		payload, err = ioutil.ReadAll(c.Request.Body)
	} else if c.Request.Method == "GET" {
		qPL := c.Request.URL.Query()["payload"]
		if len(qPL) > 0 {
			payload = []byte(qPL[0])
		}
	}

	if len(payload) > 0 {
		var emptyJSON map[string]interface{}
		if err := json.Unmarshal(payload, &emptyJSON); err != nil {
			log.WithError(err).Error(models.ErrInvalidJSON)
			c.JSON(http.StatusBadRequest, simpleError(models.ErrInvalidJSON))
			return
		}
	}

	log.WithField("payload", string(payload)).Debug("Got payload")

	appName := c.Param("app")
	if appName == "" {
		// check context, app can be added via special handlers
		a, ok := c.Get("app")
		if ok {
			appName = a.(string)
		}
	}
	// if still no appName, we gotta exit
	if appName == "" {
		log.WithError(err).Error(models.ErrAppsNotFound)
		c.JSON(http.StatusBadRequest, simpleError(models.ErrAppsNotFound))
		return
	}
	route := c.Param("route")
	if route == "" {
		route = c.Request.URL.Path
	}

	log.WithFields(logrus.Fields{"app": appName, "path": route}).Debug("Finding route on datastore")

	app, err := Api.Datastore.GetApp(appName)
	if err != nil || app == nil {
		log.WithError(err).Error(models.ErrAppsNotFound)
		c.JSON(http.StatusNotFound, simpleError(models.ErrAppsNotFound))
		return
	}

	routes, err := Api.Datastore.GetRoutesByApp(appName, &models.RouteFilter{})
	if err != nil {
		log.WithError(err).Error(models.ErrRoutesList)
		c.JSON(http.StatusInternalServerError, simpleError(models.ErrRoutesList))
		return
	}

	log.WithField("routes", routes).Debug("Got routes from datastore")
	for _, el := range routes {
		if params, match := matchRoute(el.Path, route); match {

			var stdout bytes.Buffer // TODO: should limit the size of this, error if gets too big. akin to: https://golang.org/pkg/io/#LimitReader
			stderr := runner.NewFuncLogger(appName, route, el.Image, reqID)

			envVars := map[string]string{
				"METHOD":      c.Request.Method,
				"ROUTE":       el.Path,
				"PAYLOAD":     string(payload),
				"REQUEST_URL": c.Request.URL.String(),
			}

			// app config
			for k, v := range app.Config {
				envVars["CONFIG_"+strings.ToUpper(k)] = v
			}

			// route config
			for k, v := range el.Config {
				envVars["CONFIG_"+strings.ToUpper(k)] = v
			}

			// params
			for _, param := range params {
				envVars["PARAM_"+strings.ToUpper(param.Key)] = param.Value
			}

			// headers
			for header, value := range c.Request.Header {
				envVars["HEADER_"+strings.ToUpper(header)] = strings.Join(value, " ")
			}

			cfg := &runner.Config{
				Image:   el.Image,
				Timeout: 30 * time.Second,
				ID:      reqID,
				AppName: appName,
				Stdout:  &stdout,
				Stderr:  stderr,
				Env:     envVars,
			}

			metricStart := time.Now()
			if result, err := Api.Runner.Run(c, cfg); err != nil {
				log.WithError(err).Error(models.ErrRunnerRunRoute)
				c.JSON(http.StatusInternalServerError, simpleError(models.ErrRunnerRunRoute))
			} else {
				for k, v := range el.Headers {
					c.Header(k, v[0])
				}

				if result.Status() == "success" {
					c.Data(http.StatusOK, "", stdout.Bytes())
				} else {
					// log.WithFields(logrus.Fields{"app": appName, "route": el, "req_id": reqID}).Debug(stderr.String())
					// Error count metric
					log.WithFields(logrus.Fields{
						"app": appName, "route": el.Path, "image": el.Image, "req_id": reqID,
						"metric": "function-error", "type": "count", "value": 1}).Info()

					c.AbortWithStatus(http.StatusInternalServerError)
				}
			}
			// Execution time metric
			metricElapsed := time.Since(metricStart)
			log.WithFields(logrus.Fields{
				"app": appName, "route": el.Path, "image": el.Image, "req_id": reqID,
				"metric": "function-runner", "type": "time", "value": metricElapsed}).Info()
			return
		}
	}

	log.WithError(err).Error(models.ErrRunnerRouteNotFound)
	c.JSON(http.StatusNotFound, simpleError(models.ErrRunnerRouteNotFound))
}

var fakeHandler = func(http.ResponseWriter, *http.Request, Params) {}

func matchRoute(baseRoute, route string) (Params, bool) {
	tree := &node{}
	tree.addRoute(baseRoute, fakeHandler)
	handler, p, _ := tree.getValue(route)
	if handler == nil {
		return nil, false
	}

	return p, true
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/ccirello/supervisor"
	"github.com/fnproject/fn/api"
	"github.com/fnproject/fn/api/datastore"
	"github.com/fnproject/fn/api/id"
	"github.com/fnproject/fn/api/logs"
	"github.com/fnproject/fn/api/models"
	"github.com/fnproject/fn/api/mqs"
	"github.com/fnproject/fn/api/runner"
	"github.com/fnproject/fn/api/runner/common"
	"github.com/gin-gonic/gin"
	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	"github.com/openzipkin/zipkin-go-opentracing"
	"github.com/patrickmn/go-cache"
	"github.com/spf13/viper"
)

const (
	EnvLogLevel  = "log_level"
	EnvMQURL     = "mq_url"
	EnvDBURL     = "db_url"
	EnvLOGDBURL  = "logstore_url"
	EnvPort      = "port" // be careful, Gin expects this variable to be "port"
	EnvAPIURL    = "api_url"
	EnvZipkinURL = "zipkin_url"
)

type Server struct {
	Datastore models.Datastore
	Runner    *runner.Runner
	Router    *gin.Engine
	MQ        models.MessageQueue
	Enqueue   models.Enqueue
	LogDB     models.FnLog

	apiURL string

	specialHandlers []SpecialHandler
	appListeners    []AppListener
	middlewares     []Middleware
	runnerListeners []RunnerListener

	routeCache   *cache.Cache
	singleflight singleflight // singleflight assists Datastore
}

const cacheSize = 1024

// NewFromEnv creates a new Functions server based on env vars.
func NewFromEnv(ctx context.Context) *Server {
	ds, err := datastore.New(viper.GetString(EnvDBURL))
	if err != nil {
		logrus.WithError(err).Fatalln("Error initializing datastore.")
	}

	mq, err := mqs.New(viper.GetString(EnvMQURL))
	if err != nil {
		logrus.WithError(err).Fatal("Error initializing message queue.")
	}

	var logDB models.FnLog = ds
	if ldb := viper.GetString(EnvLOGDBURL); ldb != "" && ldb != viper.GetString(EnvDBURL) {
		logDB, err = logs.New(viper.GetString(EnvLOGDBURL))
		if err != nil {
			logrus.WithError(err).Fatal("Error initializing logs store.")
		}
	}

	apiURL := viper.GetString(EnvAPIURL)

	return New(ctx, ds, mq, logDB, apiURL)
}

// New creates a new Functions server with the passed in datastore, message queue and API URL
func New(ctx context.Context, ds models.Datastore, mq models.MessageQueue, logDB models.FnLog, apiURL string, opts ...ServerOption) *Server {
	funcLogger := runner.NewFuncLogger(logDB)

	rnr, err := runner.New(ctx, funcLogger, ds)
	if err != nil {
		logrus.WithError(err).Fatalln("Failed to create a runner")
		return nil
	}

	s := &Server{
		Runner:     rnr,
		Router:     gin.New(),
		Datastore:  ds,
		MQ:         mq,
		routeCache: cache.New(5*time.Second, 5*time.Minute),
		LogDB:      logDB,
		Enqueue:    DefaultEnqueue,
		apiURL:     apiURL,
	}

	setMachineId()
	setTracer()
	s.Router.Use(loggerWrap, traceWrap)
	s.bindHandlers(ctx)

	for _, opt := range opts {
		if opt == nil {
			continue
		}
		opt(s)
	}
	return s
}

// we should use http grr
func traceWrap(c *gin.Context) {
	// try to grab a span from the request if made from another service, ignore err if not
	wireContext, _ := opentracing.GlobalTracer().Extract(
		opentracing.HTTPHeaders,
		opentracing.HTTPHeadersCarrier(c.Request.Header))

	// Create the span referring to the RPC client if available.
	// If wireContext == nil, a root span will be created.
	// TODO we should add more tags?
	serverSpan := opentracing.StartSpan("serve_http", ext.RPCServerOption(wireContext), opentracing.Tag{"path", c.Request.URL.Path})
	defer serverSpan.Finish()

	ctx := opentracing.ContextWithSpan(c.Request.Context(), serverSpan)
	c.Request = c.Request.WithContext(ctx)
	c.Next()
}

func setTracer() {
	var (
		debugMode          = false
		serviceName        = "fn-server"
		serviceHostPort    = "localhost:8080" // meh
		zipkinHTTPEndpoint = viper.GetString(EnvZipkinURL)
		// ex: "http://zipkin:9411/api/v1/spans"
	)

	if zipkinHTTPEndpoint == "" {
		return
	}

	logger := zipkintracer.LoggerFunc(func(i ...interface{}) error { logrus.Error(i...); return nil })

	collector, err := zipkintracer.NewHTTPCollector(zipkinHTTPEndpoint, zipkintracer.HTTPLogger(logger))
	if err != nil {
		logrus.WithError(err).Fatalln("couldn't start trace collector")
	}
	tracer, err := zipkintracer.NewTracer(zipkintracer.NewRecorder(collector, debugMode, serviceHostPort, serviceName),
		zipkintracer.ClientServerSameSpan(true),
		zipkintracer.TraceID128Bit(true),
	)
	if err != nil {
		logrus.WithError(err).Fatalln("couldn't start tracer")
	}

	opentracing.SetGlobalTracer(tracer)
	logrus.WithFields(logrus.Fields{"url": zipkinHTTPEndpoint}).Info("started tracer")
}

func setMachineId() {
	port := uint16(viper.GetInt(EnvPort))
	addr := whoAmI().To4()
	if addr == nil {
		addr = net.ParseIP("127.0.0.1").To4()
		logrus.Warn("could not find non-local ipv4 address to use, using '127.0.0.1' for ids, if this is a cluster beware of duplicate ids!")
	}
	id.SetMachineIdHost(addr, port)
}

// whoAmI searches for a non-local address on any network interface, returning
// the first one it finds. it could be expanded to search eth0 or en0 only but
// to date this has been unnecessary.
func whoAmI() net.IP {
	ints, _ := net.Interfaces()
	for _, i := range ints {
		if i.Name == "docker0" || i.Name == "lo" {
			// not perfect
			continue
		}
		addrs, _ := i.Addrs()
		for _, a := range addrs {
			ip, _, err := net.ParseCIDR(a.String())
			if a.Network() == "ip+net" && err == nil && ip.To4() != nil {
				if !bytes.Equal(ip, net.ParseIP("127.0.0.1")) {
					return ip
				}
			}
		}
	}
	return nil
}

func loggerWrap(c *gin.Context) {
	ctx, _ := common.LoggerWithFields(c.Request.Context(), extractFields(c))

	if appName := c.Param(api.CApp); appName != "" {
		c.Set(api.AppName, appName)
	}

	if routePath := c.Param(api.CRoute); routePath != "" {
		c.Set(api.Path, routePath)
	}

	c.Request = c.Request.WithContext(ctx)
	c.Next()
}

func DefaultEnqueue(ctx context.Context, mq models.MessageQueue, task *models.Task) (*models.Task, error) {
	ctx, _ = common.LoggerWithFields(ctx, logrus.Fields{"call_id": task.ID})
	return mq.Push(ctx, task)
}

func routeCacheKey(appname, path string) string {
	return fmt.Sprintf("%s_%s", appname, path)
}
func (s *Server) cacheget(appname, path string) (*models.Route, bool) {
	route, ok := s.routeCache.Get(routeCacheKey(appname, path))
	if !ok {
		return nil, false
	}
	return route.(*models.Route), ok
}

func (s *Server) cachedelete(appname, path string) {
	s.routeCache.Delete(routeCacheKey(appname, path))
}

func (s *Server) handleRunnerRequest(c *gin.Context) {
	s.handleRequest(c, s.Enqueue)
}

func (s *Server) handleTaskRequest(c *gin.Context) {
	ctx, _ := common.LoggerWithFields(c, nil)
	switch c.Request.Method {
	case "GET":
		task, err := s.MQ.Reserve(ctx)
		if err != nil {
			handleErrorResponse(c, err)
			return
		}
		c.JSON(http.StatusAccepted, task)
	case "DELETE":
		body, err := ioutil.ReadAll(c.Request.Body)
		if err != nil {
			handleErrorResponse(c, err)
			return
		}
		var task models.Task
		if err = json.Unmarshal(body, &task); err != nil {
			handleErrorResponse(c, err)
			return
		}

		if err := s.MQ.Delete(ctx, &task); err != nil {
			handleErrorResponse(c, err)
			return
		}
		c.JSON(http.StatusAccepted, task)
	}
}

func extractFields(c *gin.Context) logrus.Fields {
	fields := logrus.Fields{"action": path.Base(c.HandlerName())}
	for _, param := range c.Params {
		fields[param.Key] = param.Value
	}
	return fields
}

func (s *Server) Start(ctx context.Context) {
	ctx = contextWithSignal(ctx, os.Interrupt)
	s.startGears(ctx)
}

func (s *Server) startGears(ctx context.Context) {
	// By default it serves on :8080 unless a
	// PORT environment variable was defined.
	listen := fmt.Sprintf(":%d", viper.GetInt(EnvPort))
	listener, err := net.Listen("tcp", listen)
	if err != nil {
		logrus.WithError(err).Fatalln("Failed to serve functions API.")
	}

	const runHeader = `
	     ____                  __
	    / __ \_________ ______/ /__
	   / / / / ___/ __ / ___/ / _  \
	  / /_/ / /  / /_/ / /__/ /  __/
	  \_________ \__,_/\___/_/\____
	     / ____/_  __ ___  _____/ /_( )___  ____  _____
	    / /_  / / / / __ \/ ___/ __/ / __ \/ __ \/ ___/
	   / __/ / /_/ / / / / /__/ /_/ / /_/ / / / (__  )
	  /_/    \____/_/ /_/\___/\__/_/\____/_/ /_/____/
	`
	fmt.Println(runHeader)
	logrus.Infof("Serving Functions API on address `%s`", listen)

	svr := &supervisor.Supervisor{
		MaxRestarts: supervisor.AlwaysRestart,
		Log: func(msg interface{}) {
			logrus.Debug("supervisor: ", msg)
		},
	}

	svr.AddFunc(func(ctx context.Context) {
		go func() {
			err := http.Serve(listener, s.Router)
			if err != nil {
				logrus.Fatalf("Error serving API: %v", err)
			}
		}()
		<-ctx.Done()
	})

	svr.AddFunc(func(ctx context.Context) {
		runner.RunAsyncRunner(ctx, s.apiURL, s.Runner, s.Datastore)
	})

	svr.Serve(ctx)
	s.Runner.Wait() // wait for tasks to finish (safe shutdown)
}

func (s *Server) bindHandlers(ctx context.Context) {
	engine := s.Router

	engine.GET("/", handlePing)
	engine.GET("/version", handleVersion)
	engine.GET("/stats", s.handleStats)

	v1 := engine.Group("/v1")
	v1.Use(s.middlewareWrapperFunc(ctx))
	{
		v1.GET("/apps", s.handleAppList)
		v1.POST("/apps", s.handleAppCreate)

		v1.GET("/apps/:app", s.handleAppGet)
		v1.PATCH("/apps/:app", s.handleAppUpdate)
		v1.DELETE("/apps/:app", s.handleAppDelete)

		apps := v1.Group("/apps/:app")
		{
			apps.GET("/routes", s.handleRouteList)
			apps.POST("/routes", s.handleRouteCreateOrUpdate)
			apps.GET("/routes/*route", s.handleRouteGet)
			apps.PATCH("/routes/*route", s.handleRouteCreateOrUpdate)
			apps.PUT("/routes/*route", s.handleRouteCreateOrUpdate)
			apps.DELETE("/routes/*route", s.handleRouteDelete)

			apps.GET("/calls", s.handleCallList)

			apps.GET("/calls/:call", s.handleCallGet)
			apps.GET("/calls/:call/log", s.handleCallLogGet)
			apps.DELETE("/calls/:call/log", s.handleCallLogDelete)

		}
	}

	engine.DELETE("/tasks", s.handleTaskRequest)
	engine.GET("/tasks", s.handleTaskRequest)
	engine.Any("/r/:app", s.handleRunnerRequest)
	engine.Any("/r/:app/*route", s.handleRunnerRequest)

	// This final route is used for extensions, see Server.Add
	engine.NoRoute(s.handleSpecial)
}

type appResponse struct {
	Message string      `json:"message"`
	App     *models.App `json:"app"`
}

type appsResponse struct {
	Message string      `json:"message"`
	Apps    models.Apps `json:"apps"`
}

type routeResponse struct {
	Message string        `json:"message"`
	Route   *models.Route `json:"route"`
}

type routesResponse struct {
	Message string        `json:"message"`
	Routes  models.Routes `json:"routes"`
}

type tasksResponse struct {
	Message string      `json:"message"`
	Task    models.Task `json:"tasksResponse"`
}

type fnCallResponse struct {
	Message string         `json:"message"`
	Call    *models.FnCall `json:"call"`
}

type fnCallsResponse struct {
	Message string         `json:"message"`
	Calls   models.FnCalls `json:"calls"`
}

type fnCallLogResponse struct {
	Message string            `json:"message"`
	Log     *models.FnCallLog `json:"log"`
}
<commit_msg>Add appname and route path to context<commit_after>package server

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/ccirello/supervisor"
	"github.com/fnproject/fn/api"
	"github.com/fnproject/fn/api/datastore"
	"github.com/fnproject/fn/api/id"
	"github.com/fnproject/fn/api/logs"
	"github.com/fnproject/fn/api/models"
	"github.com/fnproject/fn/api/mqs"
	"github.com/fnproject/fn/api/runner"
	"github.com/fnproject/fn/api/runner/common"
	"github.com/gin-gonic/gin"
	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	"github.com/openzipkin/zipkin-go-opentracing"
	"github.com/patrickmn/go-cache"
	"github.com/spf13/viper"
)

const (
	EnvLogLevel  = "log_level"
	EnvMQURL     = "mq_url"
	EnvDBURL     = "db_url"
	EnvLOGDBURL  = "logstore_url"
	EnvPort      = "port" // be careful, Gin expects this variable to be "port"
	EnvAPIURL    = "api_url"
	EnvZipkinURL = "zipkin_url"
)

type Server struct {
	Datastore models.Datastore
	Runner    *runner.Runner
	Router    *gin.Engine
	MQ        models.MessageQueue
	Enqueue   models.Enqueue
	LogDB     models.FnLog

	apiURL string

	specialHandlers []SpecialHandler
	appListeners    []AppListener
	middlewares     []Middleware
	runnerListeners []RunnerListener

	routeCache   *cache.Cache
	singleflight singleflight // singleflight assists Datastore
}

const cacheSize = 1024

// NewFromEnv creates a new Functions server based on env vars.
func NewFromEnv(ctx context.Context) *Server {
	ds, err := datastore.New(viper.GetString(EnvDBURL))
	if err != nil {
		logrus.WithError(err).Fatalln("Error initializing datastore.")
	}

	mq, err := mqs.New(viper.GetString(EnvMQURL))
	if err != nil {
		logrus.WithError(err).Fatal("Error initializing message queue.")
	}

	var logDB models.FnLog = ds
	if ldb := viper.GetString(EnvLOGDBURL); ldb != "" && ldb != viper.GetString(EnvDBURL) {
		logDB, err = logs.New(viper.GetString(EnvLOGDBURL))
		if err != nil {
			logrus.WithError(err).Fatal("Error initializing logs store.")
		}
	}

	apiURL := viper.GetString(EnvAPIURL)

	return New(ctx, ds, mq, logDB, apiURL)
}

// New creates a new Functions server with the passed in datastore, message queue and API URL
func New(ctx context.Context, ds models.Datastore, mq models.MessageQueue, logDB models.FnLog, apiURL string, opts ...ServerOption) *Server {
	funcLogger := runner.NewFuncLogger(logDB)

	rnr, err := runner.New(ctx, funcLogger, ds)
	if err != nil {
		logrus.WithError(err).Fatalln("Failed to create a runner")
		return nil
	}

	s := &Server{
		Runner:     rnr,
		Router:     gin.New(),
		Datastore:  ds,
		MQ:         mq,
		routeCache: cache.New(5*time.Second, 5*time.Minute),
		LogDB:      logDB,
		Enqueue:    DefaultEnqueue,
		apiURL:     apiURL,
	}

	setMachineId()
	setTracer()
	s.Router.Use(loggerWrap, traceWrap)
	s.bindHandlers(ctx)

	for _, opt := range opts {
		if opt == nil {
			continue
		}
		opt(s)
	}
	return s
}

// we should use http grr
func traceWrap(c *gin.Context) {
	// try to grab a span from the request if made from another service, ignore err if not
	wireContext, _ := opentracing.GlobalTracer().Extract(
		opentracing.HTTPHeaders,
		opentracing.HTTPHeadersCarrier(c.Request.Header))

	// Create the span referring to the RPC client if available.
	// If wireContext == nil, a root span will be created.
	// TODO we should add more tags?
	serverSpan := opentracing.StartSpan("serve_http", ext.RPCServerOption(wireContext), opentracing.Tag{"path", c.Request.URL.Path})
	defer serverSpan.Finish()

	ctx := opentracing.ContextWithSpan(c.Request.Context(), serverSpan)
	c.Request = c.Request.WithContext(ctx)
	c.Next()
}

func setTracer() {
	var (
		debugMode          = false
		serviceName        = "fn-server"
		serviceHostPort    = "localhost:8080" // meh
		zipkinHTTPEndpoint = viper.GetString(EnvZipkinURL)
		// ex: "http://zipkin:9411/api/v1/spans"
	)

	if zipkinHTTPEndpoint == "" {
		return
	}

	logger := zipkintracer.LoggerFunc(func(i ...interface{}) error { logrus.Error(i...); return nil })

	collector, err := zipkintracer.NewHTTPCollector(zipkinHTTPEndpoint, zipkintracer.HTTPLogger(logger))
	if err != nil {
		logrus.WithError(err).Fatalln("couldn't start trace collector")
	}
	tracer, err := zipkintracer.NewTracer(zipkintracer.NewRecorder(collector, debugMode, serviceHostPort, serviceName),
		zipkintracer.ClientServerSameSpan(true),
		zipkintracer.TraceID128Bit(true),
	)
	if err != nil {
		logrus.WithError(err).Fatalln("couldn't start tracer")
	}

	opentracing.SetGlobalTracer(tracer)
	logrus.WithFields(logrus.Fields{"url": zipkinHTTPEndpoint}).Info("started tracer")
}

func setMachineId() {
	port := uint16(viper.GetInt(EnvPort))
	addr := whoAmI().To4()
	if addr == nil {
		addr = net.ParseIP("127.0.0.1").To4()
		logrus.Warn("could not find non-local ipv4 address to use, using '127.0.0.1' for ids, if this is a cluster beware of duplicate ids!")
	}
	id.SetMachineIdHost(addr, port)
}

// whoAmI searches for a non-local address on any network interface, returning
// the first one it finds. it could be expanded to search eth0 or en0 only but
// to date this has been unnecessary.
func whoAmI() net.IP {
	ints, _ := net.Interfaces()
	for _, i := range ints {
		if i.Name == "docker0" || i.Name == "lo" {
			// not perfect
			continue
		}
		addrs, _ := i.Addrs()
		for _, a := range addrs {
			ip, _, err := net.ParseCIDR(a.String())
			if a.Network() == "ip+net" && err == nil && ip.To4() != nil {
				if !bytes.Equal(ip, net.ParseIP("127.0.0.1")) {
					return ip
				}
			}
		}
	}
	return nil
}

func loggerWrap(c *gin.Context) {
	ctx, _ := common.LoggerWithFields(c.Request.Context(), extractFields(c))

	if appName := c.Param(api.CApp); appName != "" {
		c.Set(api.AppName, appName)
		c.Request = c.Request.WithContext(context.WithValue(c.Request.Context(), api.AppName, appName))
	}

	if routePath := c.Param(api.CRoute); routePath != "" {
		c.Set(api.Path, routePath)
		c.Request = c.Request.WithContext(context.WithValue(c.Request.Context(), api.Path, routePath))
	}

	c.Request = c.Request.WithContext(ctx)
	c.Next()
}

func DefaultEnqueue(ctx context.Context, mq models.MessageQueue, task *models.Task) (*models.Task, error) {
	ctx, _ = common.LoggerWithFields(ctx, logrus.Fields{"call_id": task.ID})
	return mq.Push(ctx, task)
}

func routeCacheKey(appname, path string) string {
	return fmt.Sprintf("%s_%s", appname, path)
}
func (s *Server) cacheget(appname, path string) (*models.Route, bool) {
	route, ok := s.routeCache.Get(routeCacheKey(appname, path))
	if !ok {
		return nil, false
	}
	return route.(*models.Route), ok
}

func (s *Server) cachedelete(appname, path string) {
	s.routeCache.Delete(routeCacheKey(appname, path))
}

func (s *Server) handleRunnerRequest(c *gin.Context) {
	s.handleRequest(c, s.Enqueue)
}

func (s *Server) handleTaskRequest(c *gin.Context) {
	ctx, _ := common.LoggerWithFields(c, nil)
	switch c.Request.Method {
	case "GET":
		task, err := s.MQ.Reserve(ctx)
		if err != nil {
			handleErrorResponse(c, err)
			return
		}
		c.JSON(http.StatusAccepted, task)
	case "DELETE":
		body, err := ioutil.ReadAll(c.Request.Body)
		if err != nil {
			handleErrorResponse(c, err)
			return
		}
		var task models.Task
		if err = json.Unmarshal(body, &task); err != nil {
			handleErrorResponse(c, err)
			return
		}

		if err := s.MQ.Delete(ctx, &task); err != nil {
			handleErrorResponse(c, err)
			return
		}
		c.JSON(http.StatusAccepted, task)
	}
}

func extractFields(c *gin.Context) logrus.Fields {
	fields := logrus.Fields{"action": path.Base(c.HandlerName())}
	for _, param := range c.Params {
		fields[param.Key] = param.Value
	}
	return fields
}

func (s *Server) Start(ctx context.Context) {
	ctx = contextWithSignal(ctx, os.Interrupt)
	s.startGears(ctx)
}

func (s *Server) startGears(ctx context.Context) {
	// By default it serves on :8080 unless a
	// PORT environment variable was defined.
	listen := fmt.Sprintf(":%d", viper.GetInt(EnvPort))
	listener, err := net.Listen("tcp", listen)
	if err != nil {
		logrus.WithError(err).Fatalln("Failed to serve functions API.")
	}

	const runHeader = `
	     ____                  __
	    / __ \_________ ______/ /__
	   / / / / ___/ __ / ___/ / _  \
	  / /_/ / /  / /_/ / /__/ /  __/
	  \_________ \__,_/\___/_/\____
	     / ____/_  __ ___  _____/ /_( )___  ____  _____
	    / /_  / / / / __ \/ ___/ __/ / __ \/ __ \/ ___/
	   / __/ / /_/ / / / / /__/ /_/ / /_/ / / / (__  )
	  /_/    \____/_/ /_/\___/\__/_/\____/_/ /_/____/
	`
	fmt.Println(runHeader)
	logrus.Infof("Serving Functions API on address `%s`", listen)

	svr := &supervisor.Supervisor{
		MaxRestarts: supervisor.AlwaysRestart,
		Log: func(msg interface{}) {
			logrus.Debug("supervisor: ", msg)
		},
	}

	svr.AddFunc(func(ctx context.Context) {
		go func() {
			err := http.Serve(listener, s.Router)
			if err != nil {
				logrus.Fatalf("Error serving API: %v", err)
			}
		}()
		<-ctx.Done()
	})

	svr.AddFunc(func(ctx context.Context) {
		runner.RunAsyncRunner(ctx, s.apiURL, s.Runner, s.Datastore)
	})

	svr.Serve(ctx)
	s.Runner.Wait() // wait for tasks to finish (safe shutdown)
}

func (s *Server) bindHandlers(ctx context.Context) {
	engine := s.Router

	engine.GET("/", handlePing)
	engine.GET("/version", handleVersion)
	engine.GET("/stats", s.handleStats)

	v1 := engine.Group("/v1")
	v1.Use(s.middlewareWrapperFunc(ctx))
	{
		v1.GET("/apps", s.handleAppList)
		v1.POST("/apps", s.handleAppCreate)

		v1.GET("/apps/:app", s.handleAppGet)
		v1.PATCH("/apps/:app", s.handleAppUpdate)
		v1.DELETE("/apps/:app", s.handleAppDelete)

		apps := v1.Group("/apps/:app")
		{
			apps.GET("/routes", s.handleRouteList)
			apps.POST("/routes", s.handleRouteCreateOrUpdate)
			apps.GET("/routes/*route", s.handleRouteGet)
			apps.PATCH("/routes/*route", s.handleRouteCreateOrUpdate)
			apps.PUT("/routes/*route", s.handleRouteCreateOrUpdate)
			apps.DELETE("/routes/*route", s.handleRouteDelete)

			apps.GET("/calls", s.handleCallList)

			apps.GET("/calls/:call", s.handleCallGet)
			apps.GET("/calls/:call/log", s.handleCallLogGet)
			apps.DELETE("/calls/:call/log", s.handleCallLogDelete)

		}
	}

	engine.DELETE("/tasks", s.handleTaskRequest)
	engine.GET("/tasks", s.handleTaskRequest)
	engine.Any("/r/:app", s.handleRunnerRequest)
	engine.Any("/r/:app/*route", s.handleRunnerRequest)

	// This final route is used for extensions, see Server.Add
	engine.NoRoute(s.handleSpecial)
}

type appResponse struct {
	Message string      `json:"message"`
	App     *models.App `json:"app"`
}

type appsResponse struct {
	Message string      `json:"message"`
	Apps    models.Apps `json:"apps"`
}

type routeResponse struct {
	Message string        `json:"message"`
	Route   *models.Route `json:"route"`
}

type routesResponse struct {
	Message string        `json:"message"`
	Routes  models.Routes `json:"routes"`
}

type tasksResponse struct {
	Message string      `json:"message"`
	Task    models.Task `json:"tasksResponse"`
}

type fnCallResponse struct {
	Message string         `json:"message"`
	Call    *models.FnCall `json:"call"`
}

type fnCallsResponse struct {
	Message string         `json:"message"`
	Calls   models.FnCalls `json:"calls"`
}

type fnCallLogResponse struct {
	Message string            `json:"message"`
	Log     *models.FnCallLog `json:"log"`
}
<|endoftext|>