<commit_before>package pflag

import (
	"fmt"
	"io"
	"net"
	"strings"
)

// -- ipSlice Value
type ipSliceValue struct {
	value   *[]net.IP
	changed bool
}

func newIPSliceValue(val []net.IP, p *[]net.IP) *ipSliceValue {
	ipsv := new(ipSliceValue)
	ipsv.value = p
	*ipsv.value = val
	return ipsv
}

// Set converts, and assigns, the comma-separated IP argument string representation as the []net.IP value of this flag.
// If Set is called on a flag that already has a []net.IP assigned, the newly converted values will be appended.
func (s *ipSliceValue) Set(val string) error {

	// remove all quote characters
	rmQuote := strings.NewReplacer(`"`, "", `'`, "", "`", "")

	// read flag arguments with CSV parser
	ipStrSlice, err := readAsCSV(rmQuote.Replace(val))
	if err != nil && err != io.EOF {
		return err
	}

	// parse ip values into slice
	out := make([]net.IP, 0, len(ipStrSlice))
	for _, ipStr := range ipStrSlice {
		ip := net.ParseIP(strings.TrimSpace(ipStr))
		if ip == nil {
			return fmt.Errorf("invalid string being converted to IP address: %s", ipStr)
		}
		out = append(out, ip)
	}

	if !s.changed {
		*s.value = out
	} else {
		*s.value = append(*s.value, out...)
	}

	s.changed = true

	return nil
}

// Type returns a string that uniquely represents this flag's type.
func (s *ipSliceValue) Type() string {
	return "ipSlice"
}

// String defines a "native" format for this net.IP slice flag value.
func (s *ipSliceValue) String() string {

	ipStrSlice := make([]string, len(*s.value))
	for i, ip := range *s.value {
		ipStrSlice[i] = ip.String()
	}

	out, _ := writeAsCSV(ipStrSlice)

	return "[" + out + "]"
}

func ipSliceConv(val string) (interface{}, error) {
	val = strings.Trim(val, "[]")
	// Emtpy string would cause a slice with one (empty) entry
	if len(val) == 0 {
		return []net.IP{}, nil
	}
	ss := strings.Split(val, ",")
	out := make([]net.IP, len(ss))
	for i, sval := range ss {
		ip := net.ParseIP(strings.TrimSpace(sval))
		if ip == nil {
			return nil, fmt.Errorf("invalid string being converted to IP address: %s", sval)
		}
		out[i] = ip
	}
	return out, nil
}

// GetIPSlice returns the []net.IP value of a flag with the given name
func (f *FlagSet) GetIPSlice(name string) ([]net.IP, error) {
	val, err := f.getFlagType(name, "ipSlice", ipSliceConv)
	if err != nil {
		return []net.IP{}, err
	}
	return val.([]net.IP), nil
}

// IPSliceVar defines a ipSlice flag with specified name, default value, and usage string.
// The argument p points to a []net.IP variable in which to store the value of the flag.
func (f *FlagSet) IPSliceVar(p *[]net.IP, name string, value []net.IP, usage string) {
	f.VarP(newIPSliceValue(value, p), name, "", usage)
}

// IPSliceVarP is like IPSliceVar, but accepts a shorthand letter that can be used after a single dash.
func (f *FlagSet) IPSliceVarP(p *[]net.IP, name, shorthand string, value []net.IP, usage string) {
	f.VarP(newIPSliceValue(value, p), name, shorthand, usage)
}

// IPSliceVar defines a []net.IP flag with specified name, default value, and usage string.
// The argument p points to a []net.IP variable in which to store the value of the flag.
func IPSliceVar(p *[]net.IP, name string, value []net.IP, usage string) {
	CommandLine.VarP(newIPSliceValue(value, p), name, "", usage)
}

// IPSliceVarP is like IPSliceVar, but accepts a shorthand letter that can be used after a single dash.
func IPSliceVarP(p *[]net.IP, name, shorthand string, value []net.IP, usage string) {
	CommandLine.VarP(newIPSliceValue(value, p), name, shorthand, usage)
}

// IPSlice defines a []net.IP flag with specified name, default value, and usage string.
// The return value is the address of a []net.IP variable that stores the value of that flag.
func (f *FlagSet) IPSlice(name string, value []net.IP, usage string) *[]net.IP {
	p := []net.IP{}
	f.IPSliceVarP(&p, name, "", value, usage)
	return &p
}

// IPSliceP is like IPSlice, but accepts a shorthand letter that can be used after a single dash.
func (f *FlagSet) IPSliceP(name, shorthand string, value []net.IP, usage string) *[]net.IP {
	p := []net.IP{}
	f.IPSliceVarP(&p, name, shorthand, value, usage)
	return &p
}

// IPSlice defines a []net.IP flag with specified name, default value, and usage string.
// The return value is the address of a []net.IP variable that stores the value of the flag.
func IPSlice(name string, value []net.IP, usage string) *[]net.IP {
	return CommandLine.IPSliceP(name, "", value, usage)
}

// IPSliceP is like IPSlice, but accepts a shorthand letter that can be used after a single dash.
func IPSliceP(name, shorthand string, value []net.IP, usage string) *[]net.IP {
	return CommandLine.IPSliceP(name, shorthand, value, usage)
}
<commit_msg>fix misspell (#197)<commit_after>package pflag

import (
	"fmt"
	"io"
	"net"
	"strings"
)

// -- ipSlice Value
type ipSliceValue struct {
	value   *[]net.IP
	changed bool
}

func newIPSliceValue(val []net.IP, p *[]net.IP) *ipSliceValue {
	ipsv := new(ipSliceValue)
	ipsv.value = p
	*ipsv.value = val
	return ipsv
}

// Set converts, and assigns, the comma-separated IP argument string representation as the []net.IP value of this flag.
// If Set is called on a flag that already has a []net.IP assigned, the newly converted values will be appended.
func (s *ipSliceValue) Set(val string) error {

	// remove all quote characters
	rmQuote := strings.NewReplacer(`"`, "", `'`, "", "`", "")

	// read flag arguments with CSV parser
	ipStrSlice, err := readAsCSV(rmQuote.Replace(val))
	if err != nil && err != io.EOF {
		return err
	}

	// parse ip values into slice
	out := make([]net.IP, 0, len(ipStrSlice))
	for _, ipStr := range ipStrSlice {
		ip := net.ParseIP(strings.TrimSpace(ipStr))
		if ip == nil {
			return fmt.Errorf("invalid string being converted to IP address: %s", ipStr)
		}
		out = append(out, ip)
	}

	if !s.changed {
		*s.value = out
	} else {
		*s.value = append(*s.value, out...)
	}

	s.changed = true

	return nil
}

// Type returns a string that uniquely represents this flag's type.
func (s *ipSliceValue) Type() string {
	return "ipSlice"
}

// String defines a "native" format for this net.IP slice flag value.
func (s *ipSliceValue) String() string {

	ipStrSlice := make([]string, len(*s.value))
	for i, ip := range *s.value {
		ipStrSlice[i] = ip.String()
	}

	out, _ := writeAsCSV(ipStrSlice)

	return "[" + out + "]"
}

func ipSliceConv(val string) (interface{}, error) {
	val = strings.Trim(val, "[]")
	// Empty string would cause a slice with one (empty) entry
	if len(val) == 0 {
		return []net.IP{}, nil
	}
	ss := strings.Split(val, ",")
	out := make([]net.IP, len(ss))
	for i, sval := range ss {
		ip := net.ParseIP(strings.TrimSpace(sval))
		if ip == nil {
			return nil, fmt.Errorf("invalid string being converted to IP address: %s", sval)
		}
		out[i] = ip
	}
	return out, nil
}

// GetIPSlice returns the []net.IP value of a flag with the given name
func (f *FlagSet) GetIPSlice(name string) ([]net.IP, error) {
	val, err := f.getFlagType(name, "ipSlice", ipSliceConv)
	if err != nil {
		return []net.IP{}, err
	}
	return val.([]net.IP), nil
}

// IPSliceVar defines a ipSlice flag with specified name, default value, and usage string.
// The argument p points to a []net.IP variable in which to store the value of the flag.
func (f *FlagSet) IPSliceVar(p *[]net.IP, name string, value []net.IP, usage string) {
	f.VarP(newIPSliceValue(value, p), name, "", usage)
}

// IPSliceVarP is like IPSliceVar, but accepts a shorthand letter that can be used after a single dash.
func (f *FlagSet) IPSliceVarP(p *[]net.IP, name, shorthand string, value []net.IP, usage string) {
	f.VarP(newIPSliceValue(value, p), name, shorthand, usage)
}

// IPSliceVar defines a []net.IP flag with specified name, default value, and usage string.
// The argument p points to a []net.IP variable in which to store the value of the flag.
func IPSliceVar(p *[]net.IP, name string, value []net.IP, usage string) {
	CommandLine.VarP(newIPSliceValue(value, p), name, "", usage)
}

// IPSliceVarP is like IPSliceVar, but accepts a shorthand letter that can be used after a single dash.
func IPSliceVarP(p *[]net.IP, name, shorthand string, value []net.IP, usage string) {
	CommandLine.VarP(newIPSliceValue(value, p), name, shorthand, usage)
}

// IPSlice defines a []net.IP flag with specified name, default value, and usage string.
// The return value is the address of a []net.IP variable that stores the value of that flag.
func (f *FlagSet) IPSlice(name string, value []net.IP, usage string) *[]net.IP {
	p := []net.IP{}
	f.IPSliceVarP(&p, name, "", value, usage)
	return &p
}

// IPSliceP is like IPSlice, but accepts a shorthand letter that can be used after a single dash.
func (f *FlagSet) IPSliceP(name, shorthand string, value []net.IP, usage string) *[]net.IP {
	p := []net.IP{}
	f.IPSliceVarP(&p, name, shorthand, value, usage)
	return &p
}

// IPSlice defines a []net.IP flag with specified name, default value, and usage string.
// The return value is the address of a []net.IP variable that stores the value of the flag.
func IPSlice(name string, value []net.IP, usage string) *[]net.IP {
	return CommandLine.IPSliceP(name, "", value, usage)
}

// IPSliceP is like IPSlice, but accepts a shorthand letter that can be used after a single dash.
func IPSliceP(name, shorthand string, value []net.IP, usage string) *[]net.IP {
	return CommandLine.IPSliceP(name, shorthand, value, usage)
}
<|endoftext|>


<commit_before>package kontrol

import (
	"errors"
	"strings"

	"github.com/coreos/go-etcd/etcd"
	"github.com/koding/kite"
)

// Storage is an interface to a kite storage.
type Storage interface {
	Get(key string) (*Node, error)
	Set(key, value string) error
	Delete(key string) error
	Watch(key string, index uint64) (*Watcher, error)
}

type Etcd struct {
	client *etcd.Client
	log    kite.Logger
}

type Watcher struct {
	recv chan *etcd.Response
	stop chan bool
}

func (w *Watcher) Stop() {
	w.stop <- true
}

func NewEtcd(machines []string) (*Etcd, error) {
	if machines == nil || len(machines) == 0 {
		return nil, errors.New("machines is empty")
	}

	client := etcd.NewClient(machines)
	ok := client.SetCluster(machines)
	if !ok {
		return nil, errors.New("cannot connect to etcd cluster: " + strings.Join(machines, ","))
	}

	return &Etcd{
		client: client,
	}, nil
}

func (e *Etcd) Delete(key string) error {
	_, err := e.client.Delete(key, true)
	return err
}

func (e *Etcd) Set(key, value string) error {
	_, err := e.client.Set(key, value, uint64(HeartbeatDelay))
	return err
}

func (e *Etcd) Update(key, value string) error {
	_, err := e.client.Update(key, value, uint64(HeartbeatDelay))
	return err
}

func (e *Etcd) Watch(key string, index uint64) (*Watcher, error) {
	// TODO: make the buffer configurable
	responses := make(chan *etcd.Response, 1000)
	stopChan := make(chan bool, 1)

	// Watch is blocking
	go func() {
		_, err := e.client.Watch(key, index, true, responses, stopChan)
		if err != nil {
			e.log.Warning("Remote client closed the watcher explicitly. Etcd client error: %s", err)
		}
	}()

	return &Watcher{
		recv: responses,
		stop: stopChan,
	}, nil
}

func (e *Etcd) Get(key string) (*Node, error) {
	resp, err := e.client.Get(key, false, true)
	if err != nil {
		return nil, err
	}

	return NewNode(resp.Node), nil
}
<commit_msg>kontrol: fix TTL values for etcd client<commit_after>package kontrol

import (
	"errors"
	"strings"
	"time"

	"github.com/coreos/go-etcd/etcd"
	"github.com/koding/kite"
)

// Storage is an interface to a kite storage.
type Storage interface {
	Get(key string) (*Node, error)
	Set(key, value string) error
	Delete(key string) error
	Watch(key string, index uint64) (*Watcher, error)
}

type Etcd struct {
	client *etcd.Client
	log    kite.Logger
}

type Watcher struct {
	recv chan *etcd.Response
	stop chan bool
}

func (w *Watcher) Stop() {
	w.stop <- true
}

func NewEtcd(machines []string) (*Etcd, error) {
	if machines == nil || len(machines) == 0 {
		return nil, errors.New("machines is empty")
	}

	client := etcd.NewClient(machines)
	ok := client.SetCluster(machines)
	if !ok {
		return nil, errors.New("cannot connect to etcd cluster: " + strings.Join(machines, ","))
	}

	return &Etcd{
		client: client,
	}, nil
}

func (e *Etcd) Delete(key string) error {
	_, err := e.client.Delete(key, true)
	return err
}

func (e *Etcd) Set(key, value string) error {
	_, err := e.client.Set(key, value, uint64(HeartbeatDelay/time.Second))
	return err
}

func (e *Etcd) Update(key, value string) error {
	_, err := e.client.Update(key, value, uint64(HeartbeatDelay/time.Second))
	return err
}

func (e *Etcd) Watch(key string, index uint64) (*Watcher, error) {
	// TODO: make the buffer configurable
	responses := make(chan *etcd.Response, 1000)
	stopChan := make(chan bool, 1)

	// Watch is blocking
	go func() {
		_, err := e.client.Watch(key, index, true, responses, stopChan)
		if err != nil {
			e.log.Warning("Remote client closed the watcher explicitly. Etcd client error: %s", err)
		}
	}()

	return &Watcher{
		recv: responses,
		stop: stopChan,
	}, nil
}

func (e *Etcd) Get(key string) (*Node, error) {
	resp, err := e.client.Get(key, false, true)
	if err != nil {
		return nil, err
	}

	return NewNode(resp.Node), nil
}
<|endoftext|>


<commit_before>package iptables

import (
	"errors"
	"fmt"
	"net"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

type Action string

const (
	Add    Action = "-A"
	Delete Action = "-D"
)

var (
	ErrIptablesNotFound = errors.New("Iptables not found")
	nat                 = []string{"-t", "nat"}
	supportsXlock       = false
)

type Chain struct {
	Name   string
	Bridge string
}

func init() {
	supportsXlock = exec.Command("iptables", "--wait", "-L", "-n").Run() == nil
}

func NewChain(name, bridge string) (*Chain, error) {
	if output, err := Raw("-t", "nat", "-N", name); err != nil {
		return nil, err
	} else if len(output) != 0 {
		return nil, fmt.Errorf("Error creating new iptables chain: %s", output)
	}
	chain := &Chain{
		Name:   name,
		Bridge: bridge,
	}

	if err := chain.Prerouting(Add, "-m", "addrtype", "--dst-type", "LOCAL"); err != nil {
		return nil, fmt.Errorf("Failed to inject docker in PREROUTING chain: %s", err)
	}
	if err := chain.Output(Add, "-m", "addrtype", "--dst-type", "LOCAL", "!", "--dst", "127.0.0.0/8"); err != nil {
		return nil, fmt.Errorf("Failed to inject docker in OUTPUT chain: %s", err)
	}
	return chain, nil
}

func RemoveExistingChain(name string) error {
	chain := &Chain{
		Name: name,
	}
	return chain.Remove()
}

func (c *Chain) Forward(action Action, ip net.IP, port int, proto, dest_addr string, dest_port int) error {
	daddr := ip.String()
	if ip.IsUnspecified() {
		// iptables interprets "0.0.0.0" as "0.0.0.0/32", whereas we
		// want "0.0.0.0/0". "0/0" is correctly interpreted as "any
		// value" by both iptables and ip6tables.
		daddr = "0/0"
	}
	if output, err := Raw("-t", "nat", fmt.Sprint(action), c.Name,
		"-p", proto,
		"-d", daddr,
		"--dport", strconv.Itoa(port),
		"!", "-i", c.Bridge,
		"-j", "DNAT",
		"--to-destination", net.JoinHostPort(dest_addr, strconv.Itoa(dest_port))); err != nil {
		return err
	} else if len(output) != 0 {
		return fmt.Errorf("Error iptables forward: %s", output)
	}

	fAction := action
	if fAction == Add {
		fAction = "-I"
	}
	if output, err := Raw(string(fAction), "FORWARD",
		"!", "-i", c.Bridge,
		"-o", c.Bridge,
		"-p", proto,
		"-d", dest_addr,
		"--dport", strconv.Itoa(dest_port),
		"-j", "ACCEPT"); err != nil {
		return err
	} else if len(output) != 0 {
		return fmt.Errorf("Error iptables forward: %s", output)
	}

	return nil
}

func (c *Chain) Prerouting(action Action, args ...string) error {
	a := append(nat, fmt.Sprint(action), "PREROUTING")
	if len(args) > 0 {
		a = append(a, args...)
	}
	if output, err := Raw(append(a, "-j", c.Name)...); err != nil {
		return err
	} else if len(output) != 0 {
		return fmt.Errorf("Error iptables prerouting: %s", output)
	}
	return nil
}

func (c *Chain) Output(action Action, args ...string) error {
	a := append(nat, fmt.Sprint(action), "OUTPUT")
	if len(args) > 0 {
		a = append(a, args...)
	}
	if output, err := Raw(append(a, "-j", c.Name)...); err != nil {
		return err
	} else if len(output) != 0 {
		return fmt.Errorf("Error iptables output: %s", output)
	}
	return nil
}

func (c *Chain) Remove() error {
	// Ignore errors - This could mean the chains were never set up
	c.Prerouting(Delete, "-m", "addrtype", "--dst-type", "LOCAL")
	c.Output(Delete, "-m", "addrtype", "--dst-type", "LOCAL", "!", "--dst", "127.0.0.0/8")
	c.Output(Delete, "-m", "addrtype", "--dst-type", "LOCAL") // Created in versions <= 0.1.6

	c.Prerouting(Delete)
	c.Output(Delete)

	Raw("-t", "nat", "-F", c.Name)
	Raw("-t", "nat", "-X", c.Name)

	return nil
}

// Check if an existing rule exists
func Exists(args ...string) bool {
	if _, err := Raw(append([]string{"-C"}, args...)...); err != nil {
		return false
	}
	return true
}

func Raw(args ...string) ([]byte, error) {
	path, err := exec.LookPath("iptables")
	if err != nil {
		return nil, ErrIptablesNotFound
	}

	if supportsXlock {
		args = append([]string{"--wait"}, args...)
	}

	if os.Getenv("DEBUG") != "" {
		fmt.Fprintf(os.Stderr, fmt.Sprintf("[debug] %s, %v\n", path, args))
	}

	output, err := exec.Command(path, args...).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("iptables failed: iptables %v: %s (%s)", strings.Join(args, " "), output, err)
	}

	// ignore iptables' message about xtables lock
	if strings.Contains(string(output), "waiting for it to exit") {
		output = []byte("")
	}

	return output, err
}
<commit_msg>pkg/iptables: Support hairpin NAT<commit_after>package iptables

import (
	"errors"
	"fmt"
	"net"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

type Action string

const (
	Add    Action = "-A"
	Delete Action = "-D"
)

var (
	ErrIptablesNotFound = errors.New("Iptables not found")
	nat                 = []string{"-t", "nat"}
	supportsXlock       = false
)

type Chain struct {
	Name   string
	Bridge string
}

func init() {
	supportsXlock = exec.Command("iptables", "--wait", "-L", "-n").Run() == nil
}

func NewChain(name, bridge string) (*Chain, error) {
	if output, err := Raw("-t", "nat", "-N", name); err != nil {
		return nil, err
	} else if len(output) != 0 {
		return nil, fmt.Errorf("Error creating new iptables chain: %s", output)
	}
	chain := &Chain{
		Name:   name,
		Bridge: bridge,
	}

	if err := chain.Prerouting(Add, "-m", "addrtype", "--dst-type", "LOCAL"); err != nil {
		return nil, fmt.Errorf("Failed to inject docker in PREROUTING chain: %s", err)
	}
	if err := chain.Output(Add, "-m", "addrtype", "--dst-type", "LOCAL", "!", "--dst", "127.0.0.0/8"); err != nil {
		return nil, fmt.Errorf("Failed to inject docker in OUTPUT chain: %s", err)
	}
	return chain, nil
}

func RemoveExistingChain(name string) error {
	chain := &Chain{
		Name: name,
	}
	return chain.Remove()
}

func (c *Chain) Forward(action Action, ip net.IP, port int, proto, dest_addr string, dest_port int) error {
	daddr := ip.String()
	if ip.IsUnspecified() {
		// iptables interprets "0.0.0.0" as "0.0.0.0/32", whereas we
		// want "0.0.0.0/0". "0/0" is correctly interpreted as "any
		// value" by both iptables and ip6tables.
		daddr = "0/0"
	}
	if output, err := Raw("-t", "nat", fmt.Sprint(action), c.Name,
		"-p", proto,
		"-d", daddr,
		"--dport", strconv.Itoa(port),
		"-j", "DNAT",
		"--to-destination", net.JoinHostPort(dest_addr, strconv.Itoa(dest_port))); err != nil {
		return err
	} else if len(output) != 0 {
		return fmt.Errorf("Error iptables forward: %s", output)
	}

	fAction := action
	if fAction == Add {
		fAction = "-I"
	}
	if output, err := Raw(string(fAction), "FORWARD",
		"!", "-i", c.Bridge,
		"-o", c.Bridge,
		"-p", proto,
		"-d", dest_addr,
		"--dport", strconv.Itoa(dest_port),
		"-j", "ACCEPT"); err != nil {
		return err
	} else if len(output) != 0 {
		return fmt.Errorf("Error iptables forward: %s", output)
	}

	if output, err := Raw("-t", "nat", string(fAction), "POSTROUTING",
		"-p", proto,
		"-s", dest_addr,
		"-d", dest_addr,
		"--dport", strconv.Itoa(dest_port),
		"-j", "MASQUERADE"); err != nil {
		return err
	} else if len(output) != 0 {
		return fmt.Errorf("Error iptables forward: %s", output)
	}

	return nil
}

func (c *Chain) Prerouting(action Action, args ...string) error {
	a := append(nat, fmt.Sprint(action), "PREROUTING")
	if len(args) > 0 {
		a = append(a, args...)
	}
	if output, err := Raw(append(a, "-j", c.Name)...); err != nil {
		return err
	} else if len(output) != 0 {
		return fmt.Errorf("Error iptables prerouting: %s", output)
	}
	return nil
}

func (c *Chain) Output(action Action, args ...string) error {
	a := append(nat, fmt.Sprint(action), "OUTPUT")
	if len(args) > 0 {
		a = append(a, args...)
	}
	if output, err := Raw(append(a, "-j", c.Name)...); err != nil {
		return err
	} else if len(output) != 0 {
		return fmt.Errorf("Error iptables output: %s", output)
	}
	return nil
}

func (c *Chain) Remove() error {
	// Ignore errors - This could mean the chains were never set up
	c.Prerouting(Delete, "-m", "addrtype", "--dst-type", "LOCAL")
	c.Output(Delete, "-m", "addrtype", "--dst-type", "LOCAL", "!", "--dst", "127.0.0.0/8")
	c.Output(Delete, "-m", "addrtype", "--dst-type", "LOCAL") // Created in versions <= 0.1.6

	c.Prerouting(Delete)
	c.Output(Delete)

	Raw("-t", "nat", "-F", c.Name)
	Raw("-t", "nat", "-X", c.Name)

	return nil
}

// Check if an existing rule exists
func Exists(args ...string) bool {
	if _, err := Raw(append([]string{"-C"}, args...)...); err != nil {
		return false
	}
	return true
}

func Raw(args ...string) ([]byte, error) {
	path, err := exec.LookPath("iptables")
	if err != nil {
		return nil, ErrIptablesNotFound
	}

	if supportsXlock {
		args = append([]string{"--wait"}, args...)
	}

	if os.Getenv("DEBUG") != "" {
		fmt.Fprintf(os.Stderr, fmt.Sprintf("[debug] %s, %v\n", path, args))
	}

	output, err := exec.Command(path, args...).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("iptables failed: iptables %v: %s (%s)", strings.Join(args, " "), output, err)
	}

	// ignore iptables' message about xtables lock
	if strings.Contains(string(output), "waiting for it to exit") {
		output = []byte("")
	}

	return output, err
}
<|endoftext|>


<commit_before>package tester

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/hashicorp/consul/api"
	"github.com/mitchellh/cli"
)

func UpgradeCommandFactory() (cli.Command, error) {
	return &Upgrade{}, nil
}

type Upgrade struct {
}

func (c *Upgrade) Help() string {
	helpText := `
Usage consul-live upgrade base version1 ... versionN

  Starts Consul using the base executable then shuts it down and upgrades in
  place using the supplied version executables. The base version is populated
  with some test data and that data is verified after each upgrade.
`
	return strings.TrimSpace(helpText)
}

func (c *Upgrade) Synopsis() string {
	return "Runs Consul through a given series of in-place upgrades"
}

func (c *Upgrade) Run(args []string) int {
	if len(args) < 2 {
		log.Println("At least two versions must be given")
		return 1
	}

	if err := c.upgrade(args); err != nil {
		log.Println(err)
		return 1
	}
	return 0
}

type ServerConfig struct {
	Server           bool   `json:"server,omitempty"`
	Bootstrap        bool   `json:"bootstrap,omitempty"`
	Bind             string `json:"bind_addr,omitempty"`
	DataDir          string `json:"data_dir,omitempty"`
	Datacenter       string `json:"datacenter,omitempty"`
	ACLMasterToken   string `json:"acl_master_token,omitempty"`
	ACLDatacenter    string `json:"acl_datacenter,omitempty"`
	ACLDefaultPolicy string `json:"acl_default_policy,omitempty"`
	LogLevel         string `json:"log_level,omitempty"`
}

func (c *Upgrade) upgrade(versions []string) error {
	base := versions[0]
	versions = versions[1:]

	var dir string
	var err error
	dir, err = ioutil.TempDir("", "consul")
	if err != nil {
		return err
	}
	defer os.RemoveAll(dir)

	config, err := ioutil.TempFile(dir, "config")
	if err != nil {
		return err
	}

	content, err := json.Marshal(ServerConfig{
		Server:           true,
		Bootstrap:        true,
		Bind:             "127.0.0.1",
		DataDir:          dir,
		Datacenter:       "dc1",
		ACLMasterToken:   "root",
		ACLDatacenter:    "dc1",
		ACLDefaultPolicy: "allow",
	})
	if err != nil {
		return err
	}
	if _, err := config.Write(content); err != nil {
		return err
	}
	if err := config.Close(); err != nil {
		return err
	}

	// Start the first version of Consul, which is our base.
	log.Printf("Starting base Consul from '%s'...\n", base)
	args := []string{
		"agent",
		"-config-file",
		config.Name(),
	}
	consul, err := NewConsul(base, args)
	if err != nil {
		return err
	}
	if err := consul.Start(); err != nil {
		return err
	}
	defer func() {
		if err := consul.Shutdown(); err != nil {
			log.Println(err)
		}
	}()

	// Wait for it to start up and elect itself.
	if err := consul.WaitForLeader(); err != nil {
		return err
	}

	// Populate it with some realistic data, enough to kick out a snapshot.
	log.Println("Populating with initial state store data...")
	client, err := api.NewClient(api.DefaultConfig())
	if err != nil {
		return err
	}
	fuzz, err := NewFuzz(client)
	if err != nil {
		return err
	}
	for {
		if err := fuzz.Populate(); err != nil {
			return err
		}

		entries, err := ioutil.ReadDir(dir + "/raft/snapshots/")
		if err != nil {
			return err
		}
		if len(entries) > 0 {
			break
		}
	}

	// Push some data in post-snapshot to make sure there's some stuff
	// in the Raft log as well.
	if err := fuzz.Populate(); err != nil {
		return err
	}
	if err := fuzz.Verify(); err != nil {
		return err
	}

	// Now shutdown the base version and try upgrading through the given
	// versions.
	if err := consul.Shutdown(); err != nil {
		return err
	}
	for _, version := range versions {
		// Start the upgraded version with the same data-dir.
		log.Printf("Upgrading to Consul from '%s'...\n", version)
		upgrade, err := NewConsul(version, args)
		if err != nil {
			return err
		}
		if err := upgrade.Start(); err != nil {
			return err
		}
		defer func() {
			if err := upgrade.Shutdown(); err != nil {
				log.Println(err)
			}
		}()

		// Wait for it to start up and elect itself.
		if err := upgrade.WaitForLeader(); err != nil {
			return err
		}

		// Make sure the data is still present.
		if err := fuzz.Verify(); err != nil {
			return err
		}

		// Add some new data for this version of Consul.
		if err := fuzz.Populate(); err != nil {
			return err
		}
		if err := fuzz.Verify(); err != nil {
			return err
		}

		// Shut it down in anticipation of the next upgrade.
		if err := upgrade.Shutdown(); err != nil {
			return err
		}
	}

	log.Println("Upgrade series complete")
	return nil
}
<commit_msg>Makes the upgrade test use go-getter so you can point at zip files or even releases URLs.<commit_after>package tester

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"

	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-getter"
	"github.com/mitchellh/cli"
)

func UpgradeCommandFactory() (cli.Command, error) {
	return &Upgrade{}, nil
}

type Upgrade struct {
}

func (c *Upgrade) Help() string {
	helpText := `
Usage consul-live upgrade base version1 ... versionN

  Starts Consul using the base executable then shuts it down and upgrades in
  place using the supplied version executables. The base version is populated
  with some test data and that data is verified after each upgrade.
`
	return strings.TrimSpace(helpText)
}

func (c *Upgrade) Synopsis() string {
	return "Runs Consul through a given series of in-place upgrades"
}

func (c *Upgrade) Run(args []string) int {
	if len(args) < 2 {
		log.Println("At least two versions must be given")
		return 1
	}

	if err := c.upgrade(args); err != nil {
		log.Println(err)
		return 1
	}
	return 0
}

type ServerConfig struct {
	Server           bool   `json:"server,omitempty"`
	Bootstrap        bool   `json:"bootstrap,omitempty"`
	Bind             string `json:"bind_addr,omitempty"`
	DataDir          string `json:"data_dir,omitempty"`
	Datacenter       string `json:"datacenter,omitempty"`
	ACLMasterToken   string `json:"acl_master_token,omitempty"`
	ACLDatacenter    string `json:"acl_datacenter,omitempty"`
	ACLDefaultPolicy string `json:"acl_default_policy,omitempty"`
	LogLevel         string `json:"log_level,omitempty"`
}

func (c *Upgrade) upgrade(versions []string) error {
	var dir string
	var err error
	dir, err = ioutil.TempDir("", "consul")
	if err != nil {
		return err
	}
	defer os.RemoveAll(dir)

	from := path.Join(dir, "consul")
	for i, version := range versions {
		if err := getter.GetAny(dir, version); err != nil {
			return err
		}
		to := path.Join(dir, fmt.Sprintf("version-%d", i))
		if err := os.Rename(from, to); err != nil {
			return err
		}
		versions[i] = to
	}

	base := versions[0]
	versions = versions[1:]

	config, err := ioutil.TempFile(dir, "config")
	if err != nil {
		return err
	}

	content, err := json.Marshal(ServerConfig{
		Server:           true,
		Bootstrap:        true,
		Bind:             "127.0.0.1",
		DataDir:          dir,
		Datacenter:       "dc1",
		ACLMasterToken:   "root",
		ACLDatacenter:    "dc1",
		ACLDefaultPolicy: "allow",
	})
	if err != nil {
		return err
	}
	if _, err := config.Write(content); err != nil {
		return err
	}
	if err := config.Close(); err != nil {
		return err
	}

	// Start the first version of Consul, which is our base.
	log.Printf("Starting base Consul from '%s'...\n", base)
	args := []string{
		"agent",
		"-config-file",
		config.Name(),
	}
	consul, err := NewConsul(base, args)
	if err != nil {
		return err
	}
	if err := consul.Start(); err != nil {
		return err
	}
	defer func() {
		if err := consul.Shutdown(); err != nil {
			log.Println(err)
		}
	}()

	// Wait for it to start up and elect itself.
	if err := consul.WaitForLeader(); err != nil {
		return err
	}

	// Populate it with some realistic data, enough to kick out a snapshot.
	log.Println("Populating with initial state store data...")
	client, err := api.NewClient(api.DefaultConfig())
	if err != nil {
		return err
	}
	fuzz, err := NewFuzz(client)
	if err != nil {
		return err
	}
	for {
		if err := fuzz.Populate(); err != nil {
			return err
		}

		entries, err := ioutil.ReadDir(dir + "/raft/snapshots/")
		if err != nil {
			return err
		}
		if len(entries) > 0 {
			break
		}
	}

	// Push some data in post-snapshot to make sure there's some stuff
	// in the Raft log as well.
	if err := fuzz.Populate(); err != nil {
		return err
	}
	if err := fuzz.Verify(); err != nil {
		return err
	}

	// Now shutdown the base version and try upgrading through the given
	// versions.
	if err := consul.Shutdown(); err != nil {
		return err
	}
	for _, version := range versions {
		// Start the upgraded version with the same data-dir.
		log.Printf("Upgrading to Consul from '%s'...\n", version)
		upgrade, err := NewConsul(version, args)
		if err != nil {
			return err
		}
		if err := upgrade.Start(); err != nil {
			return err
		}
		defer func() {
			if err := upgrade.Shutdown(); err != nil {
				log.Println(err)
			}
		}()

		// Wait for it to start up and elect itself.
		if err := upgrade.WaitForLeader(); err != nil {
			return err
		}

		// Make sure the data is still present.
		if err := fuzz.Verify(); err != nil {
			return err
		}

		// Add some new data for this version of Consul.
		if err := fuzz.Populate(); err != nil {
			return err
		}
		if err := fuzz.Verify(); err != nil {
			return err
		}

		// Shut it down in anticipation of the next upgrade.
		if err := upgrade.Shutdown(); err != nil {
			return err
		}
	}

	log.Println("Upgrade series complete")
	return nil
}
<|endoftext|>


<commit_before>package easy_middlware

import (
	"net/http"
)

type validator interface {
	ok(w http.ResponseWriter, r http.Request) (bool, error)
}

// Logging of device request time
func isValid(v validator) Middleware {
	return func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request){

			if ok, err := v.ok(w, r); !ok {
				http.Error(w, err.Error(), http.StatusBadRequest)
				return
			}

			h.ServeHTTP(w, r)
		})
	}
}

<commit_msg>- Removed invalid commit<commit_after>package easy_middlware

import (
	"net/http"
)

type validator interface {
	ok(w http.ResponseWriter, r http.Request) (bool, error)
}

func isValid(v validator) Middleware {
	return func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request){

			if ok, err := v.ok(w, r); !ok {
				http.Error(w, err.Error(), http.StatusBadRequest)
				return
			}

			h.ServeHTTP(w, r)
		})
	}
}

<|endoftext|>


<commit_before>// progress-dump -- Dumps progress.yml to a reStructuredText Grid Table
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

// progress-dump is used to output a grid table using progress.yml for inclusion
// into README.rst It gives prospective users an idea of how specification
// complete go-rst is compared to the reference docutils parser.
package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"sort"
	"strings"

	"github.com/aybabtme/rgbterm"
	"github.com/davecgh/go-spew/spew"
	"github.com/demizer/go-elog"
	"github.com/docopt/docopt.go"
	"gopkg.in/yaml.v2"
)

var APP_NAME = rgbterm.String("progress-dump", 255, 255, 135)
var APP_DESC = rgbterm.String("Dumps progress output", 0, 215, 95)
var APP_USAGE = APP_NAME + " - " + APP_DESC + `

Usage:
  progress-dump [--progress-yml <PATH>] [-h | --help]

Options:
  -h --help                        Show the help message.
  --progress-yml <PATH>  The path to progress.yml [default: ../../progress.yml]
  --readme <PATH>        The path to README.rst
`

type Item struct {
	Item      string
	Done      string
	Note      string
	Completed string
	SubItems  []Item `yaml:"sub-items"`
}

var spd = spew.ConfigState{Indent: "\t", DisableMethods: true}

type Table struct {
	Sections     []*TableSection
	TotalItems   int
	TotalDone    int
	OverAllPerc  float64
	MaxCol1Chars int
	MaxCol2Chars int
	MaxCol3Chars int
}

func (t *Table) Len() int {
	return len(t.Sections)
}

func (t *Table) Swap(i, j int) {
	t.Sections[i], t.Sections[j] = t.Sections[j], t.Sections[i]
}

func (t *Table) Less(i, j int) bool {
	return t.Sections[i].Id < t.Sections[j].Id
}

func (t *Table) Dump() {
	// The table sections are in reverse order due to the recursion.
	sort.Sort(t)

	// The addition numbers here compensate for the ascii chars that make
	// up the frame of the ascii table.
	// 1 == space
	// 3 == 2x space and "+"
	tWidth := 1 + t.MaxCol1Chars + 3 + t.MaxCol2Chars + 3 + t.MaxCol3Chars + 1
	tTop := strings.Repeat("-", tWidth)
	topWithEndPoints := fmt.Sprintf("+" + tTop + "+")

	fakeHdr := fmt.Sprintf("| %s | %s | %s |", "**Done**",
		// 8 == (4x asterisks, 2 spaces, 2 frame ascii)
		"**Item**"+strings.Repeat(" ", t.MaxCol2Chars-8),
		"**Note**"+strings.Repeat(" ", t.MaxCol3Chars-8))

	sepWithPoints := fmt.Sprintf("+-%s-+-%s-+-%s-+", "--------",
		strings.Repeat("-", t.MaxCol2Chars),
		strings.Repeat("-", t.MaxCol3Chars))

	fmt.Println(topWithEndPoints)

	for x, y := range t.Sections {
		secTitle := fmt.Sprintf("**%0.0f%% Complete -- %s**",
			y.Header.DonePerc, y.Header.Name)
		fmt.Printf("| %s |\n",
			secTitle+strings.Repeat(" ", tWidth-len(secTitle)-2))
		if x == 0 {
			fmt.Println(sepWithPoints)
			fmt.Println(fakeHdr)
			fmt.Println(sepWithPoints)
		} else {
			fmt.Println(sepWithPoints)
		}
		for _, z := range y.Rows {
			fmt.Printf("| %s | %s | %s |\n",
				z.Done+strings.Repeat(" ",
					t.MaxCol1Chars-len(z.Done)),
				z.Item+strings.Repeat(" ",
					t.MaxCol2Chars-len(z.Item)),
				z.Note+strings.Repeat(" ",
					t.MaxCol3Chars-len(z.Note)))
			fmt.Println(sepWithPoints)
		}
	}
}

type TableSection struct {
	Header *TableSectionHeader
	Rows   []*TableRow
	Id     int
}

type TableSectionHeader struct {
	Name     string
	Done     string
	DonePerc float64
}

type TableRow struct {
	Done string
	Item string
	Note string
}

type State struct {
	Table *Table
	Id    int
	Items []Item
}

func NewState() *State {
	return &State{
		Table: &Table{
			Sections:     make([]*TableSection, 0),
			MaxCol1Chars: 8, // Col1 will always be **Done**
		},
		Id: 0,
	}
}

func (s *State) ReadProgressFile(path string) {
	f, err := ioutil.ReadFile(path)
	if err != nil {
		log.Criticalln(err)
		os.Exit(1)
	}
	err = yaml.Unmarshal(f, &s.Items)
	if err != nil {
		log.Criticalln(err)
		os.Exit(1)
	}
}

func (s *State) DumpTable() {
	s.Walk(s.Items, nil, 0)
	s.CalcPercentages()
	s.Table.Dump()
}

func (s *State) Walk(z []Item, sec *TableSection, depth int) {
	for _, x := range z {
		if depth == 0 {
			s.Id++
			sec = &TableSection{
				Header: &TableSectionHeader{Name: x.Item,
					Done: x.Done},
				Id: s.Id,
			}
		}
		if x.SubItems != nil {
			name := x.Item
			if depth > 0 {
				s.Id++
				name = sec.Header.Name + " :: " + x.Item
			}
			subSec := &TableSection{
				Header: &TableSectionHeader{Name: name,
					Done: x.Done},
				Id: s.Id,
			}
			depth++
			s.Walk(x.SubItems, subSec, depth)
			depth--
			s.Table.Sections = append(s.Table.Sections, subSec)
			continue
		}
		if len(x.Item) > s.Table.MaxCol2Chars {
			s.Table.MaxCol2Chars = len(x.Item)
		}
		if len(x.Note) > s.Table.MaxCol3Chars {
			s.Table.MaxCol3Chars = len(x.Note)
		}
		nRow := &TableRow{x.Done, x.Item, x.Note}
		sec.Rows = append(sec.Rows, nRow)
		if depth == 0 {
			s.Table.Sections = append(s.Table.Sections, sec)
		}
	}
}

func (s *State) CalcPercentages() {
	for _, x := range s.Table.Sections {
		sDone := 0.0
		s.Table.TotalItems++
		if x.Header.Done == "yes" {
			s.Table.TotalDone++
			sDone++
		}
		for _, y := range x.Rows {
			s.Table.TotalItems++
			if y.Done == "yes" {
				s.Table.TotalDone++
				sDone++
			}
		}
		if sDone != 0 {
			x.Header.DonePerc = (sDone / float64(len(x.Rows)+1)) * 100
		}
	}
	s.Table.OverAllPerc = (float64(s.Table.TotalDone) /
		float64(s.Table.TotalItems)) * 100
}

func main() {
	log.SetFlags(0)
	log.SetLevel(log.LEVEL_DEBUG)

	args, err := docopt.Parse(APP_USAGE, nil, true, "progress-dump", false)
	if err != nil {
		log.Criticalln(err)
		os.Exit(1)
	}

	s := NewState()
	s.ReadProgressFile(args["--progress-yml"].(string))
	s.DumpTable()
}
<commit_msg>progress-dump: Change docopt import path<commit_after>// progress-dump -- Dumps progress.yml to a reStructuredText Grid Table
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

// progress-dump is used to output a grid table using progress.yml for inclusion
// into README.rst It gives prospective users an idea of how specification
// complete go-rst is compared to the reference docutils parser.
package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"sort"
	"strings"

	"github.com/aybabtme/rgbterm"
	"github.com/davecgh/go-spew/spew"
	"github.com/demizer/go-elog"
	"github.com/docopt/docopt-go"
	"gopkg.in/yaml.v2"
)

var APP_NAME = rgbterm.String("progress-dump", 255, 255, 135)
var APP_DESC = rgbterm.String("Dumps progress output", 0, 215, 95)
var APP_USAGE = APP_NAME + " - " + APP_DESC + `

Usage:
  progress-dump [--progress-yml <PATH>] [-h | --help]

Options:
  -h --help                        Show the help message.
  --progress-yml <PATH>  The path to progress.yml [default: ../../progress.yml]
  --readme <PATH>        The path to README.rst
`

type Item struct {
	Item      string
	Done      string
	Note      string
	Completed string
	SubItems  []Item `yaml:"sub-items"`
}

var spd = spew.ConfigState{Indent: "\t", DisableMethods: true}

type Table struct {
	Sections     []*TableSection
	TotalItems   int
	TotalDone    int
	OverAllPerc  float64
	MaxCol1Chars int
	MaxCol2Chars int
	MaxCol3Chars int
}

func (t *Table) Len() int {
	return len(t.Sections)
}

func (t *Table) Swap(i, j int) {
	t.Sections[i], t.Sections[j] = t.Sections[j], t.Sections[i]
}

func (t *Table) Less(i, j int) bool {
	return t.Sections[i].Id < t.Sections[j].Id
}

func (t *Table) Dump() {
	// The table sections are in reverse order due to the recursion.
	sort.Sort(t)

	// The addition numbers here compensate for the ascii chars that make
	// up the frame of the ascii table.
	// 1 == space
	// 3 == 2x space and "+"
	tWidth := 1 + t.MaxCol1Chars + 3 + t.MaxCol2Chars + 3 + t.MaxCol3Chars + 1
	tTop := strings.Repeat("-", tWidth)
	topWithEndPoints := fmt.Sprintf("+" + tTop + "+")

	fakeHdr := fmt.Sprintf("| %s | %s | %s |", "**Done**",
		// 8 == (4x asterisks, 2 spaces, 2 frame ascii)
		"**Item**"+strings.Repeat(" ", t.MaxCol2Chars-8),
		"**Note**"+strings.Repeat(" ", t.MaxCol3Chars-8))

	sepWithPoints := fmt.Sprintf("+-%s-+-%s-+-%s-+", "--------",
		strings.Repeat("-", t.MaxCol2Chars),
		strings.Repeat("-", t.MaxCol3Chars))

	fmt.Println(topWithEndPoints)

	for x, y := range t.Sections {
		secTitle := fmt.Sprintf("**%0.0f%% Complete -- %s**",
			y.Header.DonePerc, y.Header.Name)
		fmt.Printf("| %s |\n",
			secTitle+strings.Repeat(" ", tWidth-len(secTitle)-2))
		if x == 0 {
			fmt.Println(sepWithPoints)
			fmt.Println(fakeHdr)
			fmt.Println(sepWithPoints)
		} else {
			fmt.Println(sepWithPoints)
		}
		for _, z := range y.Rows {
			fmt.Printf("| %s | %s | %s |\n",
				z.Done+strings.Repeat(" ",
					t.MaxCol1Chars-len(z.Done)),
				z.Item+strings.Repeat(" ",
					t.MaxCol2Chars-len(z.Item)),
				z.Note+strings.Repeat(" ",
					t.MaxCol3Chars-len(z.Note)))
			fmt.Println(sepWithPoints)
		}
	}
}

type TableSection struct {
	Header *TableSectionHeader
	Rows   []*TableRow
	Id     int
}

type TableSectionHeader struct {
	Name     string
	Done     string
	DonePerc float64
}

type TableRow struct {
	Done string
	Item string
	Note string
}

type State struct {
	Table *Table
	Id    int
	Items []Item
}

func NewState() *State {
	return &State{
		Table: &Table{
			Sections:     make([]*TableSection, 0),
			MaxCol1Chars: 8, // Col1 will always be **Done**
		},
		Id: 0,
	}
}

func (s *State) ReadProgressFile(path string) {
	f, err := ioutil.ReadFile(path)
	if err != nil {
		log.Criticalln(err)
		os.Exit(1)
	}
	err = yaml.Unmarshal(f, &s.Items)
	if err != nil {
		log.Criticalln(err)
		os.Exit(1)
	}
}

func (s *State) DumpTable() {
	s.Walk(s.Items, nil, 0)
	s.CalcPercentages()
	s.Table.Dump()
}

func (s *State) Walk(z []Item, sec *TableSection, depth int) {
	for _, x := range z {
		if depth == 0 {
			s.Id++
			sec = &TableSection{
				Header: &TableSectionHeader{Name: x.Item,
					Done: x.Done},
				Id: s.Id,
			}
		}
		if x.SubItems != nil {
			name := x.Item
			if depth > 0 {
				s.Id++
				name = sec.Header.Name + " :: " + x.Item
			}
			subSec := &TableSection{
				Header: &TableSectionHeader{Name: name,
					Done: x.Done},
				Id: s.Id,
			}
			depth++
			s.Walk(x.SubItems, subSec, depth)
			depth--
			s.Table.Sections = append(s.Table.Sections, subSec)
			continue
		}
		if len(x.Item) > s.Table.MaxCol2Chars {
			s.Table.MaxCol2Chars = len(x.Item)
		}
		if len(x.Note) > s.Table.MaxCol3Chars {
			s.Table.MaxCol3Chars = len(x.Note)
		}
		nRow := &TableRow{x.Done, x.Item, x.Note}
		sec.Rows = append(sec.Rows, nRow)
		if depth == 0 {
			s.Table.Sections = append(s.Table.Sections, sec)
		}
	}
}

func (s *State) CalcPercentages() {
	for _, x := range s.Table.Sections {
		sDone := 0.0
		s.Table.TotalItems++
		if x.Header.Done == "yes" {
			s.Table.TotalDone++
			sDone++
		}
		for _, y := range x.Rows {
			s.Table.TotalItems++
			if y.Done == "yes" {
				s.Table.TotalDone++
				sDone++
			}
		}
		if sDone != 0 {
			x.Header.DonePerc = (sDone / float64(len(x.Rows)+1)) * 100
		}
	}
	s.Table.OverAllPerc = (float64(s.Table.TotalDone) /
		float64(s.Table.TotalItems)) * 100
}

func main() {
	log.SetFlags(0)
	log.SetLevel(log.LEVEL_DEBUG)

	args, err := docopt.Parse(APP_USAGE, nil, true, "progress-dump", false)
	if err != nil {
		log.Criticalln(err)
		os.Exit(1)
	}

	s := NewState()
	s.ReadProgressFile(args["--progress-yml"].(string))
	s.DumpTable()
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package cmd

import (
	"github.com/TheThingsNetwork/ttn/ttnctl/util"
	"github.com/spf13/cobra"
)

var gatewaysRegisterCmd = &cobra.Command{
	Use:   "register [GatewayID] [FrequencyPlan]",
	Short: "Register a gateway",
	Long:  `ttnctl gateways register can be used to register a gateway`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) != 2 {
			cmd.UsageFunc()(cmd)
			return
		}

		gatewayID := args[0]
		if !validID(gatewayID) {
			ctx.Fatal("Invalid Gateway ID")
		}

		frequencyPlan := args[1]

		act := util.GetAccount(ctx)
		gateway, err := act.RegisterGateway(gatewayID, frequencyPlan, nil)
		if err != nil {
			ctx.WithError(err).Fatal("Could not register gateway")
		}

		util.ForceRefreshToken(ctx)

		ctx.WithField("Gateway ID", gateway.ID).Info("Registered Gateway")
	},
}

func init() {
	gatewaysCmd.AddCommand(gatewaysRegisterCmd)
}
<commit_msg>fixup! Add ttnctl gateway register cmd<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package cmd

import (
	"github.com/TheThingsNetwork/ttn/api"
	"github.com/TheThingsNetwork/ttn/ttnctl/util"
	"github.com/spf13/cobra"
)

var gatewaysRegisterCmd = &cobra.Command{
	Use:   "register [GatewayID] [FrequencyPlan]",
	Short: "Register a gateway",
	Long:  `ttnctl gateways register can be used to register a gateway`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) != 2 {
			cmd.UsageFunc()(cmd)
			return
		}

		gatewayID := args[0]
		if !api.ValidID(gatewayID) {
			ctx.Fatal("Invalid Gateway ID")
		}

		frequencyPlan := args[1]

		act := util.GetAccount(ctx)
		gateway, err := act.RegisterGateway(gatewayID, frequencyPlan, nil)
		if err != nil {
			ctx.WithError(err).Fatal("Could not register gateway")
		}

		util.ForceRefreshToken(ctx)

		ctx.WithField("Gateway ID", gateway.ID).Info("Registered Gateway")
	},
}

func init() {
	gatewaysCmd.AddCommand(gatewaysRegisterCmd)
}
<|endoftext|>


<commit_before>package nbfx

import (
	"encoding/xml"
	"errors"
	"fmt"
)

func init() {
	records[shortElement] = &shortElementRecord{elementRecordBase{recordBase{"ShortElement", shortElement}}}
	records[element] = &elementRecord{elementRecordBase{recordBase{"Element", element}}}
	records[shortDictionaryElement] = &shortDictionaryElementRecord{elementRecordBase{recordBase{"ShortDictionaryElement", shortDictionaryElement}}}
	records[dictionaryElement] = &dictionaryElementRecord{elementRecordBase{recordBase{"DictionaryElement", dictionaryElement}}}
	addAzRecords(prefixDictionaryElementA, "PrefixDictionaryElement", func(id byte, name string) record {
		return &prefixDictionaryElementAZRecord{elementRecordBase{recordBase{name, id}}}
	})
	addAzRecords(prefixElementA, "PrefixElement", func(id byte, name string) record {
		return &prefixElementAZRecord{elementRecordBase{recordBase{name, id}}}
	})
}

type elementRecordBase struct {
	recordBase
}

func (e *elementRecordBase) isStartElement() bool { return true }

func (r *elementRecordBase) readElementAttributes(element xml.StartElement, d *decoder) (record, error) {
	var peekRecord record
	var attributeToken xml.Attr

	// get next record
	rec, err := getNextRecord(d)
	for err == nil && rec != nil {
		if err != nil {
			return nil, err
		}

		var attrReader attributeRecordDecoder
		if rec.isAttribute() {
			attrReader = rec.(attributeRecordDecoder)

			attributeToken, err = attrReader.decodeAttribute(d)
			if err != nil {
				return nil, err
			}
			element.Attr = append(element.Attr, attributeToken)

			rec, err = getNextRecord(d)
			if err != nil {
				return nil, err
			}
		} else {
			attrReader = nil
			peekRecord = rec
			rec = nil
		}
	}

	err = d.xml.EncodeToken(element)
	if err != nil {
		return nil, err
	}

	d.elementStack.Push(element)

	return peekRecord, nil
}

func (r *elementRecordBase) encodeAttributes(e *encoder, attrs []xml.Attr) error {
	var err error
	for _, attr := range attrs {
		err = r.encodeAttribute(e, attr)
		if err != nil {
			break
		}
	}
	return err
}

func (r *elementRecordBase) encodeAttribute(e *encoder, attr xml.Attr) error {
	rec, err := e.getAttributeRecordFromToken(attr)
	if err != nil {
		return err
	}
	attrRec := rec.(attributeRecordEncoder)
	return attrRec.encodeAttribute(e, attr)
}

func (r *elementRecordBase) encodeElement(e *encoder, element xml.StartElement) error {
	return errors.New(fmt.Sprint("NotImplemented: encodeElement on", r))
}

//(0x44-0x5D)
type prefixDictionaryElementAZRecord struct {
	elementRecordBase
}

func (r *prefixDictionaryElementAZRecord) decodeElement(d *decoder) (record, error) {
	name, err := readDictionaryString(d)
	if err != nil {
		return nil, err
	}
	element := xml.StartElement{Name: xml.Name{Local: string('a'+r.id-prefixDictionaryElementA) + ":" + name}}

	return r.readElementAttributes(element, d)
}

func (r *prefixDictionaryElementAZRecord) encodeElement(e *encoder, element xml.StartElement) error {
	_, err := e.bin.Write([]byte{r.id})
	if err != nil {
		return err
	}
	_, err = writeMultiByteInt31(e, e.dict[element.Name.Local])
	if err != nil {
		return err
	}
	return r.encodeAttributes(e, element.Attr)
}

//(0x42)
type shortDictionaryElementRecord struct {
	elementRecordBase
}

func (r *shortDictionaryElementRecord) getName() string {
	return "ShortDictionaryElement (0x42)"
}

func (r *shortDictionaryElementRecord) decodeElement(d *decoder) (record, error) {
	name, err := readDictionaryString(d)
	if err != nil {
		return nil, err
	}
	element := xml.StartElement{Name: xml.Name{Local: name}}

	return r.readElementAttributes(element, d)
}

func (r *shortDictionaryElementRecord) encodeElement(e *encoder, element xml.StartElement) error {
	_, err := e.bin.Write([]byte{r.id})
	if err != nil {
		return err
	}
	err = writeDictionaryBytes(e, element.Name.Local)
	if err != nil {
		return err
	}
	return r.encodeAttributes(e, element.Attr)
}

//(0x43)
type dictionaryElementRecord struct {
	elementRecordBase
}

func (r *dictionaryElementRecord) getName() string {
	return "DictionaryElement (0x43)"
}

func (r *dictionaryElementRecord) decodeElement(d *decoder) (record, error) {
	prefix, err := readString(d.bin)
	if err != nil {
		return nil, err
	}
	name, err := readDictionaryString(d)
	if err != nil {
		return nil, err
	}
	element := xml.StartElement{Name: xml.Name{Local: prefix + ":" + name}}

	return r.readElementAttributes(element, d)
}

// 0x40
type shortElementRecord struct {
	elementRecordBase
}

func (r *shortElementRecord) getName() string {
	return "shortElementRecord (0x40)"
}

func (r *shortElementRecord) decodeElement(d *decoder) (record, error) {
	name, err := readString(d.bin)
	if err != nil {
		return nil, err
	}
	element := xml.StartElement{Name: xml.Name{Local: name}}

	return r.readElementAttributes(element, d)
}

func (r *shortElementRecord) encodeElement(e *encoder, element xml.StartElement) error {
	_, err := e.bin.Write([]byte{shortElement})
	if err != nil {
		return err
	}
	_, err = writeString(e, element.Name.Local)
	if err != nil {
		return err
	}
	return r.encodeAttributes(e, element.Attr)
}

// 0x5E-0x77
type prefixElementAZRecord struct {
	elementRecordBase
}

func (r *prefixElementAZRecord) getName() string {
	return fmt.Sprint(r)
}

func (r *prefixElementAZRecord) decodeElement(d *decoder) (record, error) {
	name, err := readString(d.bin)
	if err != nil {
		return nil, err
	}
	element := xml.StartElement{Name: xml.Name{Local: string('a'+(r.id-prefixElementA)) + ":" + name}}

	return r.readElementAttributes(element, d)
}

func (r *prefixElementAZRecord) encodeElement(e *encoder, element xml.StartElement) error {
	_, err := e.bin.Write([]byte{prefixElementA + (r.id - prefixElementA)})
	if err != nil {
		return err
	}
	_, err = writeString(e, element.Name.Local)
	return err
}

// 0x41
type elementRecord struct {
	elementRecordBase
}

func (r *elementRecord) getName() string {
	return "elementRecord (0x41)"
}

func (r *elementRecord) decodeElement(d *decoder) (record, error) {
	prefix, err := readString(d.bin)
	if err != nil {
		return nil, err
	}
	name, err := readString(d.bin)
	if err != nil {
		return nil, err
	}
	element := xml.StartElement{Name: xml.Name{Local: prefix + ":" + name}}

	return r.readElementAttributes(element, d)
}
<commit_msg>Implement elementRecord.encodeElement<commit_after>package nbfx

import (
	"encoding/xml"
	"errors"
	"fmt"
)

func init() {
	records[shortElement] = &shortElementRecord{elementRecordBase{recordBase{"ShortElement", shortElement}}}
	records[element] = &elementRecord{elementRecordBase{recordBase{"Element", element}}}
	records[shortDictionaryElement] = &shortDictionaryElementRecord{elementRecordBase{recordBase{"ShortDictionaryElement", shortDictionaryElement}}}
	records[dictionaryElement] = &dictionaryElementRecord{elementRecordBase{recordBase{"DictionaryElement", dictionaryElement}}}
	addAzRecords(prefixDictionaryElementA, "PrefixDictionaryElement", func(id byte, name string) record {
		return &prefixDictionaryElementAZRecord{elementRecordBase{recordBase{name, id}}}
	})
	addAzRecords(prefixElementA, "PrefixElement", func(id byte, name string) record {
		return &prefixElementAZRecord{elementRecordBase{recordBase{name, id}}}
	})
}

type elementRecordBase struct {
	recordBase
}

func (e *elementRecordBase) isStartElement() bool { return true }

func (r *elementRecordBase) readElementAttributes(element xml.StartElement, d *decoder) (record, error) {
	var peekRecord record
	var attributeToken xml.Attr

	// get next record
	rec, err := getNextRecord(d)
	for err == nil && rec != nil {
		if err != nil {
			return nil, err
		}

		var attrReader attributeRecordDecoder
		if rec.isAttribute() {
			attrReader = rec.(attributeRecordDecoder)

			attributeToken, err = attrReader.decodeAttribute(d)
			if err != nil {
				return nil, err
			}
			element.Attr = append(element.Attr, attributeToken)

			rec, err = getNextRecord(d)
			if err != nil {
				return nil, err
			}
		} else {
			attrReader = nil
			peekRecord = rec
			rec = nil
		}
	}

	err = d.xml.EncodeToken(element)
	if err != nil {
		return nil, err
	}

	d.elementStack.Push(element)

	return peekRecord, nil
}

func (r *elementRecordBase) encodeAttributes(e *encoder, attrs []xml.Attr) error {
	var err error
	for _, attr := range attrs {
		err = r.encodeAttribute(e, attr)
		if err != nil {
			break
		}
	}
	return err
}

func (r *elementRecordBase) encodeAttribute(e *encoder, attr xml.Attr) error {
	rec, err := e.getAttributeRecordFromToken(attr)
	if err != nil {
		return err
	}
	attrRec := rec.(attributeRecordEncoder)
	return attrRec.encodeAttribute(e, attr)
}

func (r *elementRecordBase) encodeElement(e *encoder, element xml.StartElement) error {
	return errors.New(fmt.Sprint("NotImplemented: encodeElement on", r))
}

//(0x44-0x5D)
type prefixDictionaryElementAZRecord struct {
	elementRecordBase
}

func (r *prefixDictionaryElementAZRecord) decodeElement(d *decoder) (record, error) {
	name, err := readDictionaryString(d)
	if err != nil {
		return nil, err
	}
	element := xml.StartElement{Name: xml.Name{Local: string('a'+r.id-prefixDictionaryElementA) + ":" + name}}

	return r.readElementAttributes(element, d)
}

func (r *prefixDictionaryElementAZRecord) encodeElement(e *encoder, element xml.StartElement) error {
	_, err := e.bin.Write([]byte{r.id})
	if err != nil {
		return err
	}
	_, err = writeMultiByteInt31(e, e.dict[element.Name.Local])
	if err != nil {
		return err
	}
	return r.encodeAttributes(e, element.Attr)
}

//(0x42)
type shortDictionaryElementRecord struct {
	elementRecordBase
}

func (r *shortDictionaryElementRecord) getName() string {
	return "ShortDictionaryElement (0x42)"
}

func (r *shortDictionaryElementRecord) decodeElement(d *decoder) (record, error) {
	name, err := readDictionaryString(d)
	if err != nil {
		return nil, err
	}
	element := xml.StartElement{Name: xml.Name{Local: name}}

	return r.readElementAttributes(element, d)
}

func (r *shortDictionaryElementRecord) encodeElement(e *encoder, element xml.StartElement) error {
	_, err := e.bin.Write([]byte{r.id})
	if err != nil {
		return err
	}
	err = writeDictionaryBytes(e, element.Name.Local)
	if err != nil {
		return err
	}
	return r.encodeAttributes(e, element.Attr)
}

//(0x43)
type dictionaryElementRecord struct {
	elementRecordBase
}

func (r *dictionaryElementRecord) getName() string {
	return "DictionaryElement (0x43)"
}

func (r *dictionaryElementRecord) decodeElement(d *decoder) (record, error) {
	prefix, err := readString(d.bin)
	if err != nil {
		return nil, err
	}
	name, err := readDictionaryString(d)
	if err != nil {
		return nil, err
	}
	element := xml.StartElement{Name: xml.Name{Local: prefix + ":" + name}}

	return r.readElementAttributes(element, d)
}

// 0x40
type shortElementRecord struct {
	elementRecordBase
}

func (r *shortElementRecord) getName() string {
	return "shortElementRecord (0x40)"
}

func (r *shortElementRecord) decodeElement(d *decoder) (record, error) {
	name, err := readString(d.bin)
	if err != nil {
		return nil, err
	}
	element := xml.StartElement{Name: xml.Name{Local: name}}

	return r.readElementAttributes(element, d)
}

func (r *shortElementRecord) encodeElement(e *encoder, element xml.StartElement) error {
	_, err := e.bin.Write([]byte{shortElement})
	if err != nil {
		return err
	}
	_, err = writeString(e, element.Name.Local)
	if err != nil {
		return err
	}
	return r.encodeAttributes(e, element.Attr)
}

// 0x5E-0x77
type prefixElementAZRecord struct {
	elementRecordBase
}

func (r *prefixElementAZRecord) getName() string {
	return fmt.Sprint(r)
}

func (r *prefixElementAZRecord) decodeElement(d *decoder) (record, error) {
	name, err := readString(d.bin)
	if err != nil {
		return nil, err
	}
	element := xml.StartElement{Name: xml.Name{Local: string('a'+(r.id-prefixElementA)) + ":" + name}}

	return r.readElementAttributes(element, d)
}

func (r *prefixElementAZRecord) encodeElement(e *encoder, element xml.StartElement) error {
	_, err := e.bin.Write([]byte{prefixElementA + (r.id - prefixElementA)})
	if err != nil {
		return err
	}
	_, err = writeString(e, element.Name.Local)
	return err
}

// 0x41
type elementRecord struct {
	elementRecordBase
}

func (r *elementRecord) getName() string {
	return "elementRecord (0x41)"
}

func (r *elementRecord) decodeElement(d *decoder) (record, error) {
	prefix, err := readString(d.bin)
	if err != nil {
		return nil, err
	}
	name, err := readString(d.bin)
	if err != nil {
		return nil, err
	}
	element := xml.StartElement{Name: xml.Name{Local: prefix + ":" + name}}

	return r.readElementAttributes(element, d)
}

func (r *elementRecord) encodeElement(e *encoder, element xml.StartElement) error {
	err := e.bin.WriteByte(r.id)
	if err != nil {
		return err
	}
	_, err = writeString(e, element.Name.Space)
	if err != nil {
		return err
	}
	_, err = writeString(e, element.Name.Local)
	if err != nil {
		return err
	}
	return r.encodeAttributes(e, element.Attr)
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Copyright 2015 Wenbin Xiao
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package kv

import (
	"github.com/pingcap/tidb/kv/memkv"
	"github.com/pingcap/tidb/util/types"
)

type btreeBuffer struct {
	tree *memkv.Tree
}

// NewBTreeBuffer returns a breeBuffer.
func NewBTreeBuffer() MemBuffer {
	return &btreeBuffer{
		tree: memkv.NewTree(types.Collators[true]),
	}
}

// Get returns the value associated with the key; ErrNotExist error if the key does not exist.
func (b *btreeBuffer) Get(k Key) ([]byte, error) {
	v, ok := b.tree.Get(toIfaces(k))
	if !ok {
		return nil, ErrNotExist
	}
	return fromIfaces(v), nil
}

// Set associates the key with the value.
func (b *btreeBuffer) Set(k []byte, v []byte) error {
	b.tree.Set(toIfaces(k), toIfaces(v))
	return nil
}

// Release clear the whole buffer.
func (b *btreeBuffer) Release() {
	b.tree.Clear()
}

type btreeIter struct {
	e  *memkv.Enumerator
	k  string
	v  []byte
	ok bool
}

// NewIterator creates a new Iterator based on the provided param
func (b *btreeBuffer) NewIterator(param interface{}) Iterator {
	var e *memkv.Enumerator
	var err error
	if param == nil {
		e, err = b.tree.SeekFirst()
		if err != nil {
			return &btreeIter{ok: false}
		}
	} else {
		key := toIfaces(param.([]byte))
		e, _ = b.tree.Seek(key)
	}
	iter := &btreeIter{e: e}
	// the initial push...
	iter.Next()
	return iter
}

// Close implements Iterator Close.
func (i *btreeIter) Close() {
	//noop
}

// Key implements Iterator Key.
func (i *btreeIter) Key() string {
	return i.k
}

// Value implements Iterator Value.
func (i *btreeIter) Value() []byte {
	return i.v
}

// Next implements Iterator Next.
func (i *btreeIter) Next() error {
	k, v, err := i.e.Next()
	i.k, i.v, i.ok = string(fromIfaces(k)), fromIfaces(v), err == nil
	return err
}

// Valid implements Iterator Valid.
func (i *btreeIter) Valid() bool {
	return i.ok
}

func toIfaces(v []byte) []interface{} {
	return []interface{}{v}
}

func fromIfaces(v []interface{}) []byte {
	if v == nil {
		return nil
	}
	return v[0].([]byte)
}
<commit_msg>*: address review comments<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Copyright 2015 Wenbin Xiao
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package kv

import (
	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/tidb/kv/memkv"
	"github.com/pingcap/tidb/util/types"
)

type btreeBuffer struct {
	tree *memkv.Tree
}

// NewBTreeBuffer returns a breeBuffer.
func NewBTreeBuffer() MemBuffer {
	return &btreeBuffer{
		tree: memkv.NewTree(types.Collators[true]),
	}
}

// Get returns the value associated with the key; ErrNotExist error if the key does not exist.
func (b *btreeBuffer) Get(k Key) ([]byte, error) {
	v, ok := b.tree.Get(toIfaces(k))
	if !ok {
		return nil, ErrNotExist
	}
	return fromIfaces(v), nil
}

// Set associates the key with the value.
func (b *btreeBuffer) Set(k []byte, v []byte) error {
	b.tree.Set(toIfaces(k), toIfaces(v))
	return nil
}

// Release clear the whole buffer.
func (b *btreeBuffer) Release() {
	b.tree.Clear()
}

type btreeIter struct {
	e  *memkv.Enumerator
	k  string
	v  []byte
	ok bool
}

// NewIterator creates a new Iterator based on the provided param
func (b *btreeBuffer) NewIterator(param interface{}) Iterator {
	var e *memkv.Enumerator
	var err error
	if param == nil {
		e, err = b.tree.SeekFirst()
		if err != nil {
			return &btreeIter{ok: false}
		}
	} else {
		key := toIfaces(param.([]byte))
		e, _ = b.tree.Seek(key)
	}
	iter := &btreeIter{e: e}
	// the initial push...
	err = iter.Next()
	if err != nil {
		log.Error(err)
		return &btreeIter{ok: false}
	}
	return iter
}

// Close implements Iterator Close.
func (i *btreeIter) Close() {
	//noop
}

// Key implements Iterator Key.
func (i *btreeIter) Key() string {
	return i.k
}

// Value implements Iterator Value.
func (i *btreeIter) Value() []byte {
	return i.v
}

// Next implements Iterator Next.
func (i *btreeIter) Next() error {
	k, v, err := i.e.Next()
	i.k, i.v, i.ok = string(fromIfaces(k)), fromIfaces(v), err == nil
	return errors.Trace(err)
}

// Valid implements Iterator Valid.
func (i *btreeIter) Valid() bool {
	return i.ok
}

func toIfaces(v []byte) []interface{} {
	return []interface{}{v}
}

func fromIfaces(v []interface{}) []byte {
	if v == nil {
		return nil
	}
	return v[0].([]byte)
}
<|endoftext|>


<commit_before>package network

import (
	"fmt"
	"bytes"
)

// A NETWORK is a LIST of input NODEs and a LIST of output NODEs.
// The point of the network is to define a single entity which can evolve
// or learn on its own, even though it may be part of a larger framework.
type Network struct {
	// A network id
	Id        int
	// Is a name of this network */
	Name      string

	// The number of links in the net (-1 means not yet counted)
	numlinks  int

	// A list of all the nodes in the network
	all_nodes []*NNode
	// NNodes that input into the network
	Inputs    []*NNode
	// NNodes that output from the network
	Outputs   []*NNode
}

// Creates new network
func NewNetwork(in, out, all []*NNode, net_id int) *Network {
	n := Network{
		Id:net_id,
		Inputs:in,
		Outputs:out,
		all_nodes:all,
		numlinks:-1,
	}
	return &n
}

// Puts the network back into an initial state
func (n *Network) Flush() {
	// Flush back recursively
	for _, node := range n.all_nodes {
		node.Flushback()
	}
}

// Verify that network was successfully flushed for debugging
func (n *Network) FlushCheck() error {
	for _, node := range n.all_nodes {
		err := node.FlushbackCheck()
		if err != nil {
			return err
		}
	}
	return nil
}

// Prints the values of network outputs to the console
func (n *Network) PrintActivation() string {
	out := bytes.NewBufferString(fmt.Sprintf("Network %s with id %d outputs: (", n.Name, n.Id))
	for i, node := range n.Outputs {
		fmt.Fprintf(out, "[Output #%d: %s] ", i, node)
	}
	fmt.Fprint(out, ")")
	return out.String()
}

// Print the values of network inputs to the console
func (n *Network) PrintInput() string {
	out := bytes.NewBufferString(fmt.Sprintf("Network %s with id %d inputs: (", n.Name, n.Id))
	for i, node := range n.Inputs {
		fmt.Fprintf(out, "[Input #%d: %s] ", i, node)
	}
	fmt.Fprint(out, ")")
	return out.String()
}

// If at least one output is not active then return true
func (n *Network) OutputIsOff() bool {
	for _, node := range n.Outputs {
		if node.ActivationsCount == 0 {
			return true
		}
	}
	return false
}

// Activates the net such that all outputs are active
func (n *Network) Activate() (bool, error) {
	//For adding to the activesum
	add_amount := 0.0
	//Make sure we at least activate once
	one_time := false
	//Used in case the output is somehow truncated from the network
	abort_count := 0

	// Keep activating until all the outputs have become active
	// (This only happens on the first activation, because after that they are always active)
	for n.OutputIsOff() || !one_time {
		abort_count += 1

		if abort_count >= 20 {
			return false, nil//errors.New("Inputs disconnected from outputs!")
		}

		// For each neuron node, compute the sum of its incoming activation
		for _, np := range n.all_nodes {
			if np.IsNeuron() {
				np.ActivationSum = 0.0 // reset activation value
				np.isActive = false // flag node disabled

				// For each node's incoming connection, add the activity from the connection to the activesum
				for _, link := range np.Incoming {
					// Handle possible time delays
					if !link.IsTimeDelayed {
						add_amount = link.Weight * link.InNode.GetActiveOut()
						//fmt.Printf("%f -> %f\n", link.Weight, (*link.InNode).GetActiveOut())
						if link.InNode.isActive || link.InNode.IsSensor() {
							np.isActive = true
						}
					} else {
						add_amount = link.Weight * link.InNode.GetActiveOutTd()
					}
					np.ActivationSum += add_amount
				} // End {for} over incoming links
			} // End if != SENSOR
		}  // End {for} over all nodes

		// Now activate all the neuron nodes off their incoming activation
		for _, np := range n.all_nodes {
			if np.IsNeuron() {
				// Only activate if some active input came in
				if np.isActive {
					// Keep a memory of activations for potential time delayed connections
					np.saveActivations()

					// Now run the net activation through an activation function
					err := NodeActivators.ActivateNode(np)
					if err != nil {
						return false, err
					}

					// Increment the activation_count
					// First activation cannot be from nothing!!
					np.ActivationsCount++
				}
				//fmt.Printf("Node: %s, activation sum: %f, active: %t\n", np, np.ActivationSum, np.IsActive)
			}
		}
		one_time = true
	}
	return true, nil
}

// Takes an array of sensor values and loads it into SENSOR inputs ONLY
func (n *Network) LoadSensors(sensors []float64) {
	counter := 0
	for _, node := range n.Inputs {
		if node.IsSensor() {
			node.SensorLoad(sensors[counter])
			counter += 1
		}
	}
}

// Counts the number of nodes in the net
func (n *Network) NodeCount() int {
	return len(n.all_nodes)
}

// Counts the number of links in the net
func (n *Network) LinkCount() int {
	n.numlinks = 0
	for _, node := range n.all_nodes {
		n.numlinks += len(node.Incoming)
	}
	return n.numlinks
}

// Returns complexity of this network which is sum of nodes count and links count
func (n *Network) Complexity() int {
	return n.NodeCount() + n.LinkCount()
}

// This checks a POTENTIAL link between a potential in_node
// and potential out_node to see if it must be recurrent.
// Use count and thresh to jump out in the case of an infinite loop.
func (n *Network) IsRecurrent(in_node, out_node *NNode, count *int, thresh int) bool {
	// Count the node as visited
	*count++

	if *count > thresh {
		return false // Short out the whole thing - loop detected
	}

	if in_node == out_node {
		return true
	} else {
		// Check back on all links ...
		for _, link := range in_node.Incoming {
			// But skip links that are already recurrent -
			// We want to check back through the forward flow of signals only
			if !link.IsRecurrent {
				if n.IsRecurrent(link.InNode, out_node, count, thresh) {
					return true
				}
			}
		}
	}
	return false
}

// Find the maximum number of neurons between an output and an input
func (n *Network) MaxDepth() (int, error) {
	// The quick case when there are no hidden nodes
	if len(n.all_nodes) == len(n.Inputs) + len(n.Outputs) {
		return 1, nil // just one layer depth
	}

	max := 0 // The max depth
	for _, node := range n.Outputs {
		curr_depth, err := node.Depth(0)
		if err != nil {
			return curr_depth, err
		}
		if curr_depth > max {
			max = curr_depth
		}
	}
	return max, nil
}

// Returns all nodes in the network
func (n *Network) AllNodes() []*NNode {
	return n.all_nodes
}
<commit_msg>Fixed network activation method by removing obsolete saveActivations invocation - now it is in ActivateNode function<commit_after>package network

import (
	"fmt"
	"bytes"
)

// A NETWORK is a LIST of input NODEs and a LIST of output NODEs.
// The point of the network is to define a single entity which can evolve
// or learn on its own, even though it may be part of a larger framework.
type Network struct {
	// A network id
	Id        int
	// Is a name of this network */
	Name      string

	// The number of links in the net (-1 means not yet counted)
	numlinks  int

	// A list of all the nodes in the network
	all_nodes []*NNode
	// NNodes that input into the network
	Inputs    []*NNode
	// NNodes that output from the network
	Outputs   []*NNode
}

// Creates new network
func NewNetwork(in, out, all []*NNode, net_id int) *Network {
	n := Network{
		Id:net_id,
		Inputs:in,
		Outputs:out,
		all_nodes:all,
		numlinks:-1,
	}
	return &n
}

// Puts the network back into an initial state
func (n *Network) Flush() {
	// Flush back recursively
	for _, node := range n.all_nodes {
		node.Flushback()
	}
}

// Verify that network was successfully flushed for debugging
func (n *Network) FlushCheck() error {
	for _, node := range n.all_nodes {
		err := node.FlushbackCheck()
		if err != nil {
			return err
		}
	}
	return nil
}

// Prints the values of network outputs to the console
func (n *Network) PrintActivation() string {
	out := bytes.NewBufferString(fmt.Sprintf("Network %s with id %d outputs: (", n.Name, n.Id))
	for i, node := range n.Outputs {
		fmt.Fprintf(out, "[Output #%d: %s] ", i, node)
	}
	fmt.Fprint(out, ")")
	return out.String()
}

// Print the values of network inputs to the console
func (n *Network) PrintInput() string {
	out := bytes.NewBufferString(fmt.Sprintf("Network %s with id %d inputs: (", n.Name, n.Id))
	for i, node := range n.Inputs {
		fmt.Fprintf(out, "[Input #%d: %s] ", i, node)
	}
	fmt.Fprint(out, ")")
	return out.String()
}

// If at least one output is not active then return true
func (n *Network) OutputIsOff() bool {
	for _, node := range n.Outputs {
		if node.ActivationsCount == 0 {
			return true
		}
	}
	return false
}

// Activates the net such that all outputs are active
func (n *Network) Activate() (bool, error) {
	//For adding to the activesum
	add_amount := 0.0
	//Make sure we at least activate once
	one_time := false
	//Used in case the output is somehow truncated from the network
	abort_count := 0

	// Keep activating until all the outputs have become active
	// (This only happens on the first activation, because after that they are always active)
	for n.OutputIsOff() || !one_time {
		abort_count += 1

		if abort_count >= 20 {
			return false, nil//errors.New("Inputs disconnected from outputs!")
		}

		// For each neuron node, compute the sum of its incoming activation
		for _, np := range n.all_nodes {
			if np.IsNeuron() {
				np.ActivationSum = 0.0 // reset activation value
				np.isActive = false // flag node disabled

				// For each node's incoming connection, add the activity from the connection to the activesum
				for _, link := range np.Incoming {
					// Handle possible time delays
					if !link.IsTimeDelayed {
						add_amount = link.Weight * link.InNode.GetActiveOut()
						//fmt.Printf("%f -> %f\n", link.Weight, (*link.InNode).GetActiveOut())
						if link.InNode.isActive || link.InNode.IsSensor() {
							np.isActive = true
						}
					} else {
						add_amount = link.Weight * link.InNode.GetActiveOutTd()
					}
					np.ActivationSum += add_amount
				} // End {for} over incoming links
			} // End if != SENSOR
		}  // End {for} over all nodes

		// Now activate all the neuron nodes off their incoming activation
		for _, np := range n.all_nodes {
			if np.IsNeuron() {
				// Only activate if some active input came in
				if np.isActive {
					// Now run the net activation through an activation function
					err := NodeActivators.ActivateNode(np)
					if err != nil {
						return false, err
					}

					// Increment the activation_count
					// First activation cannot be from nothing!!
					np.ActivationsCount++
				}
				//fmt.Printf("Node: %s, activation sum: %f, active: %t\n", np, np.ActivationSum, np.IsActive)
			}
		}
		one_time = true
	}
	return true, nil
}

// Takes an array of sensor values and loads it into SENSOR inputs ONLY
func (n *Network) LoadSensors(sensors []float64) {
	counter := 0
	for _, node := range n.Inputs {
		if node.IsSensor() {
			node.SensorLoad(sensors[counter])
			counter += 1
		}
	}
}

// Counts the number of nodes in the net
func (n *Network) NodeCount() int {
	return len(n.all_nodes)
}

// Counts the number of links in the net
func (n *Network) LinkCount() int {
	n.numlinks = 0
	for _, node := range n.all_nodes {
		n.numlinks += len(node.Incoming)
	}
	return n.numlinks
}

// Returns complexity of this network which is sum of nodes count and links count
func (n *Network) Complexity() int {
	return n.NodeCount() + n.LinkCount()
}

// This checks a POTENTIAL link between a potential in_node
// and potential out_node to see if it must be recurrent.
// Use count and thresh to jump out in the case of an infinite loop.
func (n *Network) IsRecurrent(in_node, out_node *NNode, count *int, thresh int) bool {
	// Count the node as visited
	*count++

	if *count > thresh {
		return false // Short out the whole thing - loop detected
	}

	if in_node == out_node {
		return true
	} else {
		// Check back on all links ...
		for _, link := range in_node.Incoming {
			// But skip links that are already recurrent -
			// We want to check back through the forward flow of signals only
			if !link.IsRecurrent {
				if n.IsRecurrent(link.InNode, out_node, count, thresh) {
					return true
				}
			}
		}
	}
	return false
}

// Find the maximum number of neurons between an output and an input
func (n *Network) MaxDepth() (int, error) {
	// The quick case when there are no hidden nodes
	if len(n.all_nodes) == len(n.Inputs) + len(n.Outputs) {
		return 1, nil // just one layer depth
	}

	max := 0 // The max depth
	for _, node := range n.Outputs {
		curr_depth, err := node.Depth(0)
		if err != nil {
			return curr_depth, err
		}
		if curr_depth > max {
			max = curr_depth
		}
	}
	return max, nil
}

// Returns all nodes in the network
func (n *Network) AllNodes() []*NNode {
	return n.all_nodes
}
<|endoftext|>


<commit_before>package node

import (
	"github.com/AutoRoute/l2"
	"testing"
)

type testInterface struct {
	in  chan l2.EthFrame
	out chan l2.EthFrame
}

func (t testInterface) ReadFrame() (l2.EthFrame, error) {
	return <-t.in, nil
}

func (t testInterface) WriteFrame(e l2.EthFrame) error {
	t.out <- e
	return nil
}

func CreatePairedInterface() (l2.FrameReadWriter, l2.FrameReadWriter) {
	one := make(chan l2.EthFrame)
	two := make(chan l2.EthFrame)
	return testInterface{one, two}, testInterface{two, one}
}

func TestBasicExchange(t *testing.T) {
	var test_mac1 string = "aa:bb:cc:dd:ee:00"
	var test_mac2 string = "aa:bb:cc:dd:ee:11"

	var public_key1 PublicKey = pktest("test1")
	var public_key2 PublicKey = pktest("test2")

	one, two := CreatePairedInterface()

	nf1 := NewNeighborFinder(public_key1)
	nf2 := NewNeighborFinder(public_key2)

	outone := nf1.Find(test_mac1, one)
	outtwo := nf2.Find(test_mac2, two)
}
<commit_msg>Fixed NeighborData name.<commit_after>package node

import (
	"github.com/AutoRoute/l2"
	"testing"
)

type testInterface struct {
	in  chan l2.EthFrame
	out chan l2.EthFrame
}

func (t testInterface) ReadFrame() (l2.EthFrame, error) {
	return <-t.in, nil
}

func (t testInterface) WriteFrame(e l2.EthFrame) error {
	t.out <- e
	return nil
}

func CreatePairedInterface() (l2.FrameReadWriter, l2.FrameReadWriter) {
	one := make(chan l2.EthFrame)
	two := make(chan l2.EthFrame)
	return testInterface{one, two}, testInterface{two, one}
}

func TestBasicExchange(t *testing.T) {
	var test_mac1 string = "aa:bb:cc:dd:ee:00"
	var test_mac2 string = "aa:bb:cc:dd:ee:11"

	var public_key1 PublicKey = pktest("test1")
	var public_key2 PublicKey = pktest("test2")

	one, two := CreatePairedInterface()

	nf1 := NewNeighborData(public_key1)
	nf2 := NewNeighborData(public_key2)

	outone := nf1.Find(test_mac1, one)
	outtwo := nf2.Find(test_mac2, two)
}
<|endoftext|>


<commit_before>package poslog

import (
	"path/filepath"
	"testing"
)

func TestAppendPOSLog(t *testing.T) {
	filename := "./input/xml/POSLog-201612301530-54.xml"
	p := ImportXML(filename)
	if len(p.DayID.DayID) != 8 {
		t.Fail()
	}
	if filepath.Base(filename) != p.Filename {
		t.Fail()
	}
}
<commit_msg>remove dayid test<commit_after>package poslog

import (
	"path/filepath"
	"testing"
)

func TestAppendPOSLog(t *testing.T) {
	filename := "./input/xml/POSLog-201612301530-54.xml"
	p := ImportXML(filename)
	// if len(p.dayID.DayID) != 8 {
	// 	t.Fail()
	// }
	if filepath.Base(filename) != p.filename {
		t.Fail()
	}
}
<|endoftext|>


<commit_before>// revision.go

package basecouch

import (
	"crypto/md5"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"log"
)

// The body of a CouchDB document/revision as decoded from JSON.
type Body map[string]interface{}

func (db *Database) getRevision(docid, revid string, key RevKey) (Body, error) {
	var body Body
	err := db.bucket.Get(revKeyToString(key), &body)
	if err != nil {
		return nil, err
	}
	body["_id"] = docid
	body["_rev"] = revid
	return body, nil
}

func (db *Database) setRevision(body Body) (RevKey, error) {
	body = stripSpecialProperties(body)
	digester := sha1.New()
	digester.Write(canonicalEncoding(body))
	revKey := RevKey(fmt.Sprintf("%x", digester.Sum(nil)))
    //OPT: Using Add would be more efficient if the value's already stored, but go-couchbase
    // doesn't have an API for it yet.
	return revKey, db.bucket.Set(revKeyToString(revKey), 0, body)
}

//////// HELPERS:

func revKeyToString(key RevKey) string {
	return "rev:" + string(key)
}

func createRevID(generation int, parentRevID string, body Body) string {
	// This should produce the same results as TouchDB.
	digester := md5.New()
	digester.Write([]byte{byte(len(parentRevID))})
	digester.Write([]byte(parentRevID))
	digester.Write(canonicalEncoding(stripSpecialProperties(body)))
	return fmt.Sprintf("%d-%x", generation, digester.Sum(nil))
}

func parseRevID(revid string) (int, string) {
	if revid == "" {
		return 0, ""
	}
	var generation int
	var id string
	n, _ := fmt.Sscanf(revid, "%d-%s", &generation, &id)
	if n < 1 || generation < 1 {
		log.Printf("WARNING: parseRevID failed on %q", revid)
		return -1, ""
	}
	return generation, id
}

func compareRevIDs(id1, id2 string) int {
	gen1, sha1 := parseRevID(id1)
	gen2, sha2 := parseRevID(id2)
	switch {
	case gen1 > gen2:
		return 1
	case gen1 < gen2:
		return -1
	case sha1 > sha2:
		return 1
	case sha1 < sha2:
		return -1
	}
	return 0
}

func stripSpecialProperties(body Body) Body {
	stripped := Body{}
	for key, value := range body {
		if key == "" || key[0] != '_' || key == "_attachments" || key == "_deleted" {
			stripped[key] = value
		}
	}
	return stripped
}

func canonicalEncoding(body Body) []byte {
	encoded, err := json.Marshal(body) //FIX: Use canonical JSON encoder
	if err != nil {
		panic(fmt.Sprintf("Couldn't encode body %v", body))
	}
	return encoded
}
<commit_msg>Store rev bodies using Add, for efficiency<commit_after>// revision.go

package basecouch

import (
	"crypto/md5"
	"crypto/sha1"
    "encoding/base64"
	"encoding/json"
	"fmt"
	"log"
)

// The body of a CouchDB document/revision as decoded from JSON.
type Body map[string]interface{}

func (db *Database) getRevision(docid, revid string, key RevKey) (Body, error) {
	var body Body
	err := db.bucket.Get(revKeyToString(key), &body)
	if err != nil {
		return nil, err
	}
	body["_id"] = docid
	body["_rev"] = revid
	return body, nil
}

func (db *Database) setRevision(body Body) (RevKey, error) {
	body = stripSpecialProperties(body)
	digester := sha1.New()
	digester.Write(canonicalEncoding(body))
	revKey := RevKey(base64.StdEncoding.EncodeToString(digester.Sum(nil)))
    _,err := db.bucket.Add(revKeyToString(revKey), 0, body)
	return revKey, err
}

//////// HELPERS:

func revKeyToString(key RevKey) string {
	return "rev:" + string(key)
}

func createRevID(generation int, parentRevID string, body Body) string {
	// This should produce the same results as TouchDB.
	digester := md5.New()
	digester.Write([]byte{byte(len(parentRevID))})
	digester.Write([]byte(parentRevID))
	digester.Write(canonicalEncoding(stripSpecialProperties(body)))
	return fmt.Sprintf("%d-%x", generation, digester.Sum(nil))
}

func parseRevID(revid string) (int, string) {
	if revid == "" {
		return 0, ""
	}
	var generation int
	var id string
	n, _ := fmt.Sscanf(revid, "%d-%s", &generation, &id)
	if n < 1 || generation < 1 {
		log.Printf("WARNING: parseRevID failed on %q", revid)
		return -1, ""
	}
	return generation, id
}

func compareRevIDs(id1, id2 string) int {
	gen1, sha1 := parseRevID(id1)
	gen2, sha2 := parseRevID(id2)
	switch {
	case gen1 > gen2:
		return 1
	case gen1 < gen2:
		return -1
	case sha1 > sha2:
		return 1
	case sha1 < sha2:
		return -1
	}
	return 0
}

func stripSpecialProperties(body Body) Body {
	stripped := Body{}
	for key, value := range body {
		if key == "" || key[0] != '_' || key == "_attachments" || key == "_deleted" {
			stripped[key] = value
		}
	}
	return stripped
}

func canonicalEncoding(body Body) []byte {
	encoded, err := json.Marshal(body) //FIX: Use canonical JSON encoder
	if err != nil {
		panic(fmt.Sprintf("Couldn't encode body %v", body))
	}
	return encoded
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/grafana/grafana/pkg/log"
	"github.com/raintank/raintank-metric/metric_tank/consolidation"
	"math"
	"net/http"
	_ "net/http/pprof"
	"strconv"
	"strings"
	"sync"
	"time"
)

var bufPool = sync.Pool{
	New: func() interface{} { return make([]byte, 0) },
}

type Point struct {
	Val float64
	Ts  uint32
}

type Series struct {
	Target     string
	Datapoints []Point
	Interval   uint32
}

func graphiteJSON(b []byte, series []Series) ([]byte, error) {
	b = append(b, '[')
	for _, s := range series {
		b = append(b, `{"Target":"`...)
		b = append(b, s.Target...)
		b = append(b, `","Datapoints":[`...)
		for _, p := range s.Datapoints {
			b = append(b, '[')
			if math.IsNaN(p.Val) {
				b = append(b, `null,`...)
			} else {
				b = strconv.AppendFloat(b, p.Val, 'f', 3, 64)
				b = append(b, ',')
			}
			b = strconv.AppendUint(b, uint64(p.Ts), 10)
			b = append(b, `],`...)
		}
		b = b[:len(b)-1] // cut last comma
		b = append(b, `],"Interval":`...)
		b = strconv.AppendInt(b, int64(s.Interval), 10)
		b = append(b, `},`...)
	}
	b = b[:len(b)-1] // cut last comma
	b = append(b, ']')
	return b, nil
}

func get(store Store, defCache *DefCache, aggSettings []aggSetting) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		Get(w, req, store, defCache, aggSettings)
	}
}

// note: we don't normalize/quantize/fill-unknowns
// we just serve what we know
func Get(w http.ResponseWriter, req *http.Request, store Store, defCache *DefCache, aggSettings []aggSetting) {
	pre := time.Now()
	req.ParseForm()
	log.Info("http.Get(): INCOMING REQ %q from: %q, to: %q targets: %q, maxDataPoints: %q",
		req.Method, req.Form.Get("from"), req.Form.Get("to"), req.Form["target"], req.Form.Get("maxDataPoints"))

	maxDataPoints := uint32(800)
	maxDataPointsStr := req.Form.Get("maxDataPoints")
	var err error
	if maxDataPointsStr != "" {
		tmp, err := strconv.Atoi(maxDataPointsStr)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		maxDataPoints = uint32(tmp)
	}

	targets, ok := req.Form["target"]
	if !ok {
		http.Error(w, "missing target arg", http.StatusBadRequest)
		return
	}
	now := time.Now()
	fromUnix := uint32(now.Add(-time.Duration(24) * time.Hour).Unix())
	toUnix := uint32(now.Add(time.Duration(1) * time.Second).Unix())
	from := req.Form.Get("from")
	if from != "" {
		fromUnixInt, err := strconv.Atoi(from)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		fromUnix = uint32(fromUnixInt)
	}
	to := req.Form.Get("to")
	if to != "" {
		toUnixInt, err := strconv.Atoi(to)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		toUnix = uint32(toUnixInt)
	}
	if fromUnix >= toUnix {
		http.Error(w, "to must be higher than from", http.StatusBadRequest)
		return
	}

	reqs := make([]Req, len(targets))
	for i, target := range targets {
		var consolidateBy string
		id := target
		// yes, i am aware of the arguably grossness of the below.
		// however, it is solid based on the documented allowed input format.
		// once we need to support several functions, we can implement
		// a proper expression parser
		if strings.HasPrefix(target, "consolidateBy(") {
			if target[len(target)-2:] != "')" || !strings.Contains(target, ",'") || strings.Count(target, "'") != 2 || strings.Count(target, ",") != 1 {
				http.Error(w, "target parse error", http.StatusBadRequest)
				return
			}
			consolidateBy = target[strings.Index(target, "'")+1 : strings.LastIndex(target, "'")]
			id = target[strings.Index(target, "(")+1 : strings.Index(target, ",")]
		}

		if consolidateBy == "" {
			def, ok := defCache.Get(id)
			consolidateBy = "avg"
			if ok && def.TargetType == "counter" {
				consolidateBy = "last"
			}
		}
		var consolidator consolidation.Consolidator
		switch consolidateBy {
		case "avg", "average":
			consolidator = consolidation.Avg
		case "min":
			consolidator = consolidation.Min
		case "max":
			consolidator = consolidation.Max
		case "sum":
			consolidator = consolidation.Sum
		default:
			http.Error(w, "unrecognized consolidation function", http.StatusBadRequest)
			return
		}
		req := NewReq(id, fromUnix, toUnix, maxDataPoints, consolidator)
		reqs[i] = req
	}
	err = findMetricsForRequests(reqs, defCache)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	reqs, err = alignRequests(reqs, aggSettings)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	out := make([]Series, len(reqs))
	for i, req := range reqs {
		log.Debug("===================================")
		log.Debug("HTTP Get()          %s", req)
		points, interval, err := getTarget(store, req)
		if err != nil {
			log.Error(0, err.Error())
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		out[i] = Series{
			Target:     targets[i],
			Datapoints: points,
			Interval:   interval,
		}
	}
	js := bufPool.Get().([]byte)
	js, err = graphiteJSON(js, out)
	if err != nil {
		log.Error(0, err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	bufPool.Put(js[:0])

	w.Header().Set("Content-Type", "application/json")
	reqHandleDuration.Value(time.Now().Sub(pre))
	w.Write(js)
}

// report ApplicationStatus for use by loadBalancer healthChecks.
// We only want requests to be sent to this node if it is the primary
// node or if it has been online for at *warmUpPeriod
func appStatus(w http.ResponseWriter, req *http.Request) {
	if clusterStatus.IsPrimary() {
		w.Write([]byte("OK"))
		return
	}
	if time.Since(startupTime) < (time.Duration(*warmUpPeriod) * time.Second) {
		http.Error(w, "Service not ready", http.StatusServiceUnavailable)
		return
	}

	w.Write([]byte("OK"))
	return
}
<commit_msg>limit amount of points and timeframe you can query<commit_after>package main

import (
	"github.com/grafana/grafana/pkg/log"
	"github.com/raintank/raintank-metric/metric_tank/consolidation"
	"math"
	"net/http"
	_ "net/http/pprof"
	"strconv"
	"strings"
	"sync"
	"time"
)

var bufPool = sync.Pool{
	New: func() interface{} { return make([]byte, 0) },
}

type Point struct {
	Val float64
	Ts  uint32
}

type Series struct {
	Target     string
	Datapoints []Point
	Interval   uint32
}

func graphiteJSON(b []byte, series []Series) ([]byte, error) {
	b = append(b, '[')
	for _, s := range series {
		b = append(b, `{"Target":"`...)
		b = append(b, s.Target...)
		b = append(b, `","Datapoints":[`...)
		for _, p := range s.Datapoints {
			b = append(b, '[')
			if math.IsNaN(p.Val) {
				b = append(b, `null,`...)
			} else {
				b = strconv.AppendFloat(b, p.Val, 'f', 3, 64)
				b = append(b, ',')
			}
			b = strconv.AppendUint(b, uint64(p.Ts), 10)
			b = append(b, `],`...)
		}
		b = b[:len(b)-1] // cut last comma
		b = append(b, `],"Interval":`...)
		b = strconv.AppendInt(b, int64(s.Interval), 10)
		b = append(b, `},`...)
	}
	b = b[:len(b)-1] // cut last comma
	b = append(b, ']')
	return b, nil
}

func get(store Store, defCache *DefCache, aggSettings []aggSetting) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		Get(w, req, store, defCache, aggSettings)
	}
}

// note: we don't normalize/quantize/fill-unknowns
// we just serve what we know
func Get(w http.ResponseWriter, req *http.Request, store Store, defCache *DefCache, aggSettings []aggSetting) {
	pre := time.Now()
	req.ParseForm()
	log.Info("http.Get(): INCOMING REQ %q from: %q, to: %q targets: %q, maxDataPoints: %q",
		req.Method, req.Form.Get("from"), req.Form.Get("to"), req.Form["target"], req.Form.Get("maxDataPoints"))

	maxDataPoints := uint32(800)
	maxDataPointsStr := req.Form.Get("maxDataPoints")
	var err error
	if maxDataPointsStr != "" {
		tmp, err := strconv.Atoi(maxDataPointsStr)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		maxDataPoints = uint32(tmp)
	}

	targets, ok := req.Form["target"]
	if !ok {
		http.Error(w, "missing target arg", http.StatusBadRequest)
		return
	}
	if len(targets)*int(maxDataPoints) > 100*1000 {
		http.Error(w, "too much data requested", http.StatusBadRequest)
		return
	}

	now := time.Now()
	fromUnix := uint32(now.Add(-time.Duration(24) * time.Hour).Unix())
	toUnix := uint32(now.Add(time.Duration(1) * time.Second).Unix())
	from := req.Form.Get("from")
	if from != "" {
		fromUnixInt, err := strconv.Atoi(from)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		fromUnix = uint32(fromUnixInt)
	}
	to := req.Form.Get("to")
	if to != "" {
		toUnixInt, err := strconv.Atoi(to)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		toUnix = uint32(toUnixInt)
	}
	if fromUnix >= toUnix {
		http.Error(w, "to must be higher than from", http.StatusBadRequest)
		return
	}
	if len(targets)*int(toUnix-fromUnix) > 2*365*24*3600 {
		http.Error(w, "too much data requested", http.StatusBadRequest)
		return
	}

	reqs := make([]Req, len(targets))
	for i, target := range targets {
		var consolidateBy string
		id := target
		// yes, i am aware of the arguably grossness of the below.
		// however, it is solid based on the documented allowed input format.
		// once we need to support several functions, we can implement
		// a proper expression parser
		if strings.HasPrefix(target, "consolidateBy(") {
			if target[len(target)-2:] != "')" || !strings.Contains(target, ",'") || strings.Count(target, "'") != 2 || strings.Count(target, ",") != 1 {
				http.Error(w, "target parse error", http.StatusBadRequest)
				return
			}
			consolidateBy = target[strings.Index(target, "'")+1 : strings.LastIndex(target, "'")]
			id = target[strings.Index(target, "(")+1 : strings.Index(target, ",")]
		}

		if consolidateBy == "" {
			def, ok := defCache.Get(id)
			consolidateBy = "avg"
			if ok && def.TargetType == "counter" {
				consolidateBy = "last"
			}
		}
		var consolidator consolidation.Consolidator
		switch consolidateBy {
		case "avg", "average":
			consolidator = consolidation.Avg
		case "min":
			consolidator = consolidation.Min
		case "max":
			consolidator = consolidation.Max
		case "sum":
			consolidator = consolidation.Sum
		default:
			http.Error(w, "unrecognized consolidation function", http.StatusBadRequest)
			return
		}
		req := NewReq(id, fromUnix, toUnix, maxDataPoints, consolidator)
		reqs[i] = req
	}
	err = findMetricsForRequests(reqs, defCache)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	reqs, err = alignRequests(reqs, aggSettings)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	out := make([]Series, len(reqs))
	for i, req := range reqs {
		log.Debug("===================================")
		log.Debug("HTTP Get()          %s", req)
		points, interval, err := getTarget(store, req)
		if err != nil {
			log.Error(0, err.Error())
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		out[i] = Series{
			Target:     targets[i],
			Datapoints: points,
			Interval:   interval,
		}
	}
	js := bufPool.Get().([]byte)
	js, err = graphiteJSON(js, out)
	if err != nil {
		log.Error(0, err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	bufPool.Put(js[:0])

	w.Header().Set("Content-Type", "application/json")
	reqHandleDuration.Value(time.Now().Sub(pre))
	w.Write(js)
}

// report ApplicationStatus for use by loadBalancer healthChecks.
// We only want requests to be sent to this node if it is the primary
// node or if it has been online for at *warmUpPeriod
func appStatus(w http.ResponseWriter, req *http.Request) {
	if clusterStatus.IsPrimary() {
		w.Write([]byte("OK"))
		return
	}
	if time.Since(startupTime) < (time.Duration(*warmUpPeriod) * time.Second) {
		http.Error(w, "Service not ready", http.StatusServiceUnavailable)
		return
	}

	w.Write([]byte("OK"))
	return
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 ScyllaDB
// Use of this source code is governed by a ALv2-style
// license that can be found in the LICENSE file.

package migrate

import (
	"context"

	"github.com/scylladb/gocqlx/v2"
)

// CallbackEvent specifies type of the event when calling CallbackFunc.
type CallbackEvent uint8

// enumeration of CallbackEvents
const (
	BeforeMigration CallbackEvent = iota
	AfterMigration
	CallComment
)

// CallbackFunc enables execution of arbitrary Go code during migration.
// If error is returned the migration is aborted.
// BeforeMigration and AfterMigration are triggered before and after processing
// of each migration file respectively.
// CallComment is triggered for each comment in a form `-- CALL <name>;` (note the semicolon).
type CallbackFunc func(ctx context.Context, session gocqlx.Session, ev CallbackEvent, name string) error

// Callback is means of executing Go code during migrations.
// Use this variable to register a global callback dispatching function.
// See CallbackFunc for details.
var Callback CallbackFunc
<commit_msg>migrate: Add CallbackRegister<commit_after>// Copyright (C) 2017 ScyllaDB
// Use of this source code is governed by a ALv2-style
// license that can be found in the LICENSE file.

package migrate

import (
	"context"
	"errors"

	"github.com/scylladb/gocqlx/v2"
)

// CallbackEvent specifies type of the event when calling CallbackFunc.
type CallbackEvent uint8

// enumeration of CallbackEvents
const (
	BeforeMigration CallbackEvent = iota
	AfterMigration
	CallComment
)

// CallbackFunc enables execution of arbitrary Go code during migration.
// If error is returned the migration is aborted.
// BeforeMigration and AfterMigration are triggered before and after processing
// of each migration file respectively.
// CallComment is triggered for each comment in a form `-- CALL <name>;` (note the semicolon).
type CallbackFunc func(ctx context.Context, session gocqlx.Session, ev CallbackEvent, name string) error

// Callback is means of executing Go code during migrations.
// Use this variable to register a global callback dispatching function.
// See CallbackFunc for details.
var Callback CallbackFunc

type nameEvent struct {
	name  string
	event CallbackEvent
}

// CallbackRegister allows to register a handlers for an event type and a name.
// It dispatches calls to the registered handlers.
// If there is no handler registered for CallComment an error is returned.
type CallbackRegister map[nameEvent]CallbackFunc

// Add registers a callback handler.
func (r CallbackRegister) Add(ev CallbackEvent, name string, f CallbackFunc) {
	r[nameEvent{name, ev}] = f
}

// Callback is CallbackFunc.
func (r CallbackRegister) Callback(ctx context.Context, session gocqlx.Session, ev CallbackEvent, name string) error {
	f, ok := r[nameEvent{name, ev}]
	if !ok {
		if ev == CallComment {
			return errors.New("missing handler")
		}
		return nil
	}
	return f(ctx, session, ev, name)
}
<|endoftext|>