<commit_before>package main

import (
	"bufio"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"github.com/shopify/sarama"
	"io/ioutil"
	"os"
	"os/signal"
	"strings"
	"sync"
)

func brokers() []string {
	s := os.Getenv("KAFKA_BROKERS")
	if s == "" {
		s = "127.0.0.1:9092"
	}
	return strings.Split(s, ",")
}

func tlsConfig() (useTLS bool, config *tls.Config, err error) {
	// if SSL_CA_BUNDLE_PATH isn't set, just don't use TLS at all
	caPath := os.Getenv("SSL_CA_BUNDLE_PATH")
	if caPath == "" {
		return
	}
	useTLS = true
	config = new(tls.Config)
	caCerts, err := ioutil.ReadFile(caPath)
	if err != nil {
		err = fmt.Errorf("error reading $SSL_CA_BUNDLE_PATH: %v", err)
		return
	}

	config.RootCAs = x509.NewCertPool()
	if !config.RootCAs.AppendCertsFromPEM(caCerts) {
		err = fmt.Errorf("$SSL_CA_BUNDLE_PATH=%q was empty", caPath)
		return
	}

	// if $SSL_CERT_PATH or $SSL_KEY_PATH aren't set, skip client cert
	certPath, keyPath := os.Getenv("SSL_CERT_PATH"), os.Getenv("SSL_KEY_PATH")
	if certPath == "" || keyPath == "" {
		return
	}

	keypair, err := tls.LoadX509KeyPair(certPath, keyPath)
	if err != nil {
		err = fmt.Errorf("error reading $SSL_CRT_PATH/$SSL_KEY_PATH: %v", err)
		return
	}
	config.Certificates = []tls.Certificate{keypair}
	return
}

func offsets(client sarama.Client, topic string, partition int32) (oldest int64, newest int64) {
	oldest, err := client.GetOffset(topic, partition, sarama.OffsetOldest)
	must(err)
	newest, err = client.GetOffset(topic, partition, sarama.OffsetNewest)
	must(err)
	return oldest, newest
}

var (
	topic     string
	partition int32
	offset    int64
	n         int
)

var cmdProduce = &Command{
	Usage: "produce --topic <topic>",
	Short: "produce messages to given topic",
	Long: `
Produces a message to the given topic with the data given by
reading stdin (newline delimited).

Example:

    $ echo content | k produce --topic foo`,
	Run: runProduce,
}

func runProduce(cmd *Command, args []string) {
	brokers := brokers()
	config := sarama.NewConfig()
	useTLS, tlsConfig, err := tlsConfig()
	must(err)
	config.Net.TLS.Enable = useTLS
	config.Net.TLS.Config = tlsConfig
	config.ClientID = "k produce"
	config.Producer.Return.Successes = true
	client, err := sarama.NewClient(brokers, config)
	must(err)
	defer client.Close()

	producer, err := sarama.NewAsyncProducerFromClient(client)
	must(err)

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)
	defer close(signals)

	var wg sync.WaitGroup
	var enqueued, successes, errors int

	wg.Add(1)
	go func() {
		defer wg.Done()
		for _ = range producer.Successes() {
			successes++
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		for err := range producer.Errors() {
			fmt.Fprintf(os.Stderr, "Failed to produce message: %s\n", err)
			errors++
		}
	}()

	scanner := bufio.NewScanner(os.Stdin)
producerLoop:
	for scanner.Scan() {
		line := scanner.Text()
		idx := strings.Index(line, "\t")
		var msg *sarama.ProducerMessage
		if idx > 0 {
			msg = &sarama.ProducerMessage{Topic: topic, Key: sarama.ByteEncoder(line[0:idx]), Value: sarama.ByteEncoder(line[idx+1:])}
		} else {
			msg = &sarama.ProducerMessage{Topic: topic, Key: nil, Value: sarama.ByteEncoder(line)}
		}
		select {
		case producer.Input() <- msg:
			enqueued++
		case <-signals:
			break producerLoop
		}
	}

	producer.AsyncClose()
	wg.Wait()
	fmt.Fprintf(os.Stderr, "messages produced: %d, errors: %d\n", successes, errors)
}

var cmdConsume = &Command{
	Usage: "consume --topic <topic> [--partition <partition>] [--offset <offset>] [-n <n_messages>]",
	Short: "consume messages from given topic",
	Long: `
Consumes messages from the given topic and writes them to stdout.

Example:

    $ k consume --topic foo`,
	Run: runConsume,
}

func runConsume(cmd *Command, args []string) {
	brokers := brokers()
	config := sarama.NewConfig()
	useTLS, tlsConfig, err := tlsConfig()
	must(err)
	config.Net.TLS.Enable = useTLS
	config.Net.TLS.Config = tlsConfig
	config.ClientID = "k consume"
	config.Consumer.Return.Errors = true
	client, err := sarama.NewClient(brokers, config)
	must(err)
	defer client.Close()

	consumer, err := sarama.NewConsumerFromClient(client)
	must(err)
	defer consumer.Close()

	signals := make(chan os.Signal, 1)
	defer close(signals)
	signal.Notify(signals, os.Interrupt)

	// calculate a starting offset
	_, newestOffset := offsets(client, topic, partition)
	must(err)
	startingOffset := newestOffset
	if offset < 0 {
		startingOffset = newestOffset + offset
	} else if offset > 0 {
		startingOffset = offset
	}

	// TODO: support consuming all partitions
	fmt.Fprintf(os.Stderr, "Using starting offset: %d\n", startingOffset)
	partConsumer, err := consumer.ConsumePartition(topic, partition, startingOffset)
	must(err)
	defer partConsumer.Close()

	var received, errors int
consumerLoop:
	for {
		select {
		case msg := <-partConsumer.Messages():
			if msg.Key != nil {
				fmt.Printf("%s\t", string(msg.Key))
			}
			fmt.Println(string(msg.Value))
			received++
			if n > 0 && received >= n {
				break consumerLoop
			}
		case err := <-partConsumer.Errors():
			fmt.Fprintf(os.Stderr, "Failed to receive message: %s\n", err)
			errors++
		case <-signals:
			break consumerLoop
		}
	}

	fmt.Fprintf(os.Stderr, "Messages received: %d, errors: %d\n", received, errors)
}

var cmdOffsets = &Command{
	Usage: "offsets --topic <topic>",
	Short: "show the oldest and newest offset for a given topic and partition",
	Long: `
Prints oldest and newest offsets for the given topic to stdout.

Example:

    $ k offsets --topic foo`,
	Run: runOffsets,
}

func runOffsets(cmd *Command, args []string) {
	brokers := brokers()
	config := sarama.NewConfig()
	useTLS, tlsConfig, err := tlsConfig()
	must(err)
	config.Net.TLS.Enable = useTLS
	config.Net.TLS.Config = tlsConfig
	config.ClientID = "k offsets"
	client, err := sarama.NewClient(brokers, config)
	must(err)
	defer client.Close()

	// get partitions for topic
	parts, err := client.Partitions(topic)
	must(err)

	// print offsets for each partition
	for _, part := range parts {
		oldestOffset, newestOffset := offsets(client, topic, 0)
		must(err)
		fmt.Printf("partition=%d oldest=%d newest=%d\n", part, oldestOffset, newestOffset)
	}
}

var cmdTopics = &Command{
	Usage: "topics",
	Short: "show the list of topics",
	Long: `
Prints the list of topics to stdout.

Example:

    $ k topics`,
	Run: runTopics,
}

func runTopics(cmd *Command, args []string) {
	brokers := brokers()
	config := sarama.NewConfig()
	useTLS, tlsConfig, err := tlsConfig()
	must(err)
	config.Net.TLS.Enable = useTLS
	config.Net.TLS.Config = tlsConfig
	config.ClientID = "k topics"
	client, err := sarama.NewClient(brokers, config)
	must(err)
	defer client.Close()

	// get list of topics
	topics, err := client.Topics()
	must(err)
	for _, topic := range topics {
		fmt.Println(topic)
	}
}

func init() {
	cmdProduce.Flag.StringVarP(&topic, "topic", "t", "k", "produce to topic")
	cmdConsume.Flag.StringVarP(&topic, "topic", "t", "k", "topic to consume")
	cmdConsume.Flag.Int32VarP(&partition, "partition", "p", 0, "partition to consume")
	cmdConsume.Flag.Int64VarP(&offset, "offset", "o", 0, "starting offset for consumer")
	cmdConsume.Flag.IntVarP(&n, "n", "n", -1, "number of messages to consume")
	cmdOffsets.Flag.StringVarP(&topic, "topic", "t", "k", "get offsets for topic")
}
<commit_msg>Fix typo in environment variable.<commit_after>package main

import (
	"bufio"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"github.com/shopify/sarama"
	"io/ioutil"
	"os"
	"os/signal"
	"strings"
	"sync"
)

func brokers() []string {
	s := os.Getenv("KAFKA_BROKERS")
	if s == "" {
		s = "127.0.0.1:9092"
	}
	return strings.Split(s, ",")
}

func tlsConfig() (useTLS bool, config *tls.Config, err error) {
	// if SSL_CA_BUNDLE_PATH isn't set, just don't use TLS at all
	caPath := os.Getenv("SSL_CA_BUNDLE_PATH")
	if caPath == "" {
		return
	}
	useTLS = true
	config = new(tls.Config)
	caCerts, err := ioutil.ReadFile(caPath)
	if err != nil {
		err = fmt.Errorf("error reading $SSL_CA_BUNDLE_PATH: %v", err)
		return
	}

	config.RootCAs = x509.NewCertPool()
	if !config.RootCAs.AppendCertsFromPEM(caCerts) {
		err = fmt.Errorf("$SSL_CA_BUNDLE_PATH=%q was empty", caPath)
		return
	}

	// if $SSL_CERT_PATH or $SSL_KEY_PATH aren't set, skip client cert
	certPath, keyPath := os.Getenv("SSL_CRT_PATH"), os.Getenv("SSL_KEY_PATH")
	if certPath == "" || keyPath == "" {
		return
	}

	keypair, err := tls.LoadX509KeyPair(certPath, keyPath)
	if err != nil {
		err = fmt.Errorf("error reading $SSL_CRT_PATH/$SSL_KEY_PATH: %v", err)
		return
	}
	config.Certificates = []tls.Certificate{keypair}
	return
}

func offsets(client sarama.Client, topic string, partition int32) (oldest int64, newest int64) {
	oldest, err := client.GetOffset(topic, partition, sarama.OffsetOldest)
	must(err)
	newest, err = client.GetOffset(topic, partition, sarama.OffsetNewest)
	must(err)
	return oldest, newest
}

var (
	topic     string
	partition int32
	offset    int64
	n         int
)

var cmdProduce = &Command{
	Usage: "produce --topic <topic>",
	Short: "produce messages to given topic",
	Long: `
Produces a message to the given topic with the data given by
reading stdin (newline delimited).

Example:

    $ echo content | k produce --topic foo`,
	Run: runProduce,
}

func runProduce(cmd *Command, args []string) {
	brokers := brokers()
	config := sarama.NewConfig()
	useTLS, tlsConfig, err := tlsConfig()
	must(err)
	config.Net.TLS.Enable = useTLS
	config.Net.TLS.Config = tlsConfig
	config.ClientID = "k produce"
	config.Producer.Return.Successes = true
	client, err := sarama.NewClient(brokers, config)
	must(err)
	defer client.Close()

	producer, err := sarama.NewAsyncProducerFromClient(client)
	must(err)

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)
	defer close(signals)

	var wg sync.WaitGroup
	var enqueued, successes, errors int

	wg.Add(1)
	go func() {
		defer wg.Done()
		for _ = range producer.Successes() {
			successes++
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		for err := range producer.Errors() {
			fmt.Fprintf(os.Stderr, "Failed to produce message: %s\n", err)
			errors++
		}
	}()

	scanner := bufio.NewScanner(os.Stdin)
producerLoop:
	for scanner.Scan() {
		line := scanner.Text()
		idx := strings.Index(line, "\t")
		var msg *sarama.ProducerMessage
		if idx > 0 {
			msg = &sarama.ProducerMessage{Topic: topic, Key: sarama.ByteEncoder(line[0:idx]), Value: sarama.ByteEncoder(line[idx+1:])}
		} else {
			msg = &sarama.ProducerMessage{Topic: topic, Key: nil, Value: sarama.ByteEncoder(line)}
		}
		select {
		case producer.Input() <- msg:
			enqueued++
		case <-signals:
			break producerLoop
		}
	}

	producer.AsyncClose()
	wg.Wait()
	fmt.Fprintf(os.Stderr, "messages produced: %d, errors: %d\n", successes, errors)
}

var cmdConsume = &Command{
	Usage: "consume --topic <topic> [--partition <partition>] [--offset <offset>] [-n <n_messages>]",
	Short: "consume messages from given topic",
	Long: `
Consumes messages from the given topic and writes them to stdout.

Example:

    $ k consume --topic foo`,
	Run: runConsume,
}

func runConsume(cmd *Command, args []string) {
	brokers := brokers()
	config := sarama.NewConfig()
	useTLS, tlsConfig, err := tlsConfig()
	must(err)
	config.Net.TLS.Enable = useTLS
	config.Net.TLS.Config = tlsConfig
	config.ClientID = "k consume"
	config.Consumer.Return.Errors = true
	client, err := sarama.NewClient(brokers, config)
	must(err)
	defer client.Close()

	consumer, err := sarama.NewConsumerFromClient(client)
	must(err)
	defer consumer.Close()

	signals := make(chan os.Signal, 1)
	defer close(signals)
	signal.Notify(signals, os.Interrupt)

	// calculate a starting offset
	_, newestOffset := offsets(client, topic, partition)
	must(err)
	startingOffset := newestOffset
	if offset < 0 {
		startingOffset = newestOffset + offset
	} else if offset > 0 {
		startingOffset = offset
	}

	// TODO: support consuming all partitions
	fmt.Fprintf(os.Stderr, "Using starting offset: %d\n", startingOffset)
	partConsumer, err := consumer.ConsumePartition(topic, partition, startingOffset)
	must(err)
	defer partConsumer.Close()

	var received, errors int
consumerLoop:
	for {
		select {
		case msg := <-partConsumer.Messages():
			if msg.Key != nil {
				fmt.Printf("%s\t", string(msg.Key))
			}
			fmt.Println(string(msg.Value))
			received++
			if n > 0 && received >= n {
				break consumerLoop
			}
		case err := <-partConsumer.Errors():
			fmt.Fprintf(os.Stderr, "Failed to receive message: %s\n", err)
			errors++
		case <-signals:
			break consumerLoop
		}
	}

	fmt.Fprintf(os.Stderr, "Messages received: %d, errors: %d\n", received, errors)
}

var cmdOffsets = &Command{
	Usage: "offsets --topic <topic>",
	Short: "show the oldest and newest offset for a given topic and partition",
	Long: `
Prints oldest and newest offsets for the given topic to stdout.

Example:

    $ k offsets --topic foo`,
	Run: runOffsets,
}

func runOffsets(cmd *Command, args []string) {
	brokers := brokers()
	config := sarama.NewConfig()
	useTLS, tlsConfig, err := tlsConfig()
	must(err)
	config.Net.TLS.Enable = useTLS
	config.Net.TLS.Config = tlsConfig
	config.ClientID = "k offsets"
	client, err := sarama.NewClient(brokers, config)
	must(err)
	defer client.Close()

	// get partitions for topic
	parts, err := client.Partitions(topic)
	must(err)

	// print offsets for each partition
	for _, part := range parts {
		oldestOffset, newestOffset := offsets(client, topic, 0)
		must(err)
		fmt.Printf("partition=%d oldest=%d newest=%d\n", part, oldestOffset, newestOffset)
	}
}

var cmdTopics = &Command{
	Usage: "topics",
	Short: "show the list of topics",
	Long: `
Prints the list of topics to stdout.

Example:

    $ k topics`,
	Run: runTopics,
}

func runTopics(cmd *Command, args []string) {
	brokers := brokers()
	config := sarama.NewConfig()
	useTLS, tlsConfig, err := tlsConfig()
	must(err)
	config.Net.TLS.Enable = useTLS
	config.Net.TLS.Config = tlsConfig
	config.ClientID = "k topics"
	client, err := sarama.NewClient(brokers, config)
	must(err)
	defer client.Close()

	// get list of topics
	topics, err := client.Topics()
	must(err)
	for _, topic := range topics {
		fmt.Println(topic)
	}
}

func init() {
	cmdProduce.Flag.StringVarP(&topic, "topic", "t", "k", "produce to topic")
	cmdConsume.Flag.StringVarP(&topic, "topic", "t", "k", "topic to consume")
	cmdConsume.Flag.Int32VarP(&partition, "partition", "p", 0, "partition to consume")
	cmdConsume.Flag.Int64VarP(&offset, "offset", "o", 0, "starting offset for consumer")
	cmdConsume.Flag.IntVarP(&n, "n", "n", -1, "number of messages to consume")
	cmdOffsets.Flag.StringVarP(&topic, "topic", "t", "k", "get offsets for topic")
}
<|endoftext|>


<commit_before>package itchfs

import (
	"fmt"
	"net/url"

	itchio "github.com/itchio/go-itchio"
)

type ItchFS struct {
	ItchServer string
}

func (ifs *ItchFS) Scheme() string {
	return "itchfs"
}

type GetURLFunc func() (string, error)

type itchfsResource struct {
	getURL GetURLFunc
}

func (ir *itchfsResource) GetURL() (string, error) {
	return ir.getURL()
}

func (ir *itchfsResource) NeedsRenewal() bool {
	// FIXME: stub
	return true
}

func (ifs *ItchFS) MakeResource(u *url.URL) (*itchfsResource, error) {
	if u.Host != "" {
		return nil, fmt.Errorf("invalid itchfs URL (must start with itchfs:///): %s", u.String())
	}

	vals := u.Query()

	apiKey := vals.Get("api_key")
	if apiKey == "" {
		return nil, fmt.Errorf("missing API key")
	}

	itchClient := itchio.ClientWithKey(apiKey)
	if ifs.ItchServer != "" {
		itchClient.SetServer(ifs.ItchServer)
	}

	source, err := ObtainSource(itchClient, u.Path)
	if err != nil {
		return nil, err
	}

	getURL, err := source.makeGetURL()
	if err != nil {
		return nil, err
	}

	return &itchfsResource{getURL}, nil
}
<commit_msg>Fix NeedRenewal<commit_after>package itchfs

import (
	"fmt"
	"net/http"
	"net/url"

	itchio "github.com/itchio/go-itchio"
)

type ItchFS struct {
	ItchServer string
}

func (ifs *ItchFS) Scheme() string {
	return "itchfs"
}

type GetURLFunc func() (string, error)

type itchfsResource struct {
	getURL GetURLFunc
}

func (ir *itchfsResource) GetURL() (string, error) {
	return ir.getURL()
}

func (ir *itchfsResource) NeedsRenewal(req *http.Request) bool {
	// FIXME: stub
	return true
}

func (ifs *ItchFS) MakeResource(u *url.URL) (*itchfsResource, error) {
	if u.Host != "" {
		return nil, fmt.Errorf("invalid itchfs URL (must start with itchfs:///): %s", u.String())
	}

	vals := u.Query()

	apiKey := vals.Get("api_key")
	if apiKey == "" {
		return nil, fmt.Errorf("missing API key")
	}

	itchClient := itchio.ClientWithKey(apiKey)
	if ifs.ItchServer != "" {
		itchClient.SetServer(ifs.ItchServer)
	}

	source, err := ObtainSource(itchClient, u.Path)
	if err != nil {
		return nil, err
	}

	getURL, err := source.makeGetURL()
	if err != nil {
		return nil, err
	}

	return &itchfsResource{getURL}, nil
}
<|endoftext|>


<commit_before>package test161

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/imdario/mergo"
	yaml "gopkg.in/yaml.v2"
	"io/ioutil"
	"math/rand"
	"strconv"
	"strings"
	"text/template"
	"time"
)

// This file has everything for creating command instances from command templates.
// In test161, a command is either a command run from the kernel menu (sy1), or
// a single userspace program (i.e. /testbin/huge).
//
// Command Templates: Some commands, (e.g. argtest, factorial), have output that
// depends on the input.  For this reason, output may be specified as a golang
// template, with various functions provided.  Furthermore, random inputs can also be
// generated using a template, which can be overriden in assignment files if needed.
//
// Command Instances: The input/expected output for an instance of a command instance
// is created by executing the templates.
//
// Composite Commands: Some commands may execute other commands, i.e. triple*.  For
// these cases, there is an "external" property of the output line, which when set to
// "true", specifies that the text property should be used to look up the output from
// another command.
//

// Functions and function map & functions for command template evaluation

func add(a int, b int) int {
	return a + b
}

func atoi(s string) (int, error) {
	return strconv.Atoi(s)
}

func randInt(min, max int) (int, error) {
	if min >= max {
		return 0, errors.New("max must be greater than min")
	}

	// between 0 and max-min
	temp := rand.Intn(max - min)

	// between min and mix
	return min + temp, nil
}

const stringChars string = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*()"

func randString(min, max int) (string, error) {
	// Get the length of the string
	l, err := randInt(min, max)
	if err != nil {
		return "", err
	}

	// Make it
	b := make([]byte, l)
	for i := 0; i < l; i++ {
		b[i] = stringChars[rand.Intn(len(stringChars))]
	}

	return string(b), nil
}

func factorial(n int) int {
	if n == 0 {
		return 1
	} else {
		return n * factorial(n-1)
	}
}

// Get something of length n that we can range over.  This is useful
// for creating random inputs.
func ranger(n int) []int {
	res := make([]int, n)
	for i := 0; i < n; i++ {
		res[i] = i
	}
	return res
}

// Functions we provide to the command templates.
var funcMap template.FuncMap = template.FuncMap{
	"add":        add,
	"atoi":       atoi,
	"factorial":  factorial,
	"randInt":    randInt,
	"randString": randString,
	"ranger":     ranger,
}

// Data that we provide for command templates.
type templateData struct {
	Args   []string
	ArgLen int
}

// Command options for panics and timesout
const (
	CMD_OPT_NO    = "no"
	CMD_OPT_MAYBE = "maybe"
	CMD_OPT_YES   = "yes"
)

// Template for commands instances.  These get expanded depending on the command environment.
type CommandTemplate struct {
	Name     string             `yaml:"name"`
	Output   []*TemplOutputLine `yaml:"output"`
	Input    []string           `yaml:"input"`
	Panic    string             `yaml:"panics"`   // CMD_OPT
	TimesOut string             `yaml:"timesout"` // CMD_OPT
	Timeout  float32            `yaml:"timeout"`  // Timeout in sec. A timeout of 0.0 uses the test default.
}

// An expected line of output, which may either be expanded or not.
type TemplOutputLine struct {
	Text     string `yaml:"text"`
	Trusted  string `yaml:"trusted"`
	External string `yaml:"external"`
}

// Command instance expected output line.  The difference here is that we store the name
// of the key that we need to verify the output.
type ExpectedOutputLine struct {
	Text    string
	Trusted bool
	KeyName string
}

// Expand the golang text template using the provided tempate data.
// We do this on a per-command instance basis, since output can change
// depending on input.
func expandLine(t string, templdata interface{}) ([]string, error) {

	res := make([]string, 0)
	bb := &bytes.Buffer{}

	if tmpl, err := template.New("CommandInstance").Funcs(funcMap).Parse(t); err != nil {
		return nil, err
	} else if tmpl.Execute(bb, templdata); err != nil {
		return nil, err
	} else {
		lines := strings.Split(bb.String(), "\n")
		for _, l := range lines {
			if strings.TrimSpace(l) != "" {
				res = append(res, l)
			}
		}
	}

	return res, nil
}

// Expand template output lines into the actual expected output. This may be
// called recursively if the output line references another command.  The
// 'processed' map takes care of checking for cycles so we don't get stuck.
func expandOutput(id string, tmpl *CommandTemplate, td *templateData,
	processed map[string]bool, env *TestEnvironment) ([]*ExpectedOutputLine, error) {

	var ok bool

	// Check for cycles
	if _, ok = processed[id]; ok {
		return nil, errors.New("Cycle detected in command template output.  ID: " + id)
	}
	processed[id] = true

	// expected output
	expected := make([]*ExpectedOutputLine, 0)

	// Expand each expected output line, possibly referencing external commands
	for _, origline := range tmpl.Output {
		if origline.External == "true" {
			copy := make(map[string]bool)
			for key, _ := range processed {
				copy[key] = true
			}
			if otherTmpl, ok := env.Commands[origline.Text]; ok {
				if more, err := expandOutput(origline.Text, otherTmpl, td, copy, env); err != nil {
					return nil, err
				} else {
					expected = append(expected, more...)
				}
			}
		} else {
			if lines, err := expandLine(origline.Text, td); err != nil {
				return nil, err
			} else {
				for _, expandedline := range lines {
					expectedline := &ExpectedOutputLine{
						Text: expandedline,
					}
					if origline.Trusted == "true" {
						expectedline.Trusted = true
						expectedline.KeyName = id
					} else {
						expectedline.Trusted = false
						expectedline.KeyName = ""
					}
					expected = append(expected, expectedline)
				}
			}
		}
	}

	return expected, nil
}

func (c *Command) Id() string {
	_, id, _ := (&c.Input).splitCommand()
	return id
}

// Instantiate the command (input, expected output) using the command template.
// This needs to be must be done prior to executing the command.
func (c *Command) Instantiate(env *TestEnvironment) error {

	pfx, id, args := (&c.Input).splitCommand()
	tmpl, ok := env.Commands[id]
	if !ok {
		return fmt.Errorf("Command '%v' is not recognized by test161.\n", id)
	}

	// Individual tests can override the template in the commands files.
	// This merges the command template on top of the overrides.
	if err := mergo.Map(&c.Config, tmpl); err != nil {
		return err
	}
	tmpl = &c.Config
	c.Panic = tmpl.Panic
	c.TimesOut = tmpl.TimesOut
	c.Timeout = tmpl.Timeout

	// Input

	// Check if  we need to create some input. If args haven't already been
	// specified, and there is an input template, use that to create input.
	if len(args) == 0 && len(tmpl.Input) > 0 {
		args = make([]string, 0)
		for _, line := range tmpl.Input {
			if temp, err := expandLine(line, "No data"); err != nil {
				return err
			} else {
				args = append(args, temp...)
			}
		}
	} else if len(args) > 0 {
		// Expand the provided arguments as well
		argLine := ""
		for i, arg := range args {
			if i > 0 {
				argLine += " "
			}
			argLine += arg
		}
		if temp, err := expandLine(argLine, "No data"); err != nil {
			return err
		} else {
			// Break
			args = []string{}
			for _, arg := range temp {
				args = append(args, splitArgs(arg)...)
			}
		}
	}

	// Output

	// template data for the output
	td := &templateData{args, len(args)}
	processed := make(map[string]bool)

	if expected, err := expandOutput(id, tmpl, td, processed, env); err != nil {
		return err
	} else {
		// Piece back together a command line for the command
		commandLine := ""

		if len(pfx) > 0 {
			commandLine += pfx + " "
		}

		commandLine += id

		for _, arg := range args {
			commandLine += " " + arg
		}

		c.Input.Line = commandLine
		c.ExpectedOutput = expected

		return nil
	}

}

// CommandTemplate Collection. We just use this for loading and move the
// references into a map in the global environment.
type CommandTemplates struct {
	Templates []*CommandTemplate `yaml:"templates"`
}

func CommandTemplatesFromFile(file string) (*CommandTemplates, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, fmt.Errorf("Error reading commands file %v: %v", file, err)
	}

	c, err := CommandTemplatesFromString(string(data))
	if err != nil {
		err = fmt.Errorf("Error loading commands file %v: %v", file, err)
	}
	return c, err
}

func CommandTemplatesFromString(text string) (*CommandTemplates, error) {
	cmds := &CommandTemplates{}
	err := yaml.Unmarshal([]byte(text), cmds)

	if err != nil {
		return nil, err
	}

	for _, t := range cmds.Templates {
		t.fixDefaults()
	}

	return cmds, nil
}

func (t *CommandTemplate) fixDefaults() {

	// Fix poor default value support in go-yaml/golang.
	//
	// If we find an empty output line, delete it - these are commands
	// that do not expect output. If we find a command with no expected
	// output, add the default expected output.

	// Default for panic is to not allow it, i.e. must return to prompt
	if t.Panic != CMD_OPT_MAYBE && t.Panic != CMD_OPT_YES {
		t.Panic = CMD_OPT_NO
	}

	if t.TimesOut != CMD_OPT_MAYBE && t.TimesOut != CMD_OPT_YES {
		t.TimesOut = CMD_OPT_NO
	}

	if len(t.Output) == 1 && strings.TrimSpace(t.Output[0].Text) == "" {
		t.Output = nil
	} else if len(t.Output) == 0 {
		t.Output = []*TemplOutputLine{
			&TemplOutputLine{
				Trusted:  "true",
				External: "false",
				Text:     t.Name + ": SUCCESS",
			},
		}
	} else {
		for _, line := range t.Output {
			if line.Trusted != "false" {
				line.Trusted = "true"
			}

			if line.External != "true" {
				line.External = "false"
			}
		}
	}
}

// Seed random for the random input templates and sys161
func init() {
	rand.Seed(time.Now().UnixNano())
}
<commit_msg>Fix failing test after mergo dependency update.<commit_after>package test161

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/imdario/mergo"
	yaml "gopkg.in/yaml.v2"
	"io/ioutil"
	"math/rand"
	"strconv"
	"strings"
	"text/template"
	"time"
)

// This file has everything for creating command instances from command templates.
// In test161, a command is either a command run from the kernel menu (sy1), or
// a single userspace program (i.e. /testbin/huge).
//
// Command Templates: Some commands, (e.g. argtest, factorial), have output that
// depends on the input.  For this reason, output may be specified as a golang
// template, with various functions provided.  Furthermore, random inputs can also be
// generated using a template, which can be overriden in assignment files if needed.
//
// Command Instances: The input/expected output for an instance of a command instance
// is created by executing the templates.
//
// Composite Commands: Some commands may execute other commands, i.e. triple*.  For
// these cases, there is an "external" property of the output line, which when set to
// "true", specifies that the text property should be used to look up the output from
// another command.
//

// Functions and function map & functions for command template evaluation

func add(a int, b int) int {
	return a + b
}

func atoi(s string) (int, error) {
	return strconv.Atoi(s)
}

func randInt(min, max int) (int, error) {
	if min >= max {
		return 0, errors.New("max must be greater than min")
	}

	// between 0 and max-min
	temp := rand.Intn(max - min)

	// between min and mix
	return min + temp, nil
}

const stringChars string = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*()"

func randString(min, max int) (string, error) {
	// Get the length of the string
	l, err := randInt(min, max)
	if err != nil {
		return "", err
	}

	// Make it
	b := make([]byte, l)
	for i := 0; i < l; i++ {
		b[i] = stringChars[rand.Intn(len(stringChars))]
	}

	return string(b), nil
}

func factorial(n int) int {
	if n == 0 {
		return 1
	} else {
		return n * factorial(n-1)
	}
}

// Get something of length n that we can range over.  This is useful
// for creating random inputs.
func ranger(n int) []int {
	res := make([]int, n)
	for i := 0; i < n; i++ {
		res[i] = i
	}
	return res
}

// Functions we provide to the command templates.
var funcMap template.FuncMap = template.FuncMap{
	"add":        add,
	"atoi":       atoi,
	"factorial":  factorial,
	"randInt":    randInt,
	"randString": randString,
	"ranger":     ranger,
}

// Data that we provide for command templates.
type templateData struct {
	Args   []string
	ArgLen int
}

// Command options for panics and timesout
const (
	CMD_OPT_NO    = "no"
	CMD_OPT_MAYBE = "maybe"
	CMD_OPT_YES   = "yes"
)

// Template for commands instances.  These get expanded depending on the command environment.
type CommandTemplate struct {
	Name     string             `yaml:"name"`
	Output   []*TemplOutputLine `yaml:"output"`
	Input    []string           `yaml:"input"`
	Panic    string             `yaml:"panics"`   // CMD_OPT
	TimesOut string             `yaml:"timesout"` // CMD_OPT
	Timeout  float32            `yaml:"timeout"`  // Timeout in sec. A timeout of 0.0 uses the test default.
}

// An expected line of output, which may either be expanded or not.
type TemplOutputLine struct {
	Text     string `yaml:"text"`
	Trusted  string `yaml:"trusted"`
	External string `yaml:"external"`
}

// Command instance expected output line.  The difference here is that we store the name
// of the key that we need to verify the output.
type ExpectedOutputLine struct {
	Text    string
	Trusted bool
	KeyName string
}

func (ct *CommandTemplate) Clone() *CommandTemplate {
	clone := *ct
	clone.Output = make([]*TemplOutputLine, 0, len(ct.Output))
	clone.Input = make([]string, 0, len(ct.Input))

	for _, o := range ct.Output {
		copy := *o
		clone.Output = append(clone.Output, &copy)
	}

	for _, s := range ct.Input {
		clone.Input = append(clone.Input, s)
	}

	return &clone
}

// Expand the golang text template using the provided tempate data.
// We do this on a per-command instance basis, since output can change
// depending on input.
func expandLine(t string, templdata interface{}) ([]string, error) {

	res := make([]string, 0)
	bb := &bytes.Buffer{}

	if tmpl, err := template.New("CommandInstance").Funcs(funcMap).Parse(t); err != nil {
		return nil, err
	} else if tmpl.Execute(bb, templdata); err != nil {
		return nil, err
	} else {
		lines := strings.Split(bb.String(), "\n")
		for _, l := range lines {
			if strings.TrimSpace(l) != "" {
				res = append(res, l)
			}
		}
	}

	return res, nil
}

// Expand template output lines into the actual expected output. This may be
// called recursively if the output line references another command.  The
// 'processed' map takes care of checking for cycles so we don't get stuck.
func expandOutput(id string, tmpl *CommandTemplate, td *templateData,
	processed map[string]bool, env *TestEnvironment) ([]*ExpectedOutputLine, error) {

	var ok bool

	// Check for cycles
	if _, ok = processed[id]; ok {
		return nil, errors.New("Cycle detected in command template output.  ID: " + id)
	}
	processed[id] = true

	// expected output
	expected := make([]*ExpectedOutputLine, 0)

	// Expand each expected output line, possibly referencing external commands
	for _, origline := range tmpl.Output {
		if origline.External == "true" {
			copy := make(map[string]bool)
			for key, _ := range processed {
				copy[key] = true
			}
			if otherTmpl, ok := env.Commands[origline.Text]; ok {
				if more, err := expandOutput(origline.Text, otherTmpl, td, copy, env); err != nil {
					return nil, err
				} else {
					expected = append(expected, more...)
				}
			}
		} else {
			if lines, err := expandLine(origline.Text, td); err != nil {
				return nil, err
			} else {
				for _, expandedline := range lines {
					expectedline := &ExpectedOutputLine{
						Text: expandedline,
					}
					if origline.Trusted == "true" {
						expectedline.Trusted = true
						expectedline.KeyName = id
					} else {
						expectedline.Trusted = false
						expectedline.KeyName = ""
					}
					expected = append(expected, expectedline)
				}
			}
		}
	}

	return expected, nil
}

func (c *Command) Id() string {
	_, id, _ := (&c.Input).splitCommand()
	return id
}

// Instantiate the command (input, expected output) using the command template.
// This needs to be must be done prior to executing the command.
func (c *Command) Instantiate(env *TestEnvironment) error {

	pfx, id, args := (&c.Input).splitCommand()
	tmpl, ok := env.Commands[id]
	if !ok {
		return fmt.Errorf("Command '%v' is not recognized by test161.\n", id)
	}

	// Individual tests can override the template in the commands files.
	// This merges the overrides on top of a copy of the command template.
	tmpl = tmpl.Clone()
	if err := mergo.Merge(tmpl, &c.Config, mergo.WithOverride); err != nil {
		return err
	}
	c.Panic = tmpl.Panic
	c.TimesOut = tmpl.TimesOut
	c.Timeout = tmpl.Timeout

	// Input

	// Check if  we need to create some input. If args haven't already been
	// specified, and there is an input template, use that to create input.
	if len(args) == 0 && len(tmpl.Input) > 0 {
		args = make([]string, 0)
		for _, line := range tmpl.Input {
			if temp, err := expandLine(line, "No data"); err != nil {
				return err
			} else {
				args = append(args, temp...)
			}
		}
	} else if len(args) > 0 {
		// Expand the provided arguments as well
		argLine := ""
		for i, arg := range args {
			if i > 0 {
				argLine += " "
			}
			argLine += arg
		}
		if temp, err := expandLine(argLine, "No data"); err != nil {
			return err
		} else {
			// Break
			args = []string{}
			for _, arg := range temp {
				args = append(args, splitArgs(arg)...)
			}
		}
	}

	// Output

	// template data for the output
	td := &templateData{args, len(args)}
	processed := make(map[string]bool)

	if expected, err := expandOutput(id, tmpl, td, processed, env); err != nil {
		return err
	} else {
		// Piece back together a command line for the command
		commandLine := ""

		if len(pfx) > 0 {
			commandLine += pfx + " "
		}

		commandLine += id

		for _, arg := range args {
			commandLine += " " + arg
		}

		c.Input.Line = commandLine
		c.ExpectedOutput = expected

		return nil
	}

}

// CommandTemplate Collection. We just use this for loading and move the
// references into a map in the global environment.
type CommandTemplates struct {
	Templates []*CommandTemplate `yaml:"templates"`
}

func CommandTemplatesFromFile(file string) (*CommandTemplates, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, fmt.Errorf("Error reading commands file %v: %v", file, err)
	}

	c, err := CommandTemplatesFromString(string(data))
	if err != nil {
		err = fmt.Errorf("Error loading commands file %v: %v", file, err)
	}
	return c, err
}

func CommandTemplatesFromString(text string) (*CommandTemplates, error) {
	cmds := &CommandTemplates{}
	err := yaml.Unmarshal([]byte(text), cmds)

	if err != nil {
		return nil, err
	}

	for _, t := range cmds.Templates {
		t.fixDefaults()
	}

	return cmds, nil
}

func (t *CommandTemplate) fixDefaults() {

	// Fix poor default value support in go-yaml/golang.
	//
	// If we find an empty output line, delete it - these are commands
	// that do not expect output. If we find a command with no expected
	// output, add the default expected output.

	// Default for panic is to not allow it, i.e. must return to prompt
	if t.Panic != CMD_OPT_MAYBE && t.Panic != CMD_OPT_YES {
		t.Panic = CMD_OPT_NO
	}

	if t.TimesOut != CMD_OPT_MAYBE && t.TimesOut != CMD_OPT_YES {
		t.TimesOut = CMD_OPT_NO
	}

	if len(t.Output) == 1 && strings.TrimSpace(t.Output[0].Text) == "" {
		t.Output = nil
	} else if len(t.Output) == 0 {
		t.Output = []*TemplOutputLine{
			&TemplOutputLine{
				Trusted:  "true",
				External: "false",
				Text:     t.Name + ": SUCCESS",
			},
		}
	} else {
		for _, line := range t.Output {
			if line.Trusted != "false" {
				line.Trusted = "true"
			}

			if line.External != "true" {
				line.External = "false"
			}
		}
	}
}

// Seed random for the random input templates and sys161
func init() {
	rand.Seed(time.Now().UnixNano())
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/buildbox/agent/buildbox"
	"github.com/buildbox/agent/command"
	"github.com/codegangsta/cli"
)

var Commands []cli.Command

var AgentDescription = `Usage:

   buildbox-agent start [arguments...]

Description:

   When a job is ready to run it will call the "bootstrap-script"
   and pass it all the environment variables required for the job to run.
   This script is responsible for checking out the code, and running the
   actual build script defined in the project.

   The agent will run any jobs within a PTY (pseudo terminal) if available.

Example:

   $ buildbox-agent start --token xxx`

var DownloadHelpDescription = `Usage:

   buildbox-agent build-artifact download [arguments...]

Description:

   Downloads artifacts from Buildbox to the local machine.

   You need to ensure that your search query is surrounded by quotes if
   using a wild card as the built-in shell path globbing will provide files,
   which will break the download.

Example:

   $ buildbox-agent build-artifact download "pkg/*.tar.gz" . --build xxx

   This will search across all the artifacts for the build with files that match that part.
   The first argument is the search query, and the second argument is the download destination.

   If you're trying to download a specific file, and there are multiple artifacts from different
   jobs, you can target the paticular job you want to download the artifact from:

   $ buildbox-agent build-artifact download "pkg/*.tar.gz" . --job "tests" --build xxx

   You can also use the job's id (provided by the environment variable $BUILDBOX_JOB_ID)`

var UploadHelpDescription = `Usage:

   buildbox-agent build-artifact upload <pattern> <destination> [arguments...]

Description:

   Uploads files to a job as artifacts.

   You need to ensure that the paths are surrounded by quotes otherwise the
   built-in shell path globbing will provide the files, which is currently not
   supported.

Example:

   $ buildbox-agent build-artifact upload "log/**/*.log"

   You can also upload directy to Amazon S3 if you'd like to host your own artifacts:

   $ export AWS_SECRET_ACCESS_KEY=yyy
   $ export AWS_ACCESS_KEY_ID=xxx
   $ buildbox-agent build-artifact upload "log/**/*.log" s3://name-of-your-s3-bucket/$BUILDBOX_JOB_ID`

var SetHelpDescription = `Usage:

   buildbox-agent build-data set <key> <value> [arguments...]

Description:

   Set arbitrary data on a build using a basic key/value store.

Example:

   $ buildbox-agent build-data set "foo" "bar"`

var GetHelpDescription = `Usage:

   buildbox-agent build-data get <key> [arguments...]

Description:

   Get data from a builds key/value store.

Example:

   $ buildbox-agent build-data get "foo"`

func init() {
	// This is default location of the bootstrap for unix based operating
	// systems.
	bootstrapScriptLocation := "$HOME/.buildbox/bootstrap.sh"

	// Windows has a slightly modified default bootstrap location
	if buildbox.MachineIsWindows() {
		bootstrapScriptLocation = "$USERPROFILE\\AppData\\Local\\BuildboxAgent\\bootstrap.bat"
	}

	Commands = []cli.Command{
		{
			Name:        "start",
			Usage:       "Starts a Buildbox agent",
			Description: AgentDescription,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:   "token",
					Value:  "",
					Usage:  "Your account agent token",
					EnvVar: "BUILDBOX_AGENT_TOKEN",
				},
				cli.StringFlag{
					Name:   "access-token",
					Value:  "",
					Usage:  "DEPRECATED: The agents access token",
					EnvVar: "BUILDBOX_AGENT_ACCESS_TOKEN",
				},
				cli.StringFlag{
					Name:   "name",
					Value:  "",
					Usage:  "The name of the agent",
					EnvVar: "BUILDBOX_AGENT_NAME",
				},
				cli.StringFlag{
					Name:   "priority",
					Value:  "",
					Usage:  "The priority of the agent",
					EnvVar: "BUILDBOX_AGENT_PRIORITY",
				},
				cli.StringSliceFlag{
					Name:   "meta-data",
					Value:  &cli.StringSlice{},
					Usage:  "Meta data for the agent",
					EnvVar: "BUILDBOX_AGENT_META_DATA",
				},
				cli.StringFlag{
					Name:   "bootstrap-script",
					Value:  bootstrapScriptLocation,
					Usage:  "Path to the bootstrap script",
					EnvVar: "BUILDBOX_BOOTSTRAP_SCRIPT_PATH",
				},
				cli.StringFlag{
					Name:   "endpoint",
					Value:  "https://agent.buildbox.io/v2",
					Usage:  "The agent API endpoint",
					EnvVar: "BUILDBOX_AGENT_ENDPOINT",
				},
				cli.BoolFlag{
					Name:  "meta-data-ec2-tags",
					Usage: "Populate the meta data from the current instances EC2 Tags",
				},
				cli.BoolFlag{
					Name:  "no-pty",
					Usage: "Do not run jobs within a pseudo terminal",
				},
				cli.BoolFlag{
					Name:   "debug",
					Usage:  "Enable debug mode.",
					EnvVar: "BUILDBOX_AGENT_DEBUG",
				},
			},
			Action: command.AgentStartCommandAction,
		},
		{
			Name:  "build-artifact",
			Usage: "Upload/download artifacts from Buildbox jobs",
			Subcommands: []cli.Command{
				{
					Name:        "download",
					Usage:       "Downloads artifacts from Buildbox to the local machine.",
					Description: DownloadHelpDescription,
					Flags: []cli.Flag{
						cli.StringFlag{
							Name:   "job",
							Value:  "",
							Usage:  "Which job should the artifacts be downloaded from",
							EnvVar: "BUILDBOX_JOB_ID",
						},
						cli.StringFlag{
							Name:   "build",
							Value:  "",
							Usage:  "Which build should the artifacts be downloaded from",
							EnvVar: "BUILDBOX_BUILD_ID",
						},
						cli.StringFlag{
							Name:   "agent-access-token",
							Value:  "",
							Usage:  "The access token used to identify the agent",
							EnvVar: "BUILDBOX_AGENT_ACCESS_TOKEN",
						},
						cli.StringFlag{
							Name:   "endpoint",
							Value:  "https://agent.buildbox.io/v2",
							Usage:  "The agent API endpoint",
							EnvVar: "BUILDBOX_AGENT_ENDPOINT",
						},
						cli.BoolFlag{
							Name:   "debug",
							Usage:  "Enable debug mode",
							EnvVar: "BUILDBOX_AGENT_DEBUG",
						},
					},
					Action: command.ArtifactDownloadCommandAction,
				},
				{
					Name:        "upload",
					Usage:       "Uploads files to a job as artifacts.",
					Description: UploadHelpDescription,
					Flags: []cli.Flag{
						cli.StringFlag{
							Name:   "job",
							Value:  "",
							Usage:  "Which job should the artifacts be downloaded from",
							EnvVar: "BUILDBOX_JOB_ID",
						},
						cli.StringFlag{
							Name:   "agent-access-token",
							Value:  "",
							Usage:  "The access token used to identify the agent",
							EnvVar: "BUILDBOX_AGENT_ACCESS_TOKEN",
						},
						cli.StringFlag{
							Name:   "endpoint",
							Value:  "https://agent.buildbox.io/v2",
							Usage:  "The agent API endpoint",
							EnvVar: "BUILDBOX_AGENT_ENDPOINT",
						},
						cli.BoolFlag{
							Name:   "debug",
							Usage:  "Enable debug mode",
							EnvVar: "BUILDBOX_AGENT_DEBUG",
						},
					},
					Action: command.ArtifactUploadCommandAction,
				},
			},
		},
		{
			Name:  "build-data",
			Usage: "Get/set data from Buildbox jobs",
			Subcommands: []cli.Command{
				{
					Name:        "set",
					Usage:       "Set data on a build",
					Description: SetHelpDescription,
					Flags: []cli.Flag{
						cli.StringFlag{
							Name:   "job",
							Value:  "",
							Usage:  "Which job should the artifacts be downloaded from",
							EnvVar: "BUILDBOX_JOB_ID",
						},
						cli.StringFlag{
							Name:   "agent-access-token",
							Value:  "",
							Usage:  "The access token used to identify the agent",
							EnvVar: "BUILDBOX_AGENT_ACCESS_TOKEN",
						},
						cli.StringFlag{
							Name:   "endpoint",
							Value:  "https://agent.buildbox.io/v2",
							Usage:  "The agent API endpoint",
							EnvVar: "BUILDBOX_AGENT_ENDPOINT",
						},
						cli.BoolFlag{
							Name:   "debug",
							Usage:  "Enable debug mode",
							EnvVar: "BUILDBOX_AGENT_DEBUG",
						},
					},
					Action: command.DataSetCommandAction,
				},
				{
					Name:        "get",
					Usage:       "Get data from a build",
					Description: GetHelpDescription,
					Flags: []cli.Flag{
						cli.StringFlag{
							Name:   "job",
							Value:  "",
							Usage:  "Which job should the artifacts be downloaded from",
							EnvVar: "BUILDBOX_JOB_ID",
						},
						cli.StringFlag{
							Name:   "agent-access-token",
							Value:  "",
							Usage:  "The access token used to identify the agent",
							EnvVar: "BUILDBOX_AGENT_ACCESS_TOKEN",
						},
						cli.StringFlag{
							Name:   "endpoint",
							Value:  "https://agent.buildbox.io/v2",
							Usage:  "The agent API endpoint",
							EnvVar: "BUILDBOX_AGENT_ENDPOINT",
						},
						cli.BoolFlag{
							Name:   "debug",
							Usage:  "Enable debug mode",
							EnvVar: "BUILDBOX_AGENT_DEBUG",
						},
					},
					Action: command.DataGetCommandAction,
				},
			},
		},
	}
}
<commit_msg>Don't default `build-artifact download --job` to $BUILDBOX_JOB_ID<commit_after>package main

import (
	"github.com/buildbox/agent/buildbox"
	"github.com/buildbox/agent/command"
	"github.com/codegangsta/cli"
)

var Commands []cli.Command

var AgentDescription = `Usage:

   buildbox-agent start [arguments...]

Description:

   When a job is ready to run it will call the "bootstrap-script"
   and pass it all the environment variables required for the job to run.
   This script is responsible for checking out the code, and running the
   actual build script defined in the project.

   The agent will run any jobs within a PTY (pseudo terminal) if available.

Example:

   $ buildbox-agent start --token xxx`

var DownloadHelpDescription = `Usage:

   buildbox-agent build-artifact download [arguments...]

Description:

   Downloads artifacts from Buildbox to the local machine.

   You need to ensure that your search query is surrounded by quotes if
   using a wild card as the built-in shell path globbing will provide files,
   which will break the download.

Example:

   $ buildbox-agent build-artifact download "pkg/*.tar.gz" . --build xxx

   This will search across all the artifacts for the build with files that match that part.
   The first argument is the search query, and the second argument is the download destination.

   If you're trying to download a specific file, and there are multiple artifacts from different
   jobs, you can target the paticular job you want to download the artifact from:

   $ buildbox-agent build-artifact download "pkg/*.tar.gz" . --job "tests" --build xxx

   You can also use the job's id (provided by the environment variable $BUILDBOX_JOB_ID)`

var UploadHelpDescription = `Usage:

   buildbox-agent build-artifact upload <pattern> <destination> [arguments...]

Description:

   Uploads files to a job as artifacts.

   You need to ensure that the paths are surrounded by quotes otherwise the
   built-in shell path globbing will provide the files, which is currently not
   supported.

Example:

   $ buildbox-agent build-artifact upload "log/**/*.log"

   You can also upload directy to Amazon S3 if you'd like to host your own artifacts:

   $ export AWS_SECRET_ACCESS_KEY=yyy
   $ export AWS_ACCESS_KEY_ID=xxx
   $ buildbox-agent build-artifact upload "log/**/*.log" s3://name-of-your-s3-bucket/$BUILDBOX_JOB_ID`

var SetHelpDescription = `Usage:

   buildbox-agent build-data set <key> <value> [arguments...]

Description:

   Set arbitrary data on a build using a basic key/value store.

Example:

   $ buildbox-agent build-data set "foo" "bar"`

var GetHelpDescription = `Usage:

   buildbox-agent build-data get <key> [arguments...]

Description:

   Get data from a builds key/value store.

Example:

   $ buildbox-agent build-data get "foo"`

func init() {
	// This is default location of the bootstrap for unix based operating
	// systems.
	bootstrapScriptLocation := "$HOME/.buildbox/bootstrap.sh"

	// Windows has a slightly modified default bootstrap location
	if buildbox.MachineIsWindows() {
		bootstrapScriptLocation = "$USERPROFILE\\AppData\\Local\\BuildboxAgent\\bootstrap.bat"
	}

	Commands = []cli.Command{
		{
			Name:        "start",
			Usage:       "Starts a Buildbox agent",
			Description: AgentDescription,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:   "token",
					Value:  "",
					Usage:  "Your account agent token",
					EnvVar: "BUILDBOX_AGENT_TOKEN",
				},
				cli.StringFlag{
					Name:   "access-token",
					Value:  "",
					Usage:  "DEPRECATED: The agents access token",
					EnvVar: "BUILDBOX_AGENT_ACCESS_TOKEN",
				},
				cli.StringFlag{
					Name:   "name",
					Value:  "",
					Usage:  "The name of the agent",
					EnvVar: "BUILDBOX_AGENT_NAME",
				},
				cli.StringFlag{
					Name:   "priority",
					Value:  "",
					Usage:  "The priority of the agent",
					EnvVar: "BUILDBOX_AGENT_PRIORITY",
				},
				cli.StringSliceFlag{
					Name:   "meta-data",
					Value:  &cli.StringSlice{},
					Usage:  "Meta data for the agent",
					EnvVar: "BUILDBOX_AGENT_META_DATA",
				},
				cli.StringFlag{
					Name:   "bootstrap-script",
					Value:  bootstrapScriptLocation,
					Usage:  "Path to the bootstrap script",
					EnvVar: "BUILDBOX_BOOTSTRAP_SCRIPT_PATH",
				},
				cli.StringFlag{
					Name:   "endpoint",
					Value:  "https://agent.buildbox.io/v2",
					Usage:  "The agent API endpoint",
					EnvVar: "BUILDBOX_AGENT_ENDPOINT",
				},
				cli.BoolFlag{
					Name:  "meta-data-ec2-tags",
					Usage: "Populate the meta data from the current instances EC2 Tags",
				},
				cli.BoolFlag{
					Name:  "no-pty",
					Usage: "Do not run jobs within a pseudo terminal",
				},
				cli.BoolFlag{
					Name:   "debug",
					Usage:  "Enable debug mode.",
					EnvVar: "BUILDBOX_AGENT_DEBUG",
				},
			},
			Action: command.AgentStartCommandAction,
		},
		{
			Name:  "build-artifact",
			Usage: "Upload/download artifacts from Buildbox jobs",
			Subcommands: []cli.Command{
				{
					Name:        "download",
					Usage:       "Downloads artifacts from Buildbox to the local machine.",
					Description: DownloadHelpDescription,
					Flags: []cli.Flag{
						// We don't default to $BUILDBOX_JOB_ID with --job because downloading artifacts should
						// default to all the jobs on the build, not just the current one. --job is used
						// to scope to a paticular job if you
						cli.StringFlag{
							Name:  "job",
							Value: "",
							Usage: "Used to target a specific job to download artifacts from",
						},
						cli.StringFlag{
							Name:   "build",
							Value:  "",
							Usage:  "Which build should the artifacts be downloaded from",
							EnvVar: "BUILDBOX_BUILD_ID",
						},
						cli.StringFlag{
							Name:   "agent-access-token",
							Value:  "",
							Usage:  "The access token used to identify the agent",
							EnvVar: "BUILDBOX_AGENT_ACCESS_TOKEN",
						},
						cli.StringFlag{
							Name:   "endpoint",
							Value:  "https://agent.buildbox.io/v2",
							Usage:  "The agent API endpoint",
							EnvVar: "BUILDBOX_AGENT_ENDPOINT",
						},
						cli.BoolFlag{
							Name:   "debug",
							Usage:  "Enable debug mode",
							EnvVar: "BUILDBOX_AGENT_DEBUG",
						},
					},
					Action: command.ArtifactDownloadCommandAction,
				},
				{
					Name:        "upload",
					Usage:       "Uploads files to a job as artifacts.",
					Description: UploadHelpDescription,
					Flags: []cli.Flag{
						cli.StringFlag{
							Name:   "job",
							Value:  "",
							Usage:  "Which job should the artifacts be downloaded from",
							EnvVar: "BUILDBOX_JOB_ID",
						},
						cli.StringFlag{
							Name:   "agent-access-token",
							Value:  "",
							Usage:  "The access token used to identify the agent",
							EnvVar: "BUILDBOX_AGENT_ACCESS_TOKEN",
						},
						cli.StringFlag{
							Name:   "endpoint",
							Value:  "https://agent.buildbox.io/v2",
							Usage:  "The agent API endpoint",
							EnvVar: "BUILDBOX_AGENT_ENDPOINT",
						},
						cli.BoolFlag{
							Name:   "debug",
							Usage:  "Enable debug mode",
							EnvVar: "BUILDBOX_AGENT_DEBUG",
						},
					},
					Action: command.ArtifactUploadCommandAction,
				},
			},
		},
		{
			Name:  "build-data",
			Usage: "Get/set data from Buildbox jobs",
			Subcommands: []cli.Command{
				{
					Name:        "set",
					Usage:       "Set data on a build",
					Description: SetHelpDescription,
					Flags: []cli.Flag{
						cli.StringFlag{
							Name:   "job",
							Value:  "",
							Usage:  "Which job should the artifacts be downloaded from",
							EnvVar: "BUILDBOX_JOB_ID",
						},
						cli.StringFlag{
							Name:   "agent-access-token",
							Value:  "",
							Usage:  "The access token used to identify the agent",
							EnvVar: "BUILDBOX_AGENT_ACCESS_TOKEN",
						},
						cli.StringFlag{
							Name:   "endpoint",
							Value:  "https://agent.buildbox.io/v2",
							Usage:  "The agent API endpoint",
							EnvVar: "BUILDBOX_AGENT_ENDPOINT",
						},
						cli.BoolFlag{
							Name:   "debug",
							Usage:  "Enable debug mode",
							EnvVar: "BUILDBOX_AGENT_DEBUG",
						},
					},
					Action: command.DataSetCommandAction,
				},
				{
					Name:        "get",
					Usage:       "Get data from a build",
					Description: GetHelpDescription,
					Flags: []cli.Flag{
						cli.StringFlag{
							Name:   "job",
							Value:  "",
							Usage:  "Which job should the data be retrieved from",
							EnvVar: "BUILDBOX_JOB_ID",
						},
						cli.StringFlag{
							Name:   "agent-access-token",
							Value:  "",
							Usage:  "The access token used to identify the agent",
							EnvVar: "BUILDBOX_AGENT_ACCESS_TOKEN",
						},
						cli.StringFlag{
							Name:   "endpoint",
							Value:  "https://agent.buildbox.io/v2",
							Usage:  "The agent API endpoint",
							EnvVar: "BUILDBOX_AGENT_ENDPOINT",
						},
						cli.BoolFlag{
							Name:   "debug",
							Usage:  "Enable debug mode",
							EnvVar: "BUILDBOX_AGENT_DEBUG",
						},
					},
					Action: command.DataGetCommandAction,
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>// Package aws provides core functionality for making requests to AWS services.
package aws

// SDKName is the name of this AWS SDK
const SDKName = "aws-sdk-go"

// SDKVersion is the version of this SDK
const SDKVersion = "1.0.8"
<commit_msg>Tag release v1.0.9<commit_after>// Package aws provides core functionality for making requests to AWS services.
package aws

// SDKName is the name of this AWS SDK
const SDKName = "aws-sdk-go"

// SDKVersion is the version of this SDK
const SDKVersion = "1.0.9"
<|endoftext|>


<commit_before>// Package aws provides core functionality for making requests to AWS services.
package aws

// SDKName is the name of this AWS SDK
const SDKName = "aws-sdk-go"

// SDKVersion is the version of this SDK
const SDKVersion = "1.0.9"
<commit_msg>Tag release v1.0.10<commit_after>// Package aws provides core functionality for making requests to AWS services.
package aws

// SDKName is the name of this AWS SDK
const SDKName = "aws-sdk-go"

// SDKVersion is the version of this SDK
const SDKVersion = "1.0.10"
<|endoftext|>


<commit_before>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package index

// subColFlds is a Collection wrapper which overrides the Field
// method.
type subColFlds struct {
	Collection
	flds map[string]interface{}
}

func (scf subColFlds) Field(f string) interface{} {
	if x, ok := scf.flds[f]; ok {
		return x
	}
	return scf.Collection.Field(f)
}

// subGrpFlds is a Group wrapper which overrids the Field method
type subGrpFlds struct {
	Group
	tracks []Track
	flds   map[string]interface{}
}

func (sgf subGrpFlds) Tracks() []Track {
	if sgf.tracks != nil {
		return sgf.tracks
	}
	return sgf.Group.Tracks()
}

// Field implements Group.
func (sgf subGrpFlds) Field(field string) interface{} {
	if x, ok := sgf.flds[field]; ok {
		return x
	}
	return sgf.Group.Field(field)
}

// SumGroupIntAttr recurses through the Group and assigns the field with the sum
// of fields from children (Groups or Tracks).
func SumGroupIntAttr(field string, g Group) Group {
	if c, ok := g.(Collection); ok {
		return sumCollectionIntAttr(field, c)
	}
	return sumGroupIntAttr(field, g)
}

func sumCollectionIntAttr(field string, c Collection) Collection {
	nc := subCol{
		Collection: c,
		grps:       make(map[Key]Group, len(c.Keys())),
		flds:       make(map[string]interface{}),
	}
	var total int
	for _, k := range c.Keys() {
		g := c.Get(k)
		g = SumGroupIntAttr(field, g)
		total += g.Field(field).(int)
		nc.grps[k] = g
	}
	nc.flds = map[string]interface{}{
		field: total,
	}
	return nc
}

func sumGroupIntAttr(field string, g Group) Group {
	ng := subGrpFlds{
		Group: g,
		flds:  map[string]interface{}{},
	}
	var total int
	for _, t := range g.Tracks() {
		total += t.GetInt(field)
	}
	ng.flds[field] = total
	return ng
}

// CommonGroupAttr recurses through the Group and assigns fields on all sub groups
// which are common amoungst their children (Groups or Tracks).  If there is no common
// field, then the associated Field value is not set.
func CommonGroupAttr(attrs []Attr, g Group) Group {
	if c, ok := g.(Collection); ok {
		return commonCollectionTrackAttr(attrs, c)
	}
	return commonGroupTrackAttr(attrs, g)
}

func commonCollectionTrackAttr(attrs []Attr, c Collection) Collection {
	grps := make(map[Key]Group, len(c.Keys()))
	flds := make(map[string]interface{}, len(attrs))

	keys := c.Keys()
	if len(keys) > 0 {
		k0 := keys[0]
		g0 := c.Get(k0)
		g0 = CommonGroupAttr(attrs, g0)
		grps[k0] = g0

		for _, a := range attrs {
			flds[a.Field()] = g0.Field(a.Field())
		}

		if len(keys) > 1 {
			for _, k := range keys[1:] {
				g1 := c.Get(k)
				g1 = CommonGroupAttr(attrs, g1)
				grps[k] = g1

				for _, a := range attrs {
					f := a.Field()
					v := g1.Field(f)
					if !a.eq(flds[f], v) {
						flds[f] = a.Empty()
					}
				}
			}
		}
	}

	for _, a := range attrs {
		f := a.Field()
		if v, ok := flds[f]; ok && v == a.Empty() {
			delete(flds, f)
		}
	}

	return subCol{
		Collection: c,
		grps:       grps,
		flds:       flds,
	}
}

func commonGroupTrackAttr(attrs []Attr, g Group) Group {
	flds := make(map[string]interface{}, len(attrs))
	tracks := g.Tracks()

	if len(tracks) > 0 {
		t0 := tracks[0]
		for _, a := range attrs {
			f := a.Field()
			flds[f] = a.fn(t0)
		}

		if len(tracks) > 1 {
			for _, t := range tracks[1:] {
				for _, a := range attrs {
					f := a.Field()
					if !a.eq(flds[f], a.fn(t)) {
						flds[f] = a.Empty()
					}
				}
			}
		}
	}

	for _, a := range attrs {
		f := a.Field()
		if v, ok := flds[f]; ok && a.eq(v, a.Empty()) {
			delete(flds, f)
		}
	}

	return subGrpFlds{
		Group: g,
		flds:  flds,
	}
}

// subGrpName is a Group wrapper which overrides Name.
type subGrpName struct {
	Group
	name string
}

// Name implements Group.
func (s subGrpName) Name() string {
	return s.name
}

// RemoveEmptyCollections recursively goes through each sub Collection contained
// in the Group and removes any which don't have any tracks/groups in them.
func RemoveEmptyCollections(g Group) Group {
	gc, ok := g.(Collection)
	if ok {
		keys := gc.Keys()
		if len(keys) == 1 {
			gc0 := gc.Get(keys[0])
			_, col := gc0.(Collection)
			if !col && gc0.Name() == "" {
				return subGrpName{
					name:  gc.Name(),
					Group: gc0,
				}
			}
		}
		ngc := subCol{
			Collection: gc,
			grps:       make(map[Key]Group, len(gc.Keys())),
		}
		for _, k := range keys {
			ngc.grps[k] = RemoveEmptyCollections(gc.Get(k))
		}
		return ngc
	}
	return g
}

func firstTrack(g Group) Track {
	c, ok := g.(Collection)
	if ok {
		keys := c.Keys()
		if len(keys) > 0 {
			return firstTrack(c.Get(keys[0]))
		}
		return nil
	}

	ts := g.Tracks()
	if len(ts) > 0 {
		return ts[0]
	}
	return nil
}

func fieldsGroup(m map[string]interface{}, g Group) Group {
	if c, ok := g.(Collection); ok {
		return subColFlds{
			Collection: c,
			flds:       m,
		}
	}
	return subGrpFlds{
		Group: g,
		flds:  m,
	}
}

// Attr is a type which wraps a closure to get an attribute from an implementation of the
// Attr interface.
type Attr struct {
	field string
	empty interface{}
	eq    func(x, y interface{}) bool
	fn    func(t Track) interface{}
}

// Field returns the underlying field name.
func (g Attr) Field() string {
	return g.field
}

// Empty returns the empty value of the underlying field (the empty value of the field type).
func (g Attr) Empty() interface{} {
	return g.empty
}

// StringAttr constructs an Attr which will retrieve the string field from an implementation
// of Track.
func StringAttr(field string) Attr {
	return Attr{
		field: field,
		empty: "",
		eq:    func(x, y interface{}) bool { return x == y },
		fn:    func(t Track) interface{} { return t.GetString(field) },
	}
}

// StringsAttr returns an Attr which will retrieve the strings field from an implementation of Track.
func StringsAttr(field string) Attr {
	return Attr{
		field: field,
		empty: nil,
		eq: func(x, y interface{}) bool {
			xs := x.([]string) // NB: panics here are acceptable: should not be called on a non-'Strings' field.
			ys := y.([]string)
			if len(xs) != len(ys) {
				return false
			}
			for i, xss := range xs {
				if ys[i] != xss {
					return false
				}
			}
			return true
		},
		fn: func(t Track) interface{} {
			return t.GetStrings(field)
		},
	}
}

// IntAttr constructs an Attr which will retrieve the int field from an implementation of Track.
func IntAttr(field string) Attr {
	return Attr{
		field: field,
		empty: 0,
		eq:    func(x, y interface{}) bool { return x == y },
		fn:    func(t Track) interface{} { return t.GetInt(field) },
	}
}

// FirstTrackAttr wraps the given Group adding a string field `field` with the value taken
// from the first track.
func FirstTrackAttr(attr Attr, g Group) Group {
	t := firstTrack(g)
	if t == nil {
		return g
	}

	v := attr.fn(t)
	if attr.eq(v, attr.Empty()) {
		return g
	}
	m := map[string]interface{}{
		attr.field: v,
	}
	return fieldsGroup(m, g)
}
<commit_msg>Fix: when param to Attr.eq are interface{}(nil).<commit_after>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package index

// subColFlds is a Collection wrapper which overrides the Field
// method.
type subColFlds struct {
	Collection
	flds map[string]interface{}
}

func (scf subColFlds) Field(f string) interface{} {
	if x, ok := scf.flds[f]; ok {
		return x
	}
	return scf.Collection.Field(f)
}

// subGrpFlds is a Group wrapper which overrids the Field method
type subGrpFlds struct {
	Group
	tracks []Track
	flds   map[string]interface{}
}

func (sgf subGrpFlds) Tracks() []Track {
	if sgf.tracks != nil {
		return sgf.tracks
	}
	return sgf.Group.Tracks()
}

// Field implements Group.
func (sgf subGrpFlds) Field(field string) interface{} {
	if x, ok := sgf.flds[field]; ok {
		return x
	}
	return sgf.Group.Field(field)
}

// SumGroupIntAttr recurses through the Group and assigns the field with the sum
// of fields from children (Groups or Tracks).
func SumGroupIntAttr(field string, g Group) Group {
	if c, ok := g.(Collection); ok {
		return sumCollectionIntAttr(field, c)
	}
	return sumGroupIntAttr(field, g)
}

func sumCollectionIntAttr(field string, c Collection) Collection {
	nc := subCol{
		Collection: c,
		grps:       make(map[Key]Group, len(c.Keys())),
		flds:       make(map[string]interface{}),
	}
	var total int
	for _, k := range c.Keys() {
		g := c.Get(k)
		g = SumGroupIntAttr(field, g)
		total += g.Field(field).(int)
		nc.grps[k] = g
	}
	nc.flds = map[string]interface{}{
		field: total,
	}
	return nc
}

func sumGroupIntAttr(field string, g Group) Group {
	ng := subGrpFlds{
		Group: g,
		flds:  map[string]interface{}{},
	}
	var total int
	for _, t := range g.Tracks() {
		total += t.GetInt(field)
	}
	ng.flds[field] = total
	return ng
}

// CommonGroupAttr recurses through the Group and assigns fields on all sub groups
// which are common amoungst their children (Groups or Tracks).  If there is no common
// field, then the associated Field value is not set.
func CommonGroupAttr(attrs []Attr, g Group) Group {
	if c, ok := g.(Collection); ok {
		return commonCollectionTrackAttr(attrs, c)
	}
	return commonGroupTrackAttr(attrs, g)
}

func commonCollectionTrackAttr(attrs []Attr, c Collection) Collection {
	grps := make(map[Key]Group, len(c.Keys()))
	flds := make(map[string]interface{}, len(attrs))

	keys := c.Keys()
	if len(keys) > 0 {
		k0 := keys[0]
		g0 := c.Get(k0)
		g0 = CommonGroupAttr(attrs, g0)
		grps[k0] = g0

		for _, a := range attrs {
			flds[a.Field()] = g0.Field(a.Field())
		}

		if len(keys) > 1 {
			for _, k := range keys[1:] {
				g1 := c.Get(k)
				g1 = CommonGroupAttr(attrs, g1)
				grps[k] = g1

				for _, a := range attrs {
					f := a.Field()
					v := g1.Field(f)
					if !a.eq(flds[f], v) {
						flds[f] = a.Empty()
					}
				}
			}
		}
	}

	for _, a := range attrs {
		f := a.Field()
		if v, ok := flds[f]; ok && v == a.Empty() {
			delete(flds, f)
		}
	}

	return subCol{
		Collection: c,
		grps:       grps,
		flds:       flds,
	}
}

func commonGroupTrackAttr(attrs []Attr, g Group) Group {
	flds := make(map[string]interface{}, len(attrs))
	tracks := g.Tracks()

	if len(tracks) > 0 {
		t0 := tracks[0]
		for _, a := range attrs {
			f := a.Field()
			flds[f] = a.fn(t0)
		}

		if len(tracks) > 1 {
			for _, t := range tracks[1:] {
				for _, a := range attrs {
					f := a.Field()
					if !a.eq(flds[f], a.fn(t)) {
						flds[f] = a.Empty()
					}
				}
			}
		}
	}

	for _, a := range attrs {
		f := a.Field()
		if v, ok := flds[f]; ok && a.eq(v, a.Empty()) {
			delete(flds, f)
		}
	}

	return subGrpFlds{
		Group: g,
		flds:  flds,
	}
}

// subGrpName is a Group wrapper which overrides Name.
type subGrpName struct {
	Group
	name string
}

// Name implements Group.
func (s subGrpName) Name() string {
	return s.name
}

// RemoveEmptyCollections recursively goes through each sub Collection contained
// in the Group and removes any which don't have any tracks/groups in them.
func RemoveEmptyCollections(g Group) Group {
	gc, ok := g.(Collection)
	if ok {
		keys := gc.Keys()
		if len(keys) == 1 {
			gc0 := gc.Get(keys[0])
			_, col := gc0.(Collection)
			if !col && gc0.Name() == "" {
				return subGrpName{
					name:  gc.Name(),
					Group: gc0,
				}
			}
		}
		ngc := subCol{
			Collection: gc,
			grps:       make(map[Key]Group, len(gc.Keys())),
		}
		for _, k := range keys {
			ngc.grps[k] = RemoveEmptyCollections(gc.Get(k))
		}
		return ngc
	}
	return g
}

func firstTrack(g Group) Track {
	c, ok := g.(Collection)
	if ok {
		keys := c.Keys()
		if len(keys) > 0 {
			return firstTrack(c.Get(keys[0]))
		}
		return nil
	}

	ts := g.Tracks()
	if len(ts) > 0 {
		return ts[0]
	}
	return nil
}

func fieldsGroup(m map[string]interface{}, g Group) Group {
	if c, ok := g.(Collection); ok {
		return subColFlds{
			Collection: c,
			flds:       m,
		}
	}
	return subGrpFlds{
		Group: g,
		flds:  m,
	}
}

// Attr is a type which wraps a closure to get an attribute from an implementation of the
// Attr interface.
type Attr struct {
	field string
	empty interface{}
	eq    func(x, y interface{}) bool
	fn    func(t Track) interface{}
}

// Field returns the underlying field name.
func (g Attr) Field() string {
	return g.field
}

// Empty returns the empty value of the underlying field (the empty value of the field type).
func (g Attr) Empty() interface{} {
	return g.empty
}

// StringAttr constructs an Attr which will retrieve the string field from an implementation
// of Track.
func StringAttr(field string) Attr {
	return Attr{
		field: field,
		empty: "",
		eq:    func(x, y interface{}) bool { return x == y },
		fn:    func(t Track) interface{} { return t.GetString(field) },
	}
}

// StringsAttr returns an Attr which will retrieve the strings field from an implementation of Track.
func StringsAttr(field string) Attr {
	return Attr{
		field: field,
		empty: nil,
		eq: func(x, y interface{}) bool {
			// Annoyingly we have to cater for zero values from map[string]interface{}
			// which don't have the correct type wrapping the nil.
			if x == nil || y == nil {
				return x == nil && y == nil
			}
			xs := x.([]string) // NB: panics here are acceptable: should not be called on a non-'Strings' field.
			ys := y.([]string)
			if len(xs) != len(ys) {
				return false
			}
			for i, xss := range xs {
				if ys[i] != xss {
					return false
				}
			}
			return true
		},
		fn: func(t Track) interface{} {
			return t.GetStrings(field)
		},
	}
}

// IntAttr constructs an Attr which will retrieve the int field from an implementation of Track.
func IntAttr(field string) Attr {
	return Attr{
		field: field,
		empty: 0,
		eq:    func(x, y interface{}) bool { return x == y },
		fn:    func(t Track) interface{} { return t.GetInt(field) },
	}
}

// FirstTrackAttr wraps the given Group adding a string field `field` with the value taken
// from the first track.
func FirstTrackAttr(attr Attr, g Group) Group {
	t := firstTrack(g)
	if t == nil {
		return g
	}

	v := attr.fn(t)
	if attr.eq(v, attr.Empty()) {
		return g
	}
	m := map[string]interface{}{
		attr.field: v,
	}
	return fieldsGroup(m, g)
}
<|endoftext|>


<commit_before>
package main

import (
  "testing"
  r "github.com/dancannon/gorethink"
)

func Test_IndexPresent_Simple_Create(t *testing.T) {
  table := Table{Name: "Test_IndexPresent"}
  err := client.TablePresent(table)
  expect(t, err, nil)

  index := Index{
    Name: "Test_IndexPresent_Simple_Create",
  }
  err = client.IndexPresent(table, index)
  expect(t, err, nil)

  indices, _ := client.TableIndexList(table)
  expect(t, stringInSlice(index.Name, indices), true)

  // Do an insert and find
  obj := map[string]string{"Test_IndexPresent_Simple_Create": "cheeseypoofs"}
  _, err = r.Table(table.Name).Insert(obj).RunWrite(session)
  expect(t, err, nil)

  find, err := r.Table(table.Name).
    GetAllByIndex(index.Name, "cheeseypoofs").
    Count().Run(session)
  expect(t, err, nil)
  var count int
  err = find.One(&count)
  expect(t, err, nil)
  expect(t, count, 1)
}

func Test_IndexPresent_Simple_IndexListFail(t *testing.T) {
  table := Table{Name: "Test_IndexPresent"}
  c, _ := NewClient(Connection{host, "non-existent-db"})

  index := Index{
    Name: "Test_IndexPresent_Simple_IndexListFail",
  }
  err := c.IndexPresent(table, index)
  refute(t, err, nil)
}

func Test_IndexPresent_Simple_CreateFail(t *testing.T) {
  table := Table{Name: "Test_IndexPresent"}
  err := client.TablePresent(table)
  expect(t, err, nil)

  index := Index{
    Name: "id", // already primary key
  }
  err = client.IndexPresent(table, index)
  refute(t, err, nil)

  indices, _ := client.TableIndexList(table)
  expect(t, stringInSlice(index.Name, indices), false)
}

func Test_IndexPresent_Compound_Create(t *testing.T) {
  table := Table{Name: "Test_IndexPresent"}
  err := client.TablePresent(table)
  expect(t, err, nil)

  index := Index{
    Name: "Test_IndexPresent_Compound_Create",
    Fields: []string{"user_id", "flavor"},
  }
  err = client.IndexPresent(table, index)
  expect(t, err, nil)

  indices, _ := client.TableIndexList(table)
  expect(t, stringInSlice(index.Name, indices), true)

  // Do an insert and find
  obj := map[string]string{"user_id": "12345", "flavor": "vanilla"}
  _, err = r.Table(table.Name).Insert(obj).RunWrite(session)
  expect(t, err, nil)

  find, err := r.Table(table.Name).
    GetAllByIndex(index.Name, []string{"12345", "vanilla"}).
    Count().Run(session)
  expect(t, err, nil)
  var count int
  err = find.One(&count)
  expect(t, err, nil)
  expect(t, count, 1)
}

func Test_IndexPresent_Compound_CreateFail(t *testing.T) {
  table := Table{Name: "Test_IndexPresent"}
  err := client.TablePresent(table)
  expect(t, err, nil)

  index := Index{
    Name: "id", // already a primary key
    Fields: []string{"user_id", "flavor"},
  }
  err = client.IndexPresent(table, index)
  refute(t, err, nil)

  indices, _ := client.TableIndexList(table)
  expect(t, stringInSlice(index.Name, indices), false)
}

func Test_TableIndexList_Fail(t *testing.T) {
  table := Table{Name: "NONEXISTENT_TABLE"}
  _, err := client.TableIndexList(table)
  refute(t, err, nil)
}

func Test_IndexPresent_Exists(t *testing.T) {
  table := Table{Name: "Test_IndexPresent"}
  err := client.TablePresent(table)
  expect(t, err, nil)

  index := Index{
    Name: "Test_IndexPresent_Exists",
  }
  err = client.IndexPresent(table, index)
  expect(t, err, nil)

  // Run it again!
  err = client.IndexPresent(table, index)
  expect(t, err, nil)
}

func Test_IndexAbsent_Success(t *testing.T) {
  table := Table{Name: "Test_IndexAbsent_Success"}
  err := client.TablePresent(table)
  expect(t, err, nil)

  index := Index{Name: "Test_IndexAbsent_Success"}
  err = client.IndexPresent(table, index)
  expect(t, err, nil)

  indices, _ := client.TableIndexList(table)
  expect(t, stringInSlice(index.Name, indices), true)

  err = client.IndexAbsent(table, index)
  expect(t, err, nil)

  indices, _ = client.TableIndexList(table)
  expect(t, stringInSlice(index.Name, indices), false)
}

func Test_IndexAbsent_IndexListFail(t *testing.T) {
  table := Table{Name: "Test_IndexAbsent_IndexListFail"}
  index := Index{Name: "Test_IndexAbsent_IndexListFail"}

  badClient, _ := NewClient(Connection{host, "cheese"})
  err := badClient.IndexAbsent(table, index)
  refute(t, err, nil)
}

func Test_IndexAbsent_Fail(t *testing.T) {
  table := Table{Name: "Test_IndexAbsent_Fail"}
  index := Index{Name: "Test_IndexAbsent_Fail"}

  badClient, _ := NewClient(Connection{host, "cheese"})
  // Spoof the index list cache to think that the index already exists
  badClient.indexListCache[table.Name] = []string{index.Name}
  err := badClient.IndexAbsent(table, index)
  refute(t, err, nil)
}

func Test_IndexAbsent_Already_Absent(t *testing.T) {
  table := Table{Name: "Test_IndexAbsent_Fail"}
  err := client.TablePresent(table)
  expect(t, err, nil)

  index := Index{Name: "Test_IndexAbsent_Fail"}
  err = client.IndexPresent(table, index)
  expect(t, err, nil)

  indices, _ := client.TableIndexList(table)
  expect(t, stringInSlice(index.Name, indices), true)

  err = client.IndexAbsent(table, index)
  expect(t, err, nil)

  // Run it again!
  err = client.IndexAbsent(table, index)
  expect(t, err, nil)

}<commit_msg>Adding index wait command for test<commit_after>
package main

import (
  "testing"
  r "github.com/dancannon/gorethink"
)

func Test_IndexPresent_Simple_Create(t *testing.T) {
  table := Table{Name: "Test_IndexPresent"}
  err := client.TablePresent(table)
  expect(t, err, nil)

  index := Index{
    Name: "Test_IndexPresent_Simple_Create",
  }
  err = client.IndexPresent(table, index)
  expect(t, err, nil)
  r.Table(table.Name).IndexWait().Exec(session)

  indices, _ := client.TableIndexList(table)
  expect(t, stringInSlice(index.Name, indices), true)

  // Do an insert and find
  obj := map[string]string{"Test_IndexPresent_Simple_Create": "cheeseypoofs"}
  _, err = r.Table(table.Name).Insert(obj).RunWrite(session)
  expect(t, err, nil)

  find, err := r.Table(table.Name).
    GetAllByIndex(index.Name, "cheeseypoofs").
    Count().Run(session)
  expect(t, err, nil)
  var count int
  err = find.One(&count)
  expect(t, err, nil)
  expect(t, count, 1)
}

func Test_IndexPresent_Simple_IndexListFail(t *testing.T) {
  table := Table{Name: "Test_IndexPresent"}
  c, _ := NewClient(Connection{host, "non-existent-db"})

  index := Index{
    Name: "Test_IndexPresent_Simple_IndexListFail",
  }
  err := c.IndexPresent(table, index)
  refute(t, err, nil)
}

func Test_IndexPresent_Simple_CreateFail(t *testing.T) {
  table := Table{Name: "Test_IndexPresent"}
  err := client.TablePresent(table)
  expect(t, err, nil)

  index := Index{
    Name: "id", // already primary key
  }
  err = client.IndexPresent(table, index)
  refute(t, err, nil)

  indices, _ := client.TableIndexList(table)
  expect(t, stringInSlice(index.Name, indices), false)
}

func Test_IndexPresent_Compound_Create(t *testing.T) {
  table := Table{Name: "Test_IndexPresent"}
  err := client.TablePresent(table)
  expect(t, err, nil)

  index := Index{
    Name: "Test_IndexPresent_Compound_Create",
    Fields: []string{"user_id", "flavor"},
  }
  err = client.IndexPresent(table, index)
  expect(t, err, nil)
  r.Table(table.Name).IndexWait().Exec(session)

  indices, _ := client.TableIndexList(table)
  expect(t, stringInSlice(index.Name, indices), true)

  // Do an insert and find
  obj := map[string]string{"user_id": "12345", "flavor": "vanilla"}
  _, err = r.Table(table.Name).Insert(obj).RunWrite(session)
  expect(t, err, nil)

  find, err := r.Table(table.Name).
    GetAllByIndex(index.Name, []string{"12345", "vanilla"}).
    Count().Run(session)
  expect(t, err, nil)
  var count int
  err = find.One(&count)
  expect(t, err, nil)
  expect(t, count, 1)
}

func Test_IndexPresent_Compound_CreateFail(t *testing.T) {
  table := Table{Name: "Test_IndexPresent"}
  err := client.TablePresent(table)
  expect(t, err, nil)

  index := Index{
    Name: "id", // already a primary key
    Fields: []string{"user_id", "flavor"},
  }
  err = client.IndexPresent(table, index)
  refute(t, err, nil)

  indices, _ := client.TableIndexList(table)
  expect(t, stringInSlice(index.Name, indices), false)
}

func Test_TableIndexList_Fail(t *testing.T) {
  table := Table{Name: "NONEXISTENT_TABLE"}
  _, err := client.TableIndexList(table)
  refute(t, err, nil)
}

func Test_IndexPresent_Exists(t *testing.T) {
  table := Table{Name: "Test_IndexPresent"}
  err := client.TablePresent(table)
  expect(t, err, nil)

  index := Index{
    Name: "Test_IndexPresent_Exists",
  }
  err = client.IndexPresent(table, index)
  expect(t, err, nil)

  // Run it again!
  err = client.IndexPresent(table, index)
  expect(t, err, nil)
}

func Test_IndexAbsent_Success(t *testing.T) {
  table := Table{Name: "Test_IndexAbsent_Success"}
  err := client.TablePresent(table)
  expect(t, err, nil)

  index := Index{Name: "Test_IndexAbsent_Success"}
  err = client.IndexPresent(table, index)
  expect(t, err, nil)
  r.Table(table.Name).IndexWait().Exec(session)

  indices, _ := client.TableIndexList(table)
  expect(t, stringInSlice(index.Name, indices), true)

  err = client.IndexAbsent(table, index)
  expect(t, err, nil)

  indices, _ = client.TableIndexList(table)
  expect(t, stringInSlice(index.Name, indices), false)
}

func Test_IndexAbsent_IndexListFail(t *testing.T) {
  table := Table{Name: "Test_IndexAbsent_IndexListFail"}
  index := Index{Name: "Test_IndexAbsent_IndexListFail"}

  badClient, _ := NewClient(Connection{host, "cheese"})
  err := badClient.IndexAbsent(table, index)
  refute(t, err, nil)
}

func Test_IndexAbsent_Fail(t *testing.T) {
  table := Table{Name: "Test_IndexAbsent_Fail"}
  index := Index{Name: "Test_IndexAbsent_Fail"}

  badClient, _ := NewClient(Connection{host, "cheese"})
  // Spoof the index list cache to think that the index already exists
  badClient.indexListCache[table.Name] = []string{index.Name}
  err := badClient.IndexAbsent(table, index)
  refute(t, err, nil)
}

func Test_IndexAbsent_Already_Absent(t *testing.T) {
  table := Table{Name: "Test_IndexAbsent_Fail"}
  err := client.TablePresent(table)
  expect(t, err, nil)

  index := Index{Name: "Test_IndexAbsent_Fail"}
  err = client.IndexPresent(table, index)
  expect(t, err, nil)

  indices, _ := client.TableIndexList(table)
  expect(t, stringInSlice(index.Name, indices), true)

  err = client.IndexAbsent(table, index)
  expect(t, err, nil)

  // Run it again!
  err = client.IndexAbsent(table, index)
  expect(t, err, nil)

}<|endoftext|>


<commit_before>// Copyright 2015 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package networking

import (
	"errors"
	"fmt"
	"net"
	"os"
	"syscall"

	"github.com/appc/cni/pkg/ns"
	cnitypes "github.com/appc/cni/pkg/types"
	"github.com/appc/spec/schema/types"
	"github.com/hashicorp/errwrap"
	"github.com/vishvananda/netlink"

	"github.com/coreos/rkt/common"
	"github.com/coreos/rkt/networking/netinfo"
	"github.com/coreos/rkt/pkg/log"
)

const (
	IfNamePattern = "eth%d"
	selfNetNS     = "/proc/self/ns/net"
)

// ForwardedPort describes a port that will be
// forwarded (mapped) from the host to the pod
type ForwardedPort struct {
	Protocol string
	HostPort uint
	PodPort  uint
}

// Networking describes the networking details of a pod.
type Networking struct {
	podEnv

	hostNS *os.File
	nets   []activeNet
}

// NetConf local struct extends cnitypes.NetConf with information about masquerading
// similar to CNI plugins
type NetConf struct {
	cnitypes.NetConf
	IPMasq bool `json:"ipMasq"`
	MTU    int  `json:"mtu"`
}

var stderr *log.Logger

// Setup creates a new networking namespace and executes network plugins to
// set up networking. It returns in the new pod namespace
func Setup(podRoot string, podID types.UUID, fps []ForwardedPort, netList common.NetList, localConfig, flavor string, debug bool) (*Networking, error) {

	stderr = log.New(os.Stderr, "networking", debug)

	if flavor == "kvm" {
		return kvmSetup(podRoot, podID, fps, netList, localConfig)
	}

	// TODO(jonboulle): currently podRoot is _always_ ".", and behaviour in other
	// circumstances is untested. This should be cleaned up.
	n := Networking{
		podEnv: podEnv{
			podRoot:      podRoot,
			podID:        podID,
			netsLoadList: netList,
			localConfig:  localConfig,
		},
	}

	hostNS, podNS, err := basicNetNS()
	if err != nil {
		return nil, err
	}
	// we're in podNS!
	n.hostNS = hostNS

	nspath := n.podNSPath()

	if err = bindMountFile(selfNetNS, nspath); err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			if err := syscall.Unmount(nspath, 0); err != nil {
				stderr.PrintE(fmt.Sprintf("error unmounting %q", nspath), err)
			}
		}
	}()

	n.nets, err = n.loadNets()
	if err != nil {
		return nil, errwrap.Wrap(errors.New("error loading network definitions"), err)
	}

	err = withNetNS(podNS, hostNS, func() error {
		if err := n.setupNets(n.nets); err != nil {
			return err
		}
		return n.forwardPorts(fps, n.GetDefaultIP())
	})
	if err != nil {
		return nil, err
	}

	return &n, nil
}

// Load creates the Networking object from saved state.
// Assumes the current netns is that of the host.
func Load(podRoot string, podID *types.UUID) (*Networking, error) {
	// the current directory is pod root
	pdirfd, err := syscall.Open(podRoot, syscall.O_RDONLY|syscall.O_DIRECTORY, 0)
	if err != nil {
		return nil, errwrap.Wrap(fmt.Errorf("failed to open pod root directory (%v)", podRoot), err)
	}
	defer syscall.Close(pdirfd)

	nis, err := netinfo.LoadAt(pdirfd)
	if err != nil {
		return nil, err
	}

	hostNS, err := os.Open(selfNetNS)
	if err != nil {
		return nil, err
	}

	var nets []activeNet
	for _, ni := range nis {
		n, err := loadNet(ni.ConfPath)
		if err != nil {
			if !os.IsNotExist(err) {
				stderr.PrintE(fmt.Sprintf("error loading %q; ignoring", ni.ConfPath), err)
			}
			continue
		}

		// make a copy of ni to make it a unique object as it's saved via ptr
		rti := ni
		n.runtime = &rti
		nets = append(nets, *n)
	}

	return &Networking{
		podEnv: podEnv{
			podRoot: podRoot,
			podID:   *podID,
		},
		hostNS: hostNS,
		nets:   nets,
	}, nil
}

func (n *Networking) GetDefaultIP() net.IP {
	if len(n.nets) == 0 {
		return nil
	}
	return n.nets[len(n.nets)-1].runtime.IP
}

func (n *Networking) GetDefaultHostIP() (net.IP, error) {
	if len(n.nets) == 0 {
		return nil, fmt.Errorf("no networks found")
	}
	return n.nets[len(n.nets)-1].runtime.HostIP, nil
}

// Teardown cleans up a produced Networking object.
func (n *Networking) Teardown(flavor string, debug bool) {

	stderr = log.New(os.Stderr, "networking", debug)

	// Teardown everything in reverse order of setup.
	// This should be idempotent -- be tolerant of missing stuff

	if flavor == "kvm" {
		n.kvmTeardown()
		return
	}

	if err := n.enterHostNS(); err != nil {
		stderr.PrintE("error switching to host netns", err)
		return
	}

	if err := n.unforwardPorts(); err != nil {
		stderr.PrintE("error removing forwarded ports", err)
	}

	n.teardownNets(n.nets)

	if err := syscall.Unmount(n.podNSPath(), 0); err != nil {
		// if already unmounted, umount(2) returns EINVAL
		if !os.IsNotExist(err) && err != syscall.EINVAL {
			stderr.PrintE(fmt.Sprintf("error unmounting %q", n.podNSPath()), err)
		}
	}
}

// sets up new netns with just lo
func basicNetNS() (hostNS, podNS *os.File, err error) {
	hostNS, podNS, err = newNetNS()
	if err != nil {
		err = errwrap.Wrap(errors.New("failed to create new netns"), err)
		return
	}
	// we're in podNS!!

	if err = loUp(); err != nil {
		hostNS.Close()
		podNS.Close()
		return nil, nil, err
	}

	return
}

// enterHostNS moves into the host's network namespace.
func (n *Networking) enterHostNS() error {
	return ns.SetNS(n.hostNS, syscall.CLONE_NEWNET)
}

// Save writes out the info about active nets
// for "rkt list" and friends to display
func (e *Networking) Save() error {
	var nis []netinfo.NetInfo
	for _, n := range e.nets {
		nis = append(nis, *n.runtime)
	}

	return netinfo.Save(e.podRoot, nis)
}

func newNetNS() (hostNS, childNS *os.File, err error) {
	defer func() {
		if err != nil {
			if hostNS != nil {
				hostNS.Close()
			}
			if childNS != nil {
				childNS.Close()
			}
		}
	}()

	hostNS, err = os.Open(selfNetNS)
	if err != nil {
		return
	}

	if err = syscall.Unshare(syscall.CLONE_NEWNET); err != nil {
		return
	}

	childNS, err = os.Open(selfNetNS)
	if err != nil {
		ns.SetNS(hostNS, syscall.CLONE_NEWNET)
		return
	}

	return
}

// execute f() in tgtNS
func withNetNS(curNS, tgtNS *os.File, f func() error) error {
	if err := ns.SetNS(tgtNS, syscall.CLONE_NEWNET); err != nil {
		return err
	}

	if err := f(); err != nil {
		// Attempt to revert the net ns in a known state
		if err := ns.SetNS(curNS, syscall.CLONE_NEWNET); err != nil {
			stderr.PrintE("cannot revert the net namespace", err)
		}
		return err
	}

	return ns.SetNS(curNS, syscall.CLONE_NEWNET)
}

func loUp() error {
	lo, err := netlink.LinkByName("lo")
	if err != nil {
		return errwrap.Wrap(errors.New("failed to lookup lo"), err)
	}

	if err := netlink.LinkSetUp(lo); err != nil {
		return errwrap.Wrap(errors.New("failed to set lo up"), err)
	}

	return nil
}

func bindMountFile(src, dst string) error {
	// mount point has to be an existing file
	f, err := os.Create(dst)
	if err != nil {
		return err
	}
	f.Close()

	return syscall.Mount(src, dst, "none", syscall.MS_BIND, "")
}
<commit_msg>port forwarding: enable route_localnet on default interface<commit_after>// Copyright 2015 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package networking

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"syscall"

	"github.com/appc/cni/pkg/ns"
	cnitypes "github.com/appc/cni/pkg/types"
	"github.com/appc/spec/schema/types"
	"github.com/hashicorp/errwrap"
	"github.com/vishvananda/netlink"

	"github.com/coreos/rkt/common"
	"github.com/coreos/rkt/networking/netinfo"
	"github.com/coreos/rkt/pkg/log"
)

const (
	IfNamePattern = "eth%d"
	selfNetNS     = "/proc/self/ns/net"
)

// ForwardedPort describes a port that will be
// forwarded (mapped) from the host to the pod
type ForwardedPort struct {
	Protocol string
	HostPort uint
	PodPort  uint
}

// Networking describes the networking details of a pod.
type Networking struct {
	podEnv

	hostNS *os.File
	nets   []activeNet
}

// NetConf local struct extends cnitypes.NetConf with information about masquerading
// similar to CNI plugins
type NetConf struct {
	cnitypes.NetConf
	IPMasq bool `json:"ipMasq"`
	MTU    int  `json:"mtu"`
}

var stderr *log.Logger

// Setup creates a new networking namespace and executes network plugins to
// set up networking. It returns in the new pod namespace
func Setup(podRoot string, podID types.UUID, fps []ForwardedPort, netList common.NetList, localConfig, flavor string, debug bool) (*Networking, error) {

	stderr = log.New(os.Stderr, "networking", debug)

	if flavor == "kvm" {
		return kvmSetup(podRoot, podID, fps, netList, localConfig)
	}

	// TODO(jonboulle): currently podRoot is _always_ ".", and behaviour in other
	// circumstances is untested. This should be cleaned up.
	n := Networking{
		podEnv: podEnv{
			podRoot:      podRoot,
			podID:        podID,
			netsLoadList: netList,
			localConfig:  localConfig,
		},
	}

	hostNS, podNS, err := basicNetNS()
	if err != nil {
		return nil, err
	}
	// we're in podNS!
	n.hostNS = hostNS

	nspath := n.podNSPath()

	if err = bindMountFile(selfNetNS, nspath); err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			if err := syscall.Unmount(nspath, 0); err != nil {
				stderr.PrintE(fmt.Sprintf("error unmounting %q", nspath), err)
			}
		}
	}()

	n.nets, err = n.loadNets()
	if err != nil {
		return nil, errwrap.Wrap(errors.New("error loading network definitions"), err)
	}

	err = withNetNS(podNS, hostNS, func() error {
		if err := n.setupNets(n.nets); err != nil {
			return err
		}
		if len(fps) > 0 {
			if err = n.enableDefaultLocalnetRouting(); err != nil {
				return err
			}
			return n.forwardPorts(fps, n.GetDefaultIP())
		}
		return nil
	})
	if err != nil {
		return nil, err
	}

	return &n, nil
}

// enableDefaultLocalnetRouting enables the route_localnet attribute on the supposedly default network interface.
// This allows setting up loopback NAT so the host can access the pod's forwarded ports on the localhost address.
func (n *Networking) enableDefaultLocalnetRouting() error {
	routeLocalnetFormat := ""

	defaultHostIP, err := n.GetDefaultHostIP()
	if err != nil {
		return err
	}

	defaultHostIPstring := defaultHostIP.String()
	switch {
	case strings.Contains(defaultHostIPstring, "."):
		routeLocalnetFormat = "/proc/sys/net/ipv4/conf/%s/route_localnet"
	case strings.Contains(defaultHostIPstring, ":"):
		return fmt.Errorf("unexpected IPv6 Address returned for default host interface: %q", defaultHostIPstring)
	default:
		return fmt.Errorf("unknown type for default Host IP: %q", defaultHostIPstring)
	}

	hostIfaces, err := n.GetIfacesByIP(defaultHostIP)
	if err != nil {
		return err
	}

	for _, hostIface := range hostIfaces {
		routeLocalnetPath := fmt.Sprintf(routeLocalnetFormat, hostIface.Name)
		routeLocalnetValue, err := ioutil.ReadFile(routeLocalnetPath)
		if err != nil {
			return err
		}
		if string(routeLocalnetValue) != "1" {
			routeLocalnetFile, err := os.OpenFile(routeLocalnetPath, os.O_WRONLY, 0)
			if err != nil {
				return err
			}
			defer routeLocalnetFile.Close()

			if _, err = io.WriteString(routeLocalnetFile, "1"); err != nil {
				return err
			}
		}
	}

	return nil
}

// Load creates the Networking object from saved state.
// Assumes the current netns is that of the host.
func Load(podRoot string, podID *types.UUID) (*Networking, error) {
	// the current directory is pod root
	pdirfd, err := syscall.Open(podRoot, syscall.O_RDONLY|syscall.O_DIRECTORY, 0)
	if err != nil {
		return nil, errwrap.Wrap(fmt.Errorf("failed to open pod root directory (%v)", podRoot), err)
	}
	defer syscall.Close(pdirfd)

	nis, err := netinfo.LoadAt(pdirfd)
	if err != nil {
		return nil, err
	}

	hostNS, err := os.Open(selfNetNS)
	if err != nil {
		return nil, err
	}

	var nets []activeNet
	for _, ni := range nis {
		n, err := loadNet(ni.ConfPath)
		if err != nil {
			if !os.IsNotExist(err) {
				stderr.PrintE(fmt.Sprintf("error loading %q; ignoring", ni.ConfPath), err)
			}
			continue
		}

		// make a copy of ni to make it a unique object as it's saved via ptr
		rti := ni
		n.runtime = &rti
		nets = append(nets, *n)
	}

	return &Networking{
		podEnv: podEnv{
			podRoot: podRoot,
			podID:   *podID,
		},
		hostNS: hostNS,
		nets:   nets,
	}, nil
}

func (n *Networking) GetDefaultIP() net.IP {
	if len(n.nets) == 0 {
		return nil
	}
	return n.nets[len(n.nets)-1].runtime.IP
}

func (n *Networking) GetDefaultHostIP() (net.IP, error) {
	if len(n.nets) == 0 {
		return nil, fmt.Errorf("no networks found")
	}
	return n.nets[len(n.nets)-1].runtime.HostIP, nil
}

// GetIfacesByIP searches for and returns the interfaces with the given IP
// Disregards the subnet mask since not every net.IP object contains
// On success it will return the list of found interfaces
func (n *Networking) GetIfacesByIP(ifaceIP net.IP) ([]net.Interface, error) {
	ifaces, err := net.Interfaces()
	if err != nil {
		return nil, err
	}

	searchAddr := strings.Split(ifaceIP.String(), "/")[0]
	resultInterfaces := make([]net.Interface, 0)

	for _, iface := range ifaces {
		if iface.Flags&net.FlagLoopback != 0 {
			continue
		}

		addrs, err := iface.Addrs()
		if err != nil {
			return nil, errwrap.Wrap(fmt.Errorf("cannot get addresses for interface %v", iface.Name), err)
		}

		for _, addr := range addrs {
			currentAddr := strings.Split(addr.String(), "/")[0]
			if searchAddr == currentAddr {
				resultInterfaces = append(resultInterfaces, iface)
				break
			}
		}
	}

	if len(resultInterfaces) == 0 {
		return nil, fmt.Errorf("no interface found with IP %q", ifaceIP)
	}

	return resultInterfaces, nil
}

// Teardown cleans up a produced Networking object.
func (n *Networking) Teardown(flavor string, debug bool) {

	stderr = log.New(os.Stderr, "networking", debug)

	// Teardown everything in reverse order of setup.
	// This should be idempotent -- be tolerant of missing stuff

	if flavor == "kvm" {
		n.kvmTeardown()
		return
	}

	if err := n.enterHostNS(); err != nil {
		stderr.PrintE("error switching to host netns", err)
		return
	}

	if err := n.unforwardPorts(); err != nil {
		stderr.PrintE("error removing forwarded ports", err)
	}

	n.teardownNets(n.nets)

	if err := syscall.Unmount(n.podNSPath(), 0); err != nil {
		// if already unmounted, umount(2) returns EINVAL
		if !os.IsNotExist(err) && err != syscall.EINVAL {
			stderr.PrintE(fmt.Sprintf("error unmounting %q", n.podNSPath()), err)
		}
	}
}

// sets up new netns with just lo
func basicNetNS() (hostNS, podNS *os.File, err error) {
	hostNS, podNS, err = newNetNS()
	if err != nil {
		err = errwrap.Wrap(errors.New("failed to create new netns"), err)
		return
	}
	// we're in podNS!!

	if err = loUp(); err != nil {
		hostNS.Close()
		podNS.Close()
		return nil, nil, err
	}

	return
}

// enterHostNS moves into the host's network namespace.
func (n *Networking) enterHostNS() error {
	return ns.SetNS(n.hostNS, syscall.CLONE_NEWNET)
}

// Save writes out the info about active nets
// for "rkt list" and friends to display
func (e *Networking) Save() error {
	var nis []netinfo.NetInfo
	for _, n := range e.nets {
		nis = append(nis, *n.runtime)
	}

	return netinfo.Save(e.podRoot, nis)
}

func newNetNS() (hostNS, childNS *os.File, err error) {
	defer func() {
		if err != nil {
			if hostNS != nil {
				hostNS.Close()
			}
			if childNS != nil {
				childNS.Close()
			}
		}
	}()

	hostNS, err = os.Open(selfNetNS)
	if err != nil {
		return
	}

	if err = syscall.Unshare(syscall.CLONE_NEWNET); err != nil {
		return
	}

	childNS, err = os.Open(selfNetNS)
	if err != nil {
		ns.SetNS(hostNS, syscall.CLONE_NEWNET)
		return
	}

	return
}

// execute f() in tgtNS
func withNetNS(curNS, tgtNS *os.File, f func() error) error {
	if err := ns.SetNS(tgtNS, syscall.CLONE_NEWNET); err != nil {
		return err
	}

	if err := f(); err != nil {
		// Attempt to revert the net ns in a known state
		if err := ns.SetNS(curNS, syscall.CLONE_NEWNET); err != nil {
			stderr.PrintE("cannot revert the net namespace", err)
		}
		return err
	}

	return ns.SetNS(curNS, syscall.CLONE_NEWNET)
}

func loUp() error {
	lo, err := netlink.LinkByName("lo")
	if err != nil {
		return errwrap.Wrap(errors.New("failed to lookup lo"), err)
	}

	if err := netlink.LinkSetUp(lo); err != nil {
		return errwrap.Wrap(errors.New("failed to set lo up"), err)
	}

	return nil
}

func bindMountFile(src, dst string) error {
	// mount point has to be an existing file
	f, err := os.Create(dst)
	if err != nil {
		return err
	}
	f.Close()

	return syscall.Mount(src, dst, "none", syscall.MS_BIND, "")
}
<|endoftext|>