<commit_before>package cmd

import (
	"fmt"
	"io"
	"math/rand"
	"os"
	"strings"
	"time"

	"github.com/micro/cli"
	"github.com/micro/go-micro/client"
	"github.com/micro/go-micro/server"

	// brokers
	"github.com/micro/go-micro/broker"
	"github.com/micro/go-micro/broker/http"
	"github.com/micro/go-micro/broker/mqtt"

	// registries
	"github.com/micro/go-micro/registry"
	"github.com/micro/go-micro/registry/consul"
	"github.com/micro/go-micro/registry/mdns"

	// selectors
	"github.com/micro/go-micro/selector"
	"github.com/micro/go-micro/selector/blacklist"
	"github.com/micro/go-micro/selector/cache"

	// transports
	"github.com/micro/go-micro/transport"
	thttp "github.com/micro/go-micro/transport/http"
)

type Cmd interface {
	// The cli app within this cmd
	App() *cli.App
	// Adds options, parses flags and initialise
	// exits on error
	Init(opts ...Option) error
	// Options set within this command
	Options() Options
}

type cmd struct {
	opts Options
	app  *cli.App
}

type Option func(o *Options)

var (
	DefaultCmd = newCmd()

	DefaultFlags = []cli.Flag{
		cli.StringFlag{
			Name:   "client_request_timeout",
			EnvVar: "MICRO_CLIENT_REQUEST_TIMEOUT",
			Usage:  "Sets the client request timeout. e.g 500ms, 5s, 1m. Default: 5s",
		},
		cli.IntFlag{
			Name:   "client_retries",
			EnvVar: "MICRO_CLIENT_RETRIES",
			Usage:  "Sets the client retries. Default: 1",
		},
		cli.StringFlag{
			Name:   "server_name",
			EnvVar: "MICRO_SERVER_NAME",
			Usage:  "Name of the server. go.micro.srv.example",
		},
		cli.StringFlag{
			Name:   "server_version",
			EnvVar: "MICRO_SERVER_VERSION",
			Usage:  "Version of the server. 1.1.0",
		},
		cli.StringFlag{
			Name:   "server_id",
			EnvVar: "MICRO_SERVER_ID",
			Usage:  "Id of the server. Auto-generated if not specified",
		},
		cli.StringFlag{
			Name:   "server_address",
			EnvVar: "MICRO_SERVER_ADDRESS",
			Usage:  "Bind address for the server. 127.0.0.1:8080",
		},
		cli.StringFlag{
			Name:   "server_advertise",
			EnvVar: "MICRO_SERVER_ADVERTISE",
			Usage:  "Used instead of the server_address when registering with discovery. 127.0.0.1:8080",
		},
		cli.StringSliceFlag{
			Name:   "server_metadata",
			EnvVar: "MICRO_SERVER_METADATA",
			Value:  &cli.StringSlice{},
			Usage:  "A list of key-value pairs defining metadata. version=1.0.0",
		},
		cli.StringFlag{
			Name:   "broker",
			EnvVar: "MICRO_BROKER",
			Usage:  "Broker for pub/sub. http, nats, rabbitmq",
		},
		cli.StringFlag{
			Name:   "broker_address",
			EnvVar: "MICRO_BROKER_ADDRESS",
			Usage:  "Comma-separated list of broker addresses",
		},
		cli.StringFlag{
			Name:   "registry",
			EnvVar: "MICRO_REGISTRY",
			Usage:  "Registry for discovery. memory, consul, etcd, kubernetes",
		},
		cli.StringFlag{
			Name:   "registry_address",
			EnvVar: "MICRO_REGISTRY_ADDRESS",
			Usage:  "Comma-separated list of registry addresses",
		},
		cli.StringFlag{
			Name:   "selector",
			EnvVar: "MICRO_SELECTOR",
			Usage:  "Selector used to pick nodes for querying",
		},
		cli.StringFlag{
			Name:   "transport",
			EnvVar: "MICRO_TRANSPORT",
			Usage:  "Transport mechanism used; http, rabbitmq, nats",
		},
		cli.StringFlag{
			Name:   "transport_address",
			EnvVar: "MICRO_TRANSPORT_ADDRESS",
			Usage:  "Comma-separated list of transport addresses",
		},
	}

	DefaultBrokers = map[string]func(...broker.Option) broker.Broker{
		"http": http.NewBroker,
		"mqtt": mqtt.NewBroker,
	}

	DefaultRegistries = map[string]func(...registry.Option) registry.Registry{
		"consul": consul.NewRegistry,
		"mdns":   mdns.NewRegistry,
	}

	DefaultSelectors = map[string]func(...selector.Option) selector.Selector{
		"cache":     cache.NewSelector,
		"blacklist": blacklist.NewSelector,
	}

	DefaultTransports = map[string]func(...transport.Option) transport.Transport{
		"http": thttp.NewTransport,
	}

	// used for default selection as the fall back
	defaultBroker    = "http"
	defaultRegistry  = "consul"
	defaultSelector  = "random"
	defaultTransport = "http"
)

func init() {
	rand.Seed(time.Now().Unix())
	help := cli.HelpPrinter
	cli.HelpPrinter = func(writer io.Writer, templ string, data interface{}) {
		help(writer, templ, data)
		os.Exit(0)
	}
}

func newCmd(opts ...Option) Cmd {
	options := Options{
		Broker:    &broker.DefaultBroker,
		Client:    &client.DefaultClient,
		Registry:  &registry.DefaultRegistry,
		Server:    &server.DefaultServer,
		Selector:  &selector.DefaultSelector,
		Transport: &transport.DefaultTransport,

		Brokers:    DefaultBrokers,
		Registries: DefaultRegistries,
		Selectors:  DefaultSelectors,
		Transports: DefaultTransports,
	}

	for _, o := range opts {
		o(&options)
	}

	if len(options.Description) == 0 {
		options.Description = "a go-micro service"
	}

	cmd := new(cmd)
	cmd.opts = options
	cmd.app = cli.NewApp()
	cmd.app.Name = cmd.opts.Name
	cmd.app.Version = cmd.opts.Version
	cmd.app.Usage = cmd.opts.Description
	cmd.app.Before = cmd.Before
	cmd.app.Flags = DefaultFlags
	cmd.app.Action = func(c *cli.Context) {}

	if len(options.Version) == 0 {
		cmd.app.HideVersion = true
	}

	return cmd
}

func (c *cmd) App() *cli.App {
	return c.app
}

func (c *cmd) Options() Options {
	return c.opts
}

func (c *cmd) Before(ctx *cli.Context) error {
	// If flags are set then use them otherwise do nothing
	var serverOpts []server.Option
	var clientOpts []client.Option

	// Set the broker
	if name := ctx.String("broker"); len(name) > 0 || len(ctx.String("broker_address")) > 0 {
		if len(name) == 0 {
			name = defaultBroker
		}

		if b, ok := c.opts.Brokers[name]; ok {
			n := b(broker.Addrs(strings.Split(ctx.String("broker_address"), ",")...))
			*c.opts.Broker = n
		} else {
			return fmt.Errorf("Broker %s not found", name)
		}

		serverOpts = append(serverOpts, server.Broker(*c.opts.Broker))
		clientOpts = append(clientOpts, client.Broker(*c.opts.Broker))

	}

	// Set the registry
	if name := ctx.String("registry"); len(name) > 0 || len(ctx.String("registry_address")) > 0 {
		if len(name) == 0 {
			name = defaultRegistry
		}

		if r, ok := c.opts.Registries[name]; ok {
			n := r(registry.Addrs(strings.Split(ctx.String("registry_address"), ",")...))
			*c.opts.Registry = n
		} else {
			return fmt.Errorf("Registry %s not found", name)
		}

		serverOpts = append(serverOpts, server.Registry(*c.opts.Registry))
		clientOpts = append(clientOpts, client.Registry(*c.opts.Registry))

		(*c.opts.Selector).Init(selector.Registry(*c.opts.Registry))
		clientOpts = append(clientOpts, client.Selector(*c.opts.Selector))

		(*c.opts.Broker).Init(broker.Registry(*c.opts.Registry))
	}

	// Set the selector
	if name := ctx.String("selector"); len(name) > 0 {
		if s, ok := c.opts.Selectors[name]; ok {
			n := s(selector.Registry(*c.opts.Registry))
			*c.opts.Selector = n
		} else {
			return fmt.Errorf("Selector %s not found", name)
		}

		// No server option here. Should there be?
		clientOpts = append(clientOpts, client.Selector(*c.opts.Selector))
	}

	// Set the transport
	if name := ctx.String("transport"); len(name) > 0 || len(ctx.String("transport_address")) > 0 {
		if len(name) == 0 {
			name = defaultTransport
		}

		if t, ok := c.opts.Transports[name]; ok {
			n := t(transport.Addrs(strings.Split(ctx.String("transport_address"), ",")...))
			*c.opts.Transport = n
		} else {
			return fmt.Errorf("Transport %s not found", name)
		}

		serverOpts = append(serverOpts, server.Transport(*c.opts.Transport))
		clientOpts = append(clientOpts, client.Transport(*c.opts.Transport))
	}

	// Parse the server options
	metadata := make(map[string]string)
	for _, d := range ctx.StringSlice("server_metadata") {
		var key, val string
		parts := strings.Split(d, "=")
		key = parts[0]
		if len(parts) > 1 {
			val = strings.Join(parts[1:], "=")
		}
		metadata[key] = val
	}

	if len(metadata) > 0 {
		serverOpts = append(serverOpts, server.Metadata(metadata))
	}

	if len(ctx.String("server_name")) > 0 {
		serverOpts = append(serverOpts, server.Name(ctx.String("server_name")))
	}

	if len(ctx.String("server_version")) > 0 {
		serverOpts = append(serverOpts, server.Version(ctx.String("server_version")))
	}

	if len(ctx.String("server_id")) > 0 {
		serverOpts = append(serverOpts, server.Id(ctx.String("server_id")))
	}

	if len(ctx.String("server_address")) > 0 {
		serverOpts = append(serverOpts, server.Address(ctx.String("server_address")))
	}

	if len(ctx.String("server_advertise")) > 0 {
		serverOpts = append(serverOpts, server.Advertise(ctx.String("server_advertise")))
	}

	// client opts
	if r := ctx.Int("client_retries"); r > 0 {
		clientOpts = append(clientOpts, client.Retries(r))
	}

	if t := ctx.String("client_request_timeout"); len(t) > 0 {
		d, err := time.ParseDuration(t)
		if err != nil {
			return fmt.Errorf("failed to parse client_request_timeout: %v", t)
		}
		clientOpts = append(clientOpts, client.RequestTimeout(d))
	}

	// We have some command line opts for the server.
	// Lets set it up
	if len(serverOpts) > 0 {
		(*c.opts.Server).Init(serverOpts...)
	}

	// Use an init option?
	if len(clientOpts) > 0 {
		(*c.opts.Client).Init(clientOpts...)
	}

	return nil
}

func (c *cmd) Init(opts ...Option) error {
	for _, o := range opts {
		o(&c.opts)
	}
	c.app.Name = c.opts.Name
	c.app.Version = c.opts.Version
	c.app.Usage = c.opts.Description
	c.app.RunAndExitOnError()
	return nil
}

func DefaultOptions() Options {
	return DefaultCmd.Options()
}

func App() *cli.App {
	return DefaultCmd.App()
}

func Init(opts ...Option) error {
	return DefaultCmd.Init(opts...)
}

func NewCmd(opts ...Option) Cmd {
	return newCmd(opts...)
}
<commit_msg>Add default to selector list<commit_after>package cmd

import (
	"fmt"
	"io"
	"math/rand"
	"os"
	"strings"
	"time"

	"github.com/micro/cli"
	"github.com/micro/go-micro/client"
	"github.com/micro/go-micro/server"

	// brokers
	"github.com/micro/go-micro/broker"
	"github.com/micro/go-micro/broker/http"
	"github.com/micro/go-micro/broker/mqtt"

	// registries
	"github.com/micro/go-micro/registry"
	"github.com/micro/go-micro/registry/consul"
	"github.com/micro/go-micro/registry/mdns"

	// selectors
	"github.com/micro/go-micro/selector"
	"github.com/micro/go-micro/selector/blacklist"
	"github.com/micro/go-micro/selector/cache"

	// transports
	"github.com/micro/go-micro/transport"
	thttp "github.com/micro/go-micro/transport/http"
)

type Cmd interface {
	// The cli app within this cmd
	App() *cli.App
	// Adds options, parses flags and initialise
	// exits on error
	Init(opts ...Option) error
	// Options set within this command
	Options() Options
}

type cmd struct {
	opts Options
	app  *cli.App
}

type Option func(o *Options)

var (
	DefaultCmd = newCmd()

	DefaultFlags = []cli.Flag{
		cli.StringFlag{
			Name:   "client_request_timeout",
			EnvVar: "MICRO_CLIENT_REQUEST_TIMEOUT",
			Usage:  "Sets the client request timeout. e.g 500ms, 5s, 1m. Default: 5s",
		},
		cli.IntFlag{
			Name:   "client_retries",
			EnvVar: "MICRO_CLIENT_RETRIES",
			Usage:  "Sets the client retries. Default: 1",
		},
		cli.StringFlag{
			Name:   "server_name",
			EnvVar: "MICRO_SERVER_NAME",
			Usage:  "Name of the server. go.micro.srv.example",
		},
		cli.StringFlag{
			Name:   "server_version",
			EnvVar: "MICRO_SERVER_VERSION",
			Usage:  "Version of the server. 1.1.0",
		},
		cli.StringFlag{
			Name:   "server_id",
			EnvVar: "MICRO_SERVER_ID",
			Usage:  "Id of the server. Auto-generated if not specified",
		},
		cli.StringFlag{
			Name:   "server_address",
			EnvVar: "MICRO_SERVER_ADDRESS",
			Usage:  "Bind address for the server. 127.0.0.1:8080",
		},
		cli.StringFlag{
			Name:   "server_advertise",
			EnvVar: "MICRO_SERVER_ADVERTISE",
			Usage:  "Used instead of the server_address when registering with discovery. 127.0.0.1:8080",
		},
		cli.StringSliceFlag{
			Name:   "server_metadata",
			EnvVar: "MICRO_SERVER_METADATA",
			Value:  &cli.StringSlice{},
			Usage:  "A list of key-value pairs defining metadata. version=1.0.0",
		},
		cli.StringFlag{
			Name:   "broker",
			EnvVar: "MICRO_BROKER",
			Usage:  "Broker for pub/sub. http, nats, rabbitmq",
		},
		cli.StringFlag{
			Name:   "broker_address",
			EnvVar: "MICRO_BROKER_ADDRESS",
			Usage:  "Comma-separated list of broker addresses",
		},
		cli.StringFlag{
			Name:   "registry",
			EnvVar: "MICRO_REGISTRY",
			Usage:  "Registry for discovery. memory, consul, etcd, kubernetes",
		},
		cli.StringFlag{
			Name:   "registry_address",
			EnvVar: "MICRO_REGISTRY_ADDRESS",
			Usage:  "Comma-separated list of registry addresses",
		},
		cli.StringFlag{
			Name:   "selector",
			EnvVar: "MICRO_SELECTOR",
			Usage:  "Selector used to pick nodes for querying",
		},
		cli.StringFlag{
			Name:   "transport",
			EnvVar: "MICRO_TRANSPORT",
			Usage:  "Transport mechanism used; http, rabbitmq, nats",
		},
		cli.StringFlag{
			Name:   "transport_address",
			EnvVar: "MICRO_TRANSPORT_ADDRESS",
			Usage:  "Comma-separated list of transport addresses",
		},
	}

	DefaultBrokers = map[string]func(...broker.Option) broker.Broker{
		"http": http.NewBroker,
		"mqtt": mqtt.NewBroker,
	}

	DefaultRegistries = map[string]func(...registry.Option) registry.Registry{
		"consul": consul.NewRegistry,
		"mdns":   mdns.NewRegistry,
	}

	DefaultSelectors = map[string]func(...selector.Option) selector.Selector{
		"default": selector.NewSelector,
		"cache":     cache.NewSelector,
		"blacklist": blacklist.NewSelector,
	}

	DefaultTransports = map[string]func(...transport.Option) transport.Transport{
		"http": thttp.NewTransport,
	}

	// used for default selection as the fall back
	defaultBroker    = "http"
	defaultRegistry  = "consul"
	defaultSelector  = "default"
	defaultTransport = "http"
)

func init() {
	rand.Seed(time.Now().Unix())
	help := cli.HelpPrinter
	cli.HelpPrinter = func(writer io.Writer, templ string, data interface{}) {
		help(writer, templ, data)
		os.Exit(0)
	}
}

func newCmd(opts ...Option) Cmd {
	options := Options{
		Broker:    &broker.DefaultBroker,
		Client:    &client.DefaultClient,
		Registry:  &registry.DefaultRegistry,
		Server:    &server.DefaultServer,
		Selector:  &selector.DefaultSelector,
		Transport: &transport.DefaultTransport,

		Brokers:    DefaultBrokers,
		Registries: DefaultRegistries,
		Selectors:  DefaultSelectors,
		Transports: DefaultTransports,
	}

	for _, o := range opts {
		o(&options)
	}

	if len(options.Description) == 0 {
		options.Description = "a go-micro service"
	}

	cmd := new(cmd)
	cmd.opts = options
	cmd.app = cli.NewApp()
	cmd.app.Name = cmd.opts.Name
	cmd.app.Version = cmd.opts.Version
	cmd.app.Usage = cmd.opts.Description
	cmd.app.Before = cmd.Before
	cmd.app.Flags = DefaultFlags
	cmd.app.Action = func(c *cli.Context) {}

	if len(options.Version) == 0 {
		cmd.app.HideVersion = true
	}

	return cmd
}

func (c *cmd) App() *cli.App {
	return c.app
}

func (c *cmd) Options() Options {
	return c.opts
}

func (c *cmd) Before(ctx *cli.Context) error {
	// If flags are set then use them otherwise do nothing
	var serverOpts []server.Option
	var clientOpts []client.Option

	// Set the broker
	if name := ctx.String("broker"); len(name) > 0 || len(ctx.String("broker_address")) > 0 {
		if len(name) == 0 {
			name = defaultBroker
		}

		if b, ok := c.opts.Brokers[name]; ok {
			n := b(broker.Addrs(strings.Split(ctx.String("broker_address"), ",")...))
			*c.opts.Broker = n
		} else {
			return fmt.Errorf("Broker %s not found", name)
		}

		serverOpts = append(serverOpts, server.Broker(*c.opts.Broker))
		clientOpts = append(clientOpts, client.Broker(*c.opts.Broker))

	}

	// Set the registry
	if name := ctx.String("registry"); len(name) > 0 || len(ctx.String("registry_address")) > 0 {
		if len(name) == 0 {
			name = defaultRegistry
		}

		if r, ok := c.opts.Registries[name]; ok {
			n := r(registry.Addrs(strings.Split(ctx.String("registry_address"), ",")...))
			*c.opts.Registry = n
		} else {
			return fmt.Errorf("Registry %s not found", name)
		}

		serverOpts = append(serverOpts, server.Registry(*c.opts.Registry))
		clientOpts = append(clientOpts, client.Registry(*c.opts.Registry))

		(*c.opts.Selector).Init(selector.Registry(*c.opts.Registry))
		clientOpts = append(clientOpts, client.Selector(*c.opts.Selector))

		(*c.opts.Broker).Init(broker.Registry(*c.opts.Registry))
	}

	// Set the selector
	if name := ctx.String("selector"); len(name) > 0 {
		if s, ok := c.opts.Selectors[name]; ok {
			n := s(selector.Registry(*c.opts.Registry))
			*c.opts.Selector = n
		} else {
			return fmt.Errorf("Selector %s not found", name)
		}

		// No server option here. Should there be?
		clientOpts = append(clientOpts, client.Selector(*c.opts.Selector))
	}

	// Set the transport
	if name := ctx.String("transport"); len(name) > 0 || len(ctx.String("transport_address")) > 0 {
		if len(name) == 0 {
			name = defaultTransport
		}

		if t, ok := c.opts.Transports[name]; ok {
			n := t(transport.Addrs(strings.Split(ctx.String("transport_address"), ",")...))
			*c.opts.Transport = n
		} else {
			return fmt.Errorf("Transport %s not found", name)
		}

		serverOpts = append(serverOpts, server.Transport(*c.opts.Transport))
		clientOpts = append(clientOpts, client.Transport(*c.opts.Transport))
	}

	// Parse the server options
	metadata := make(map[string]string)
	for _, d := range ctx.StringSlice("server_metadata") {
		var key, val string
		parts := strings.Split(d, "=")
		key = parts[0]
		if len(parts) > 1 {
			val = strings.Join(parts[1:], "=")
		}
		metadata[key] = val
	}

	if len(metadata) > 0 {
		serverOpts = append(serverOpts, server.Metadata(metadata))
	}

	if len(ctx.String("server_name")) > 0 {
		serverOpts = append(serverOpts, server.Name(ctx.String("server_name")))
	}

	if len(ctx.String("server_version")) > 0 {
		serverOpts = append(serverOpts, server.Version(ctx.String("server_version")))
	}

	if len(ctx.String("server_id")) > 0 {
		serverOpts = append(serverOpts, server.Id(ctx.String("server_id")))
	}

	if len(ctx.String("server_address")) > 0 {
		serverOpts = append(serverOpts, server.Address(ctx.String("server_address")))
	}

	if len(ctx.String("server_advertise")) > 0 {
		serverOpts = append(serverOpts, server.Advertise(ctx.String("server_advertise")))
	}

	// client opts
	if r := ctx.Int("client_retries"); r > 0 {
		clientOpts = append(clientOpts, client.Retries(r))
	}

	if t := ctx.String("client_request_timeout"); len(t) > 0 {
		d, err := time.ParseDuration(t)
		if err != nil {
			return fmt.Errorf("failed to parse client_request_timeout: %v", t)
		}
		clientOpts = append(clientOpts, client.RequestTimeout(d))
	}

	// We have some command line opts for the server.
	// Lets set it up
	if len(serverOpts) > 0 {
		(*c.opts.Server).Init(serverOpts...)
	}

	// Use an init option?
	if len(clientOpts) > 0 {
		(*c.opts.Client).Init(clientOpts...)
	}

	return nil
}

func (c *cmd) Init(opts ...Option) error {
	for _, o := range opts {
		o(&c.opts)
	}
	c.app.Name = c.opts.Name
	c.app.Version = c.opts.Version
	c.app.Usage = c.opts.Description
	c.app.RunAndExitOnError()
	return nil
}

func DefaultOptions() Options {
	return DefaultCmd.Options()
}

func App() *cli.App {
	return DefaultCmd.App()
}

func Init(opts ...Option) error {
	return DefaultCmd.Init(opts...)
}

func NewCmd(opts ...Option) Cmd {
	return newCmd(opts...)
}
<|endoftext|>


<commit_before>package cmd

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
)

type NotDefinedError string

func (suffix NotDefinedError) Error() string {
	return fmt.Sprintf("metadata: GCE metadata %q not defined", string(suffix))
}

func Get(suffix string) (string, error) {
	val, _, err := getETag(suffix)
	return val, err
}

func getETag(suffix string) (value, etag string, err error) {
	// Using a fixed IP makes it very difficult to spoof the metadata service in
	// a container, which is an important use-case for local testing of cloud
	// deployments. To enable spoofing of the metadata service, the environment
	// variable GCE_METADATA_HOST is first inspected to decide where metadata
	// requests shall go.
	client := &http.Client{}
	host := os.Getenv("GCE_METADATA_HOST")
	if host == "" {
		// Using 169.254.169.254 instead of "metadata" here because Go
		// binaries built with the "netgo" tag and without cgo won't
		// know the search suffix for "metadata" is
		// ".google.internal", and this IP address is documented as
		// being stable anyway.
		host = "169.254.169.254"
	}
	url := "http://" + host + "/computeMetadata/v1/" + suffix
	req, _ := http.NewRequest("GET", url, nil)
	req.Header.Set("Metadata-Flavor", "Google")
	res, err := client.Do(req)
	if err != nil {
		return "", "", err
	}
	defer res.Body.Close()
	if res.StatusCode == http.StatusNotFound {
		return "", "", NotDefinedError(suffix)
	}
	if res.StatusCode != 200 {
		return "", "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url)
	}
	all, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return "", "", err
	}
	return string(all), res.Header.Get("Etag"), nil
}

func getTrimmed(suffix string) (s string, err error) {
	s, err = Get(suffix)
	s = strings.TrimSpace(s)
	return
}

func lines(suffix string) ([]string, error) {
	j, err := Get(suffix)
	if err != nil {
		return nil, err
	}
	s := strings.Split(strings.TrimSpace(j), "\n")
	for i := range s {
		s[i] = strings.TrimSpace(s[i])
	}
	return s, nil
}

// ProjectID returns the current instance's project ID string.
func ProjectID() (string, error) {
	return getTrimmed("project/project-id")
}

// NumericProjectID returns the current instance's numeric project ID.
func NumericProjectID() (string, error) {
	return getTrimmed("project/numeric-project-id")
}

// InternalIP returns the instance's primary internal IP address.
func InternalIP() (string, error) {
	return getTrimmed("instance/network-interfaces/0/ip")
}

// ExternalIP returns the instance's primary external (public) IP address.
func ExternalIP() (string, error) {
	return getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip")
}

// Hostname returns the instance's hostname. This will be of the form
// "<instanceID>.c.<projID>.internal".
func Hostname() (string, error) {
	return getTrimmed("instance/hostname")
}

// MachineType returns the instance's machine type.
func MachineType() (string, error) {
	machine, err := getTrimmed("instance/machine-type")
	// machine-type is of the form "projects/<projNum>/machineTypes/<machine-typeName>
	if err != nil {
		return "", err
	}
	return machine[strings.LastIndex(machine, "/")+1:], nil
}

// Description returns the instance's description.
func Description() (string, error) {
	return getTrimmed("instance/description")
}

// InstanceTags returns the list of user-defined instance tags,
// assigned when initially creating a GCE instance.
func InstanceTags() ([]string, error) {
	var s []string
	j, err := Get("instance/tags")
	if err != nil {
		return nil, err
	}
	if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil {
		return nil, err
	}
	return s, nil
}

// InstanceID returns the current VM's numeric instance ID.
func InstanceID() (string, error) {
	return getTrimmed("instance/id")
}

// InstanceName returns the current VM's instance ID string.
func InstanceName() (string, error) {
	host, err := Hostname()
	if err != nil {
		return "", err
	}
	return strings.Split(host, ".")[0], nil
}

// Zone returns the current VM's zone, such as "us-central1-b".
func Zone() (string, error) {
	zone, err := getTrimmed("instance/zone")
	// zone is of the form "projects/<projNum>/zones/<zoneName>".
	if err != nil {
		return "", err
	}
	return zone[strings.LastIndex(zone, "/")+1:], nil
}

// InstanceAttributes returns the list of user-defined attributes,
// assigned when initially creating a GCE VM instance. The value of an
// attribute can be obtained with InstanceAttributeValue.
func InstanceAttributes() ([]string, error) { return lines("instance/attributes/") }

// InstanceAttributeValue returns the value of the provided VM
// instance attribute.
//
// If the requested attribute is not defined, the returned error will
// be of type NotDefinedError.
//
// InstanceAttributeValue may return ("", nil) if the attribute was
// defined to be the empty string.
func InstanceAttributeValue(attr string) (string, error) {
	return Get("instance/attributes/" + attr)
}
<commit_msg>Made more simple Get function<commit_after>package cmd

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
)

type NotDefinedError string

func (suffix NotDefinedError) Error() string {
	return fmt.Sprintf("metadata: GCE metadata %q not defined", string(suffix))
}

//func Get(suffix string) (string, error) {
//	val, _, err := getETag(suffix)
//	return val, err
//}

func Get(suffix string) (value, err error) {
	client := &http.Client{}
	host = "169.254.169.254"
	url := "http://" + host + "/computeMetadata/v1/" + suffix
	req, _ := http.NewRequest("GET", url, nil)
	req.Header.Set("Metadata-Flavor", "Google")
	res, err := client.Do(req)
	if err != nil {
		return "", "", err
	}
	defer res.Body.Close()
	if res.StatusCode == http.StatusNotFound {
		return "", "", NotDefinedError(suffix)
	}
	if res.StatusCode != 200 {
		return "", "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url)
	}
	all, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return "", "", err
	}
	return string(all), nil
}

func getTrimmed(suffix string) (s string, err error) {
	s, err = Get(suffix)
	s = strings.TrimSpace(s)
	return
}

func lines(suffix string) ([]string, error) {
	j, err := Get(suffix)
	if err != nil {
		return nil, err
	}
	s := strings.Split(strings.TrimSpace(j), "\n")
	for i := range s {
		s[i] = strings.TrimSpace(s[i])
	}
	return s, nil
}

// ProjectID returns the current instance's project ID string.
func ProjectID() (string, error) {
	return getTrimmed("project/project-id")
}

// NumericProjectID returns the current instance's numeric project ID.
func NumericProjectID() (string, error) {
	return getTrimmed("project/numeric-project-id")
}

// InternalIP returns the instance's primary internal IP address.
func InternalIP() (string, error) {
	return getTrimmed("instance/network-interfaces/0/ip")
}

// ExternalIP returns the instance's primary external (public) IP address.
func ExternalIP() (string, error) {
	return getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip")
}

// Hostname returns the instance's hostname. This will be of the form
// "<instanceID>.c.<projID>.internal".
func Hostname() (string, error) {
	return getTrimmed("instance/hostname")
}

// MachineType returns the instance's machine type.
func MachineType() (string, error) {
	machine, err := getTrimmed("instance/machine-type")
	// machine-type is of the form "projects/<projNum>/machineTypes/<machine-typeName>
	if err != nil {
		return "", err
	}
	return machine[strings.LastIndex(machine, "/")+1:], nil
}

// Description returns the instance's description.
func Description() (string, error) {
	return getTrimmed("instance/description")
}

// InstanceTags returns the list of user-defined instance tags,
// assigned when initially creating a GCE instance.
func InstanceTags() ([]string, error) {
	var s []string
	j, err := Get("instance/tags")
	if err != nil {
		return nil, err
	}
	if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil {
		return nil, err
	}
	return s, nil
}

// InstanceID returns the current VM's numeric instance ID.
func InstanceID() (string, error) {
	return getTrimmed("instance/id")
}

// InstanceName returns the current VM's instance ID string.
func InstanceName() (string, error) {
	host, err := Hostname()
	if err != nil {
		return "", err
	}
	return strings.Split(host, ".")[0], nil
}

// Zone returns the current VM's zone, such as "us-central1-b".
func Zone() (string, error) {
	zone, err := getTrimmed("instance/zone")
	// zone is of the form "projects/<projNum>/zones/<zoneName>".
	if err != nil {
		return "", err
	}
	return zone[strings.LastIndex(zone, "/")+1:], nil
}

// InstanceAttributes returns the list of user-defined attributes,
// assigned when initially creating a GCE VM instance. The value of an
// attribute can be obtained with InstanceAttributeValue.
func InstanceAttributes() ([]string, error) { return lines("instance/attributes/") }

// InstanceAttributeValue returns the value of the provided VM
// instance attribute.
//
// If the requested attribute is not defined, the returned error will
// be of type NotDefinedError.
//
// InstanceAttributeValue may return ("", nil) if the attribute was
// defined to be the empty string.
func InstanceAttributeValue(attr string) (string, error) {
	return Get("instance/attributes/" + attr)
}
<|endoftext|>


<commit_before>package vault

import (
	"archive/zip"
	"bytes"
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"testing"
	"time"

	"github.com/hashicorp/go-version"
	"github.com/hashicorp/nomad/command/agent"
	"github.com/hashicorp/nomad/helper"
	"github.com/hashicorp/nomad/nomad/structs/config"
	"github.com/hashicorp/nomad/testutil"
	vapi "github.com/hashicorp/vault/api"
	"github.com/stretchr/testify/require"
	"golang.org/x/sync/errgroup"
)

var (
	integration = flag.Bool("integration", false, "run integration tests")
	minVaultVer = version.Must(version.NewVersion("0.6.2"))
)

// syncVault discovers available versions of Vault, downloads the binaries,
// returns a map of version to binary path as well as a sorted list of
// versions.
func syncVault(t *testing.T) ([]*version.Version, map[string]string) {

	binDir := filepath.Join(os.TempDir(), "vault-bins/")

	urls := vaultVersions(t)

	sorted, versions, err := pruneVersions(urls)
	require.NoError(t, err)

	// Get the binaries we need to download
	missing, err := missingVault(binDir, versions)
	require.NoError(t, err)

	// Create the directory for the binaries
	require.NoError(t, createBinDir(binDir))

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()

	// Limit to N concurrent downloads
	sema := make(chan int, 5)

	// Download in parallel
	start := time.Now()
	g, _ := errgroup.WithContext(ctx)
	for ver, url := range missing {
		dst := filepath.Join(binDir, ver)
		url := url
		g.Go(func() error {
			sema <- 1
			defer func() {
				<-sema
			}()
			return getVault(dst, url)
		})
	}
	require.NoError(t, g.Wait())
	if n := len(missing); n > 0 {
		t.Logf("Downloaded %d versions of Vault in %s", n, time.Now().Sub(start))
	}

	binaries := make(map[string]string, len(versions))
	for ver, _ := range versions {
		binaries[ver] = filepath.Join(binDir, ver)
	}
	return sorted, binaries
}

// vaultVersions discovers available Vault versions from releases.hashicorp.com
// and returns a map of version to url.
func vaultVersions(t *testing.T) map[string]string {
	resp, err := http.Get("https://releases.hashicorp.com/vault/index.json")
	require.NoError(t, err)

	respJson := struct {
		Versions map[string]struct {
			Builds []struct {
				Version string `json:"version"`
				Os      string `json:"os"`
				Arch    string `json:"arch"`
				URL     string `json:"url"`
			} `json:"builds"`
		}
	}{}
	require.NoError(t, json.NewDecoder(resp.Body).Decode(&respJson))
	require.NoError(t, resp.Body.Close())

	versions := map[string]string{}
	for vk, vv := range respJson.Versions {
		gover, err := version.NewVersion(vk)
		if err != nil {
			t.Logf("error parsing Vault version %q -> %v", vk, err)
			continue
		}

		// Skip ancient versions
		if gover.LessThan(minVaultVer) {
			continue
		}

		// Skip prerelease and enterprise versions
		if gover.Prerelease() != "" || gover.Metadata() != "" {
			continue
		}

		url := ""
		for _, b := range vv.Builds {
			buildver, err := version.NewVersion(b.Version)
			if err != nil {
				t.Logf("error parsing Vault build version %q -> %v", b.Version, err)
				continue
			}

			if buildver.Prerelease() != "" {
				continue
			}

			if buildver.Metadata() != "" {
				continue
			}

			if b.Os != runtime.GOOS {
				continue
			}

			if b.Arch != runtime.GOARCH {
				continue
			}

			// Match!
			url = b.URL
			break
		}

		if url != "" {
			versions[vk] = url
		}
	}

	return versions
}

// pruneVersions only takes the latest Z for each X.Y.Z release. Returns a
// sorted list and map of kept versions.
func pruneVersions(all map[string]string) ([]*version.Version, map[string]string, error) {
	if len(all) == 0 {
		return nil, nil, fmt.Errorf("0 Vault versions")
	}

	sorted := make([]*version.Version, 0, len(all))

	for k := range all {
		sorted = append(sorted, version.Must(version.NewVersion(k)))
	}

	sort.Sort(version.Collection(sorted))

	keep := make([]*version.Version, 0, len(all))

	for _, v := range sorted {
		segments := v.Segments()
		if len(segments) < 3 {
			// Drop malformed versions
			continue
		}

		if len(keep) == 0 {
			keep = append(keep, v)
			continue
		}

		last := keep[len(keep)-1].Segments()

		if segments[0] == last[0] && segments[1] == last[1] {
			// current X.Y == last X.Y, replace last with current
			keep[len(keep)-1] = v
		} else {
			// current X.Y != last X.Y, append
			keep = append(keep, v)
		}
	}

	// Create a new map of canonicalized versions to urls
	urls := make(map[string]string, len(keep))
	for _, v := range keep {
		origURL := all[v.Original()]
		if origURL == "" {
			return nil, nil, fmt.Errorf("missing version %s", v.Original())
		}
		urls[v.String()] = origURL
	}

	return keep, urls, nil
}

// createBinDir creates the binary directory
func createBinDir(binDir string) error {
	// Check if the directory exists, otherwise create it
	f, err := os.Stat(binDir)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("failed to stat directory: %v", err)
	}

	if f != nil && f.IsDir() {
		return nil
	} else if f != nil {
		if err := os.RemoveAll(binDir); err != nil {
			return fmt.Errorf("failed to remove file at directory path: %v", err)
		}
	}

	// Create the directory
	if err := os.Mkdir(binDir, 075); err != nil {
		return fmt.Errorf("failed to make directory: %v", err)
	}
	if err := os.Chmod(binDir, 0755); err != nil {
		return fmt.Errorf("failed to chmod: %v", err)
	}

	return nil
}

// missingVault returns the binaries that must be downloaded. versions key must
// be the Vault version.
func missingVault(binDir string, versions map[string]string) (map[string]string, error) {
	files, err := ioutil.ReadDir(binDir)
	if err != nil {
		if os.IsNotExist(err) {
			return versions, nil
		}

		return nil, fmt.Errorf("failed to stat directory: %v", err)
	}

	// Copy versions so we don't mutate it
	missingSet := make(map[string]string, len(versions))
	for k, v := range versions {
		missingSet[k] = v
	}

	for _, f := range files {
		delete(missingSet, f.Name())
	}

	return missingSet, nil
}

// getVault downloads the given Vault binary
func getVault(dst, url string) error {
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Wrap in an in-mem buffer
	b := bytes.NewBuffer(nil)
	if _, err := io.Copy(b, resp.Body); err != nil {
		return fmt.Errorf("error reading response body: %v", err)
	}
	resp.Body.Close()

	zreader, err := zip.NewReader(bytes.NewReader(b.Bytes()), resp.ContentLength)
	if err != nil {
		return err
	}

	if l := len(zreader.File); l != 1 {
		return fmt.Errorf("unexpected number of files in zip: %v", l)
	}

	// Copy the file to its destination
	out, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0777)
	if err != nil {
		return err
	}
	defer out.Close()

	zfile, err := zreader.File[0].Open()
	if err != nil {
		return fmt.Errorf("failed to open zip file: %v", err)
	}

	if _, err := io.Copy(out, zfile); err != nil {
		return fmt.Errorf("failed to decompress file to destination: %v", err)
	}

	return nil
}

// TestVaultCompatibility tests compatibility across Vault versions
func TestVaultCompatibility(t *testing.T) {
	if !*integration {
		t.Skip("skipping test in non-integration mode: add -integration flag to run")
	}

	sorted, vaultBinaries := syncVault(t)

	for _, v := range sorted {
		ver := v.String()
		bin := vaultBinaries[ver]
		require.NotZerof(t, bin, "missing version: %s", ver)
		t.Run(ver, func(t *testing.T) {
			testVaultCompatibility(t, bin, ver)
		})
	}
}

// testVaultCompatibility tests compatibility with the given vault binary
func testVaultCompatibility(t *testing.T, vault string, version string) {
	require := require.New(t)

	// Create a Vault server
	v := testutil.NewTestVaultFromPath(t, vault)
	defer v.Stop()

	token := setupVault(t, v.Client, version)

	// Create a Nomad agent using the created vault
	nomad := agent.NewTestAgent(t, t.Name(), func(c *agent.Config) {
		if c.Vault == nil {
			c.Vault = &config.VaultConfig{}
		}
		c.Vault.Enabled = helper.BoolToPtr(true)
		c.Vault.Token = token
		c.Vault.Role = "nomad-cluster"
		c.Vault.AllowUnauthenticated = helper.BoolToPtr(true)
		c.Vault.Addr = v.HTTPAddr
	})
	defer nomad.Shutdown()

	// Submit the Nomad job that requests a Vault token and cats that the Vault
	// token is there
	c := nomad.Client()
	j := c.Jobs()
	_, _, err := j.Register(job, nil)
	require.NoError(err)

	// Wait for there to be an allocation terminated successfully
	//var allocID string
	testutil.WaitForResult(func() (bool, error) {
		// Get the allocations for the job
		allocs, _, err := j.Allocations(*job.ID, false, nil)
		if err != nil {
			return false, err
		}
		l := len(allocs)
		switch l {
		case 0:
			return false, fmt.Errorf("want one alloc; got zero")
		case 1:
		default:
			// exit early
			t.Fatalf("too many allocations; something failed")
		}
		alloc := allocs[0]
		//allocID = alloc.ID
		if alloc.ClientStatus == "complete" {
			return true, nil
		}

		return false, fmt.Errorf("client status %q", alloc.ClientStatus)
	}, func(err error) {
		t.Fatalf("allocation did not finish: %v", err)
	})

}

// setupVault takes the Vault client and creates the required policies and
// roles. It returns the token that should be used by Nomad
func setupVault(t *testing.T, client *vapi.Client, vaultVersion string) string {
	// Write the policy
	sys := client.Sys()

	// pre-0.9.0 vault servers do not work with our new vault client for the policy endpoint
	// perform this using a raw HTTP request
	newApi := version.Must(version.NewVersion("0.9.0"))
	testVersion := version.Must(version.NewVersion(vaultVersion))
	if testVersion.LessThan(newApi) {
		body := map[string]string{
			"rules": policy,
		}
		request := client.NewRequest("PUT", "/v1/sys/policy/nomad-server")
		if err := request.SetJSONBody(body); err != nil {
			t.Fatalf("failed to set JSON body on legacy policy creation: %v", err)
		}
		if _, err := client.RawRequest(request); err != nil {
			t.Fatalf("failed to create legacy policy: %v", err)
		}
	} else {
		if err := sys.PutPolicy("nomad-server", policy); err != nil {
			t.Fatalf("failed to create policy: %v", err)
		}
	}

	// Build the role
	l := client.Logical()
	l.Write("auth/token/roles/nomad-cluster", role)

	// Create a new token with the role
	a := client.Auth().Token()
	req := vapi.TokenCreateRequest{
		Policies: []string{"nomad-server"},
		Period:   "72h",
		NoParent: true,
	}
	s, err := a.Create(&req)
	if err != nil {
		t.Fatalf("failed to create child token: %v", err)
	}

	// Get the client token
	if s == nil || s.Auth == nil {
		t.Fatalf("bad secret response: %+v", s)
	}

	return s.Auth.ClientToken
}
<commit_msg>test: remove errgroup to take advantage of vet<commit_after>package vault

import (
	"archive/zip"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"testing"
	"time"

	"github.com/hashicorp/go-version"
	"github.com/hashicorp/nomad/command/agent"
	"github.com/hashicorp/nomad/helper"
	"github.com/hashicorp/nomad/nomad/structs/config"
	"github.com/hashicorp/nomad/testutil"
	vapi "github.com/hashicorp/vault/api"
	"github.com/stretchr/testify/require"
)

var (
	integration = flag.Bool("integration", false, "run integration tests")
	minVaultVer = version.Must(version.NewVersion("0.6.2"))
)

// syncVault discovers available versions of Vault, downloads the binaries,
// returns a map of version to binary path as well as a sorted list of
// versions.
func syncVault(t *testing.T) ([]*version.Version, map[string]string) {

	binDir := filepath.Join(os.TempDir(), "vault-bins/")

	urls := vaultVersions(t)

	sorted, versions, err := pruneVersions(urls)
	require.NoError(t, err)

	// Get the binaries we need to download
	missing, err := missingVault(binDir, versions)
	require.NoError(t, err)

	// Create the directory for the binaries
	require.NoError(t, createBinDir(binDir))

	// Download in parallel
	start := time.Now()
	errCh := make(chan error, len(missing))
	for ver, url := range missing {
		dst := filepath.Join(binDir, ver)
		url := url
		go func() {
			errCh <- getVault(dst, url)
		}()
	}
	for i := 0; i < len(missing); i++ {
		select {
		case err := <-errCh:
			require.NoError(t, err)
		case <-time.After(5 * time.Minute):
			t.Fatalf("timed out downloading Vault binaries")
		}
	}
	if n := len(missing); n > 0 {
		t.Logf("Downloaded %d versions of Vault in %s", n, time.Now().Sub(start))
	}

	binaries := make(map[string]string, len(versions))
	for ver, _ := range versions {
		binaries[ver] = filepath.Join(binDir, ver)
	}
	return sorted, binaries
}

// vaultVersions discovers available Vault versions from releases.hashicorp.com
// and returns a map of version to url.
func vaultVersions(t *testing.T) map[string]string {
	resp, err := http.Get("https://releases.hashicorp.com/vault/index.json")
	require.NoError(t, err)

	respJson := struct {
		Versions map[string]struct {
			Builds []struct {
				Version string `json:"version"`
				Os      string `json:"os"`
				Arch    string `json:"arch"`
				URL     string `json:"url"`
			} `json:"builds"`
		}
	}{}
	require.NoError(t, json.NewDecoder(resp.Body).Decode(&respJson))
	require.NoError(t, resp.Body.Close())

	versions := map[string]string{}
	for vk, vv := range respJson.Versions {
		gover, err := version.NewVersion(vk)
		if err != nil {
			t.Logf("error parsing Vault version %q -> %v", vk, err)
			continue
		}

		// Skip ancient versions
		if gover.LessThan(minVaultVer) {
			continue
		}

		// Skip prerelease and enterprise versions
		if gover.Prerelease() != "" || gover.Metadata() != "" {
			continue
		}

		url := ""
		for _, b := range vv.Builds {
			buildver, err := version.NewVersion(b.Version)
			if err != nil {
				t.Logf("error parsing Vault build version %q -> %v", b.Version, err)
				continue
			}

			if buildver.Prerelease() != "" {
				continue
			}

			if buildver.Metadata() != "" {
				continue
			}

			if b.Os != runtime.GOOS {
				continue
			}

			if b.Arch != runtime.GOARCH {
				continue
			}

			// Match!
			url = b.URL
			break
		}

		if url != "" {
			versions[vk] = url
		}
	}

	return versions
}

// pruneVersions only takes the latest Z for each X.Y.Z release. Returns a
// sorted list and map of kept versions.
func pruneVersions(all map[string]string) ([]*version.Version, map[string]string, error) {
	if len(all) == 0 {
		return nil, nil, fmt.Errorf("0 Vault versions")
	}

	sorted := make([]*version.Version, 0, len(all))

	for k := range all {
		sorted = append(sorted, version.Must(version.NewVersion(k)))
	}

	sort.Sort(version.Collection(sorted))

	keep := make([]*version.Version, 0, len(all))

	for _, v := range sorted {
		segments := v.Segments()
		if len(segments) < 3 {
			// Drop malformed versions
			continue
		}

		if len(keep) == 0 {
			keep = append(keep, v)
			continue
		}

		last := keep[len(keep)-1].Segments()

		if segments[0] == last[0] && segments[1] == last[1] {
			// current X.Y == last X.Y, replace last with current
			keep[len(keep)-1] = v
		} else {
			// current X.Y != last X.Y, append
			keep = append(keep, v)
		}
	}

	// Create a new map of canonicalized versions to urls
	urls := make(map[string]string, len(keep))
	for _, v := range keep {
		origURL := all[v.Original()]
		if origURL == "" {
			return nil, nil, fmt.Errorf("missing version %s", v.Original())
		}
		urls[v.String()] = origURL
	}

	return keep, urls, nil
}

// createBinDir creates the binary directory
func createBinDir(binDir string) error {
	// Check if the directory exists, otherwise create it
	f, err := os.Stat(binDir)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("failed to stat directory: %v", err)
	}

	if f != nil && f.IsDir() {
		return nil
	} else if f != nil {
		if err := os.RemoveAll(binDir); err != nil {
			return fmt.Errorf("failed to remove file at directory path: %v", err)
		}
	}

	// Create the directory
	if err := os.Mkdir(binDir, 075); err != nil {
		return fmt.Errorf("failed to make directory: %v", err)
	}
	if err := os.Chmod(binDir, 0755); err != nil {
		return fmt.Errorf("failed to chmod: %v", err)
	}

	return nil
}

// missingVault returns the binaries that must be downloaded. versions key must
// be the Vault version.
func missingVault(binDir string, versions map[string]string) (map[string]string, error) {
	files, err := ioutil.ReadDir(binDir)
	if err != nil {
		if os.IsNotExist(err) {
			return versions, nil
		}

		return nil, fmt.Errorf("failed to stat directory: %v", err)
	}

	// Copy versions so we don't mutate it
	missingSet := make(map[string]string, len(versions))
	for k, v := range versions {
		missingSet[k] = v
	}

	for _, f := range files {
		delete(missingSet, f.Name())
	}

	return missingSet, nil
}

// getVault downloads the given Vault binary
func getVault(dst, url string) error {
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Wrap in an in-mem buffer
	b := bytes.NewBuffer(nil)
	if _, err := io.Copy(b, resp.Body); err != nil {
		return fmt.Errorf("error reading response body: %v", err)
	}
	resp.Body.Close()

	zreader, err := zip.NewReader(bytes.NewReader(b.Bytes()), resp.ContentLength)
	if err != nil {
		return err
	}

	if l := len(zreader.File); l != 1 {
		return fmt.Errorf("unexpected number of files in zip: %v", l)
	}

	// Copy the file to its destination
	out, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0777)
	if err != nil {
		return err
	}
	defer out.Close()

	zfile, err := zreader.File[0].Open()
	if err != nil {
		return fmt.Errorf("failed to open zip file: %v", err)
	}

	if _, err := io.Copy(out, zfile); err != nil {
		return fmt.Errorf("failed to decompress file to destination: %v", err)
	}

	return nil
}

// TestVaultCompatibility tests compatibility across Vault versions
func TestVaultCompatibility(t *testing.T) {
	if !*integration {
		t.Skip("skipping test in non-integration mode: add -integration flag to run")
	}

	sorted, vaultBinaries := syncVault(t)

	for _, v := range sorted {
		ver := v.String()
		bin := vaultBinaries[ver]
		require.NotZerof(t, bin, "missing version: %s", ver)
		t.Run(ver, func(t *testing.T) {
			testVaultCompatibility(t, bin, ver)
		})
	}
}

// testVaultCompatibility tests compatibility with the given vault binary
func testVaultCompatibility(t *testing.T, vault string, version string) {
	require := require.New(t)

	// Create a Vault server
	v := testutil.NewTestVaultFromPath(t, vault)
	defer v.Stop()

	token := setupVault(t, v.Client, version)

	// Create a Nomad agent using the created vault
	nomad := agent.NewTestAgent(t, t.Name(), func(c *agent.Config) {
		if c.Vault == nil {
			c.Vault = &config.VaultConfig{}
		}
		c.Vault.Enabled = helper.BoolToPtr(true)
		c.Vault.Token = token
		c.Vault.Role = "nomad-cluster"
		c.Vault.AllowUnauthenticated = helper.BoolToPtr(true)
		c.Vault.Addr = v.HTTPAddr
	})
	defer nomad.Shutdown()

	// Submit the Nomad job that requests a Vault token and cats that the Vault
	// token is there
	c := nomad.Client()
	j := c.Jobs()
	_, _, err := j.Register(job, nil)
	require.NoError(err)

	// Wait for there to be an allocation terminated successfully
	//var allocID string
	testutil.WaitForResult(func() (bool, error) {
		// Get the allocations for the job
		allocs, _, err := j.Allocations(*job.ID, false, nil)
		if err != nil {
			return false, err
		}
		l := len(allocs)
		switch l {
		case 0:
			return false, fmt.Errorf("want one alloc; got zero")
		case 1:
		default:
			// exit early
			t.Fatalf("too many allocations; something failed")
		}
		alloc := allocs[0]
		//allocID = alloc.ID
		if alloc.ClientStatus == "complete" {
			return true, nil
		}

		return false, fmt.Errorf("client status %q", alloc.ClientStatus)
	}, func(err error) {
		t.Fatalf("allocation did not finish: %v", err)
	})

}

// setupVault takes the Vault client and creates the required policies and
// roles. It returns the token that should be used by Nomad
func setupVault(t *testing.T, client *vapi.Client, vaultVersion string) string {
	// Write the policy
	sys := client.Sys()

	// pre-0.9.0 vault servers do not work with our new vault client for the policy endpoint
	// perform this using a raw HTTP request
	newApi := version.Must(version.NewVersion("0.9.0"))
	testVersion := version.Must(version.NewVersion(vaultVersion))
	if testVersion.LessThan(newApi) {
		body := map[string]string{
			"rules": policy,
		}
		request := client.NewRequest("PUT", "/v1/sys/policy/nomad-server")
		if err := request.SetJSONBody(body); err != nil {
			t.Fatalf("failed to set JSON body on legacy policy creation: %v", err)
		}
		if _, err := client.RawRequest(request); err != nil {
			t.Fatalf("failed to create legacy policy: %v", err)
		}
	} else {
		if err := sys.PutPolicy("nomad-server", policy); err != nil {
			t.Fatalf("failed to create policy: %v", err)
		}
	}

	// Build the role
	l := client.Logical()
	l.Write("auth/token/roles/nomad-cluster", role)

	// Create a new token with the role
	a := client.Auth().Token()
	req := vapi.TokenCreateRequest{
		Policies: []string{"nomad-server"},
		Period:   "72h",
		NoParent: true,
	}
	s, err := a.Create(&req)
	if err != nil {
		t.Fatalf("failed to create child token: %v", err)
	}

	// Get the client token
	if s == nil || s.Auth == nil {
		t.Fatalf("bad secret response: %+v", s)
	}

	return s.Auth.ClientToken
}
<|endoftext|>


<commit_before>package campaign

import (
	"bytes"
	"crypto/rand"
	"encoding/base64"
	"errors"
	"fmt"
	"github.com/supme/gonder/models"
	//	"github.com/eaigner/dkim"
	"golang.org/x/net/idna"
	"golang.org/x/net/proxy"
	"io/ioutil"
	"net"
	"net/http"
	"net/smtp"
	"strings"
	"time"
	"mime"
)

type (
	// Mail parameters and data
	//
	// Iface: send email from selected interface
	//       "": default interface
	//       "12.34.56.78": ip address interface
	//       "socks://12.34.56.78:1234": send from remote socks server
	//       "1,4,6,8,9": id`s for send rotate by selected profile (set Host: "group")
	// Host: host name of selected interface ip
	MailData struct {
		Iface        string
		Host         string
		From_email   string
		From_name    string
		To_email     string
		To_name      string
		Extra_header string
		Subject      string
		Html         string
		Attachments  []Attachment
		conn         net.Conn
		socksConn    proxy.Dialer
		netConn      net.Dialer
	}

	// Attach file to mail
	//
	// Location: folder
	// Name: file
	Attachment struct {
		Location string
		Name     string
	}
)

// Send email
func (m *MailData) Send() error {

	// trim space
	m.To_email = strings.TrimSpace(m.To_email)
	// punycode convert
	splitEmail := strings.Split(m.To_email, "@")
	if len(splitEmail) != 2 {
		return errors.New(fmt.Sprintf("Bad email"))
	}
	domain, err := idna.ToASCII(splitEmail[1])
	if err != nil {
		return errors.New(fmt.Sprintf("Domain name failed: %v", err))
	}
	m.To_email = strings.Split(m.To_email, "@")[0] + "@" + domain

	if m.Iface == "" {
		// default interface
		m.netConn = net.Dialer{}
	} else {
		if m.Iface[0:8] == "socks://" {
			m.Iface = m.Iface[8:]
			var err error
			m.socksConn, err = proxy.SOCKS5("tcp", m.Iface, nil, proxy.FromEnvironment())
			if err != nil {
				return err
			}
		} else {
			connectAddr := net.ParseIP(m.Iface)
			tcpAddr := &net.TCPAddr{
				IP: connectAddr,
			}
			m.netConn = net.Dialer{LocalAddr: tcpAddr}
		}
	}
	start := time.Now()
	//record, err := net.LookupMX(c.domain)
	record, err := models.DomainGetMX(domain)
	if err != nil {
		return err
	}
	lookupTime := time.Since(start)

	start = time.Now()
	var serverMx string
	for i := range record {
		smx := net.JoinHostPort(record[i].Host, "25")
		if m.socksConn != nil {
			m.conn, err = m.socksConn.Dial("tcp", smx)
		} else {
			m.conn, err = m.netConn.Dial("tcp", smx)
		}
		if err == nil {
			serverMx = record[i].Host
			connTime := time.Since(start)
			fmt.Printf("Connect time to %s %s. Lookup time %s.\n\r", domain, connTime, lookupTime)
			break
		}
	}
	if err != nil {
		return err
	}
	defer m.conn.Close()

	// 5 minute by RFC
	m.conn.SetDeadline(time.Now().Add(5 * time.Minute))

	c, err := smtp.NewClient(m.conn, serverMx)
	if err != nil {
		return errors.New(fmt.Sprintf("%v (NewClient)", err))
	}

	if err := c.Hello(m.Host); err != nil {
		return errors.New(fmt.Sprintf("%v (Hello)", err))
	}

	// Set the sender and recipient first
	if err := c.Mail(m.From_email); err != nil {
		return errors.New(fmt.Sprintf("%v (Mail)", err))
	}

	if err := c.Rcpt(m.To_email); err != nil {
		return errors.New(fmt.Sprintf("%v (Rcpt)", err))
	}

	//dkim.New()

	w, err := c.Data()
	if err != nil {
		return errors.New(fmt.Sprintf("%v (Data)", err))
	}
	_, err = fmt.Fprint(w, m.makeMail())
	if err != nil {
		return errors.New(fmt.Sprintf("%v (SendData)", err))
	}

	err = w.Close()
	if err != nil {
		return errors.New(fmt.Sprintf("%v (Close)", err))
	}

	return c.Quit()
}

func (m *MailData) makeMail() string {
	marker := makeMarker()

	var msg bytes.Buffer

	if m.From_name == "" {
		msg.WriteString(`From: ` + m.From_email + "\n")
	} else {
		msg.WriteString(`From: "` + encodeRFC2045(m.From_name) + `" <` + m.From_email + `>` + "\n")
	}

	if m.To_name == "" {
		msg.WriteString(`To: ` + m.To_email + "\n")
	} else {
		msg.WriteString(`To: "` + encodeRFC2045(m.To_name) + `" <` + m.To_email + `>` + "\n")
	}

	// -------------- head ----------------------------------------------------------

	msg.WriteString("Subject: " + encodeRFC2045(m.Subject) + "\n")
	msg.WriteString("MIME-Version: 1.0\n")
	msg.WriteString("X-Mailer: Gonder v" + models.Config.Version + "\n")
	msg.WriteString("Date: " + time.Now().Format(time.RFC1123Z) + "\n")
	msg.WriteString("Content-Type: multipart/mixed;\n	boundary=\"" + marker + "\"\n")
	msg.WriteString(m.Extra_header + "\n\n")
	// ------------- /head ---------------------------------------------------------

	// ------------- body ----------------------------------------------------------
	msg.WriteString("\n--" + marker + "\nContent-Type: text/html; charset=\"utf-8\"\nContent-Transfer-Encoding: 8bit\n\n")
	msg.WriteString(m.Html)
	// ------------ /body ---------------------------------------------------------

	// ----------- attachments ----------------------------------------------------
	for _, file := range m.Attachments {

		msg.WriteString("\n--" + marker)
		//read and encode attachment
		content, err := ioutil.ReadFile(file.Location + file.Name)
		if err != nil {
			fmt.Println(err)
		}
		encoded := base64.StdEncoding.EncodeToString(content)

		//part 3 will be the attachment
		msg.WriteString(fmt.Sprintf("\nContent-Type: %s;\n	name=\"%s\"\nContent-Transfer-Encoding: base64\nContent-Disposition: attachment;\n	filename=\"%s\"\n\n", http.DetectContentType(content), file.Name, file.Name))
		//split the encoded file in lines (doesn't matter, but low enough not to hit a max limit)
		lineMaxLength := 500
		nbrLines := len(encoded) / lineMaxLength
		for i := 0; i < nbrLines; i++ {
			msg.WriteString(encoded[i*lineMaxLength:(i+1)*lineMaxLength] + "\n")
		}

		//append last line in buffer
		msg.WriteString(encoded[nbrLines*lineMaxLength:])
		msg.WriteString("\n")

	}
	// ----------- /attachments ---------------------------------------------------

	return msg.String()
}

func makeMarker() string {
	b := make([]byte, 30)
	rand.Read(b)
	en := base64.StdEncoding // or URLEncoding
	d := make([]byte, en.EncodedLen(len(b)))
	en.Encode(d, b)
	return string(d)
}

func encodeRFC2045(s string) string {
	return mime.BEncoding.Encode("utf-8", s)
}

func encodeRFC2047(s string) string {
	return mime.QEncoding.Encode("utf-8", s)
}
<commit_msg>multipart only if mail has attachments change html content encoding 8bit to base64<commit_after>package campaign

import (
	"bytes"
	"crypto/rand"
	"encoding/base64"
	"errors"
	"fmt"
	"github.com/supme/gonder/models"
	"golang.org/x/net/idna"
	"golang.org/x/net/proxy"
	"io/ioutil"
	"net"
	"net/http"
	"net/smtp"
	"strings"
	"time"
	"mime"
)

type (
	// Mail parameters and data
	//
	// Iface: send email from selected interface
	//       "": default interface
	//       "12.34.56.78": ip address interface
	//       "socks://12.34.56.78:1234": send from remote socks server
	//       "1,4,6,8,9": id`s for send rotate by selected profile (set Host: "group")
	// Host: host name of selected interface ip
	MailData struct {
		Iface        string
		Host         string
		From_email   string
		From_name    string
		To_email     string
		To_name      string
		Extra_header string
		Subject      string
		Html         string
		Attachments  []Attachment
		conn         net.Conn
		socksConn    proxy.Dialer
		netConn      net.Dialer
	}

	// Attach file to mail
	//
	// Location: folder
	// Name: file
	Attachment struct {
		Location string
		Name     string
	}
)

// Send email
func (m *MailData) Send() error {

	// trim space
	m.To_email = strings.TrimSpace(m.To_email)
	// punycode convert
	splitEmail := strings.Split(m.To_email, "@")
	if len(splitEmail) != 2 {
		return errors.New(fmt.Sprintf("Bad email"))
	}
	domain, err := idna.ToASCII(splitEmail[1])
	if err != nil {
		return errors.New(fmt.Sprintf("Domain name failed: %v", err))
	}
	m.To_email = strings.Split(m.To_email, "@")[0] + "@" + domain

	if m.Iface == "" {
		// default interface
		m.netConn = net.Dialer{}
	} else {
		if m.Iface[0:8] == "socks://" {
			m.Iface = m.Iface[8:]
			var err error
			m.socksConn, err = proxy.SOCKS5("tcp", m.Iface, nil, proxy.FromEnvironment())
			if err != nil {
				return err
			}
		} else {
			connectAddr := net.ParseIP(m.Iface)
			tcpAddr := &net.TCPAddr{
				IP: connectAddr,
			}
			m.netConn = net.Dialer{LocalAddr: tcpAddr}
		}
	}
	start := time.Now()
	//record, err := net.LookupMX(c.domain)
	record, err := models.DomainGetMX(domain)
	if err != nil {
		return err
	}
	lookupTime := time.Since(start)

	start = time.Now()
	var serverMx string
	for i := range record {
		smx := net.JoinHostPort(record[i].Host, "25")
		if m.socksConn != nil {
			m.conn, err = m.socksConn.Dial("tcp", smx)
		} else {
			m.conn, err = m.netConn.Dial("tcp", smx)
		}
		if err == nil {
			serverMx = record[i].Host
			connTime := time.Since(start)
			fmt.Printf("Connect time to %s %s. Lookup time %s.\n\r", domain, connTime, lookupTime)
			break
		}
	}
	if err != nil {
		return err
	}
	defer m.conn.Close()

	// 5 minute by RFC
	m.conn.SetDeadline(time.Now().Add(5 * time.Minute))

	c, err := smtp.NewClient(m.conn, serverMx)
	if err != nil {
		return errors.New(fmt.Sprintf("%v (NewClient)", err))
	}

	if err := c.Hello(m.Host); err != nil {
		return errors.New(fmt.Sprintf("%v (Hello)", err))
	}

	// Set the sender and recipient first
	if err := c.Mail(m.From_email); err != nil {
		return errors.New(fmt.Sprintf("%v (Mail)", err))
	}

	if err := c.Rcpt(m.To_email); err != nil {
		return errors.New(fmt.Sprintf("%v (Rcpt)", err))
	}

	//dkim.New()

	w, err := c.Data()
	if err != nil {
		return errors.New(fmt.Sprintf("%v (Data)", err))
	}
	_, err = fmt.Fprint(w, m.makeMail())
	if err != nil {
		return errors.New(fmt.Sprintf("%v (SendData)", err))
	}

	err = w.Close()
	if err != nil {
		return errors.New(fmt.Sprintf("%v (Close)", err))
	}

	return c.Quit()
}

func (m *MailData) makeMail() string {

	var (
		multipart bool = false
		msg bytes.Buffer
		marker string
	)

	if len(m.Attachments) != 0 {
		multipart = true
		marker = makeMarker()
	}

	if m.From_name == "" {
		msg.WriteString(`From: ` + m.From_email + "\n")
	} else {
		msg.WriteString(`From: "` + encodeRFC2045(m.From_name) + `" <` + m.From_email + `>` + "\n")
	}

	if m.To_name == "" {
		msg.WriteString(`To: ` + m.To_email + "\n")
	} else {
		msg.WriteString(`To: "` + encodeRFC2045(m.To_name) + `" <` + m.To_email + `>` + "\n")
	}

	// -------------- head ----------------------------------------------------------

	msg.WriteString("Subject: " + encodeRFC2045(m.Subject) + "\n")
	msg.WriteString("MIME-Version: 1.0\n")
	msg.WriteString("X-Mailer: Gonder v" + models.Config.Version + "\n")
	msg.WriteString("Date: " + time.Now().Format(time.RFC1123Z) + "\n")
	if multipart {
		msg.WriteString("Content-Type: multipart/mixed;\n	boundary=\"" + marker + "\"\n")
	} else {
		msg.WriteString("Content-Transfer-Encoding: base64\nContent-Type: text/html; charset=\"utf-8\"\n")
	}
	msg.WriteString(m.Extra_header + "\n")
	// ------------- /head ---------------------------------------------------------

	// ------------- body ----------------------------------------------------------
	if multipart {
		msg.WriteString("--" + marker + "\n")
		msg.WriteString("Content-Transfer-Encoding: base64\nContent-Type: text/html; charset=\"utf-8\"\n\n")
	}
	msg.WriteString(base64code([]byte(m.Html)))
	msg.WriteString("\n")
	// ------------ /body ---------------------------------------------------------

	// ----------- attachments ----------------------------------------------------
	for _, file := range m.Attachments {

		msg.WriteString("\n--" + marker)
		content, err := ioutil.ReadFile(file.Location + file.Name)
		if err != nil {
			fmt.Println(err)
		}
		msg.WriteString(fmt.Sprintf("\nContent-Type: %s;\n	name=\"%s\"\nContent-Transfer-Encoding: base64\nContent-Disposition: attachment;\n	filename=\"%s\"\n\n", http.DetectContentType(content), file.Name, file.Name))
		msg.WriteString(base64code(content))
		msg.WriteString("\n")
	}
	// ----------- /attachments ---------------------------------------------------

	return msg.String()
}

func base64code(data []byte) string {
	var (
		result bytes.Buffer
	)
	encoded := base64.StdEncoding.EncodeToString(data)
	lineMaxLength := 76
	nbrLines := len(encoded) / lineMaxLength
	for i := 0; i < nbrLines; i++ {
		result.WriteString(encoded[i*lineMaxLength:(i+1)*lineMaxLength])
		result.WriteString("\n")
	}
	result.WriteString(encoded[nbrLines*lineMaxLength:])
	result.WriteString("\n")
	return result.String()
}

func makeMarker() string {
	b := make([]byte, 30)
	rand.Read(b)
	en := base64.StdEncoding // or URLEncoding
	d := make([]byte, en.EncodedLen(len(b)))
	en.Encode(d, b)
	return string(d)
}

func encodeRFC2045(s string) string {
	return mime.BEncoding.Encode("utf-8", s)
}

func encodeRFC2047(s string) string {
	return mime.QEncoding.Encode("utf-8", s)
}
<|endoftext|>


<commit_before>package classificationbox_test

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/machinebox/sdk-go/classificationbox"
	"github.com/matryer/is"
)

func TestPredict(t *testing.T) {
	is := is.New(t)
	var apiCalls int
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		apiCalls++
		is.Equal(r.Method, http.MethodPost)
		is.Equal(r.URL.Path, "/classificationbox/models/model1/predict")
		is.Equal(r.Header.Get("Accept"), "application/json; charset=utf-8")
		is.Equal(r.Header.Get("Content-Type"), "application/json; charset=utf-8")
		var req classificationbox.PredictRequest
		is.NoErr(json.NewDecoder(r.Body).Decode(&req))
		is.Equal(req.Inputs[0].Key, "title")
		resp := classificationbox.PredictResponse{
			Classes: []classificationbox.PredictedClass{
				{
					ID:    "choice1",
					Score: 0.7,
				},
				{
					ID:    "choice2",
					Score: 0.2,
				},
				{
					ID:    "choice3",
					Score: 0.1,
				},
			},
		}
		is.NoErr(json.NewEncoder(w).Encode(struct {
			classificationbox.PredictResponse
			Success bool `json:"success"`
		}{
			Success:         true,
			PredictResponse: resp,
		}))
	}))
	defer srv.Close()
	cb := classificationbox.New(srv.URL)
	predictReq := classificationbox.PredictRequest{
		Inputs: []classificationbox.Feature{
			{
				Key:   "title",
				Type:  "text",
				Value: "Machine Box releases new product",
			},
		},
	}
	predictResp, err := cb.Predict(context.Background(), "model1", predictReq)
	is.NoErr(err)
	is.Equal(apiCalls, 1)                 // apiCalls
	is.Equal(len(predictResp.Classes), 3) // len(predictResp.Choices)
}
<commit_msg>fixed broken test after type rename<commit_after>package classificationbox_test

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/machinebox/sdk-go/classificationbox"
	"github.com/matryer/is"
)

func TestPredict(t *testing.T) {
	is := is.New(t)
	var apiCalls int
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		apiCalls++
		is.Equal(r.Method, http.MethodPost)
		is.Equal(r.URL.Path, "/classificationbox/models/model1/predict")
		is.Equal(r.Header.Get("Accept"), "application/json; charset=utf-8")
		is.Equal(r.Header.Get("Content-Type"), "application/json; charset=utf-8")
		var req classificationbox.PredictRequest
		is.NoErr(json.NewDecoder(r.Body).Decode(&req))
		is.Equal(req.Inputs[0].Key, "title")
		resp := classificationbox.PredictResponse{
			Classes: []classificationbox.Class{
				{
					ID:    "choice1",
					Score: 0.7,
				},
				{
					ID:    "choice2",
					Score: 0.2,
				},
				{
					ID:    "choice3",
					Score: 0.1,
				},
			},
		}
		is.NoErr(json.NewEncoder(w).Encode(struct {
			classificationbox.PredictResponse
			Success bool `json:"success"`
		}{
			Success:         true,
			PredictResponse: resp,
		}))
	}))
	defer srv.Close()
	cb := classificationbox.New(srv.URL)
	predictReq := classificationbox.PredictRequest{
		Inputs: []classificationbox.Feature{
			{
				Key:   "title",
				Type:  "text",
				Value: "Machine Box releases new product",
			},
		},
	}
	predictResp, err := cb.Predict(context.Background(), "model1", predictReq)
	is.NoErr(err)
	is.Equal(apiCalls, 1)                 // apiCalls
	is.Equal(len(predictResp.Classes), 3) // len(predictResp.Choices)
}
<|endoftext|>


<commit_before>// investigate.go contains functions to "investigate" individual files for type, size, binary/text
// directories for character of their content and so on
package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	c "github.com/mitchellh/colorstring"
)

const (
	numberOfReadTestBytes = 1024
)

// InvestigateFile prepares detailed file/directory summary
func (fi FileInfo) InvestigateFile(i int, updated chan FileListUpdate) {
	m := fi.f.Mode()
	done := true
	switch {
	case m&os.ModeSymlink != 0:
		fi.special = "symlink"
		link, err := filepath.EvalSymlinks(mode.targetPath + "/" + fi.f.Name())
		if err == nil {
			fi.description = "link: [green]" + presentPath(link) // will eventually use strings.TrimPrefix to shorten for things like homepath
		} else {
			fi.description = "got error trying to resolve symlink"
		}
	case m&os.ModeDevice != 0:
		fi.special = "device"
	case m&os.ModeNamedPipe != 0:
		fi.special = "unix named pipe"
	case m&os.ModeSocket != 0:
		fi.special = "unix domain socket"
	case m&os.ModeAppend != 0:
		fi.special = "append-only file"
	case m&os.ModeExclusive != 0:
		fi.special = "exclusive-use file"
	case m&os.ModeDir != 0:
		fi.special = "dir"
	default:
		fi.special = "regular"
		go fi.investigateRegFile(i, updated)
		done = false
	}

	updated <- FileListUpdate{i, &fi, done}
}

func (fi FileInfo) investigateRegFile(i int, updated chan FileListUpdate) {
	if fi.f.Size() == 0 {
		fi.description = "Empty File"
		fi.special = "Empty File"
		updated <- FileListUpdate{i, &fi, true}
		return
	}
	isTxt, err := CheckIfTextFile(fi)
	if err != nil {
		updated <- FileListUpdate{i, nil, true}
		return
	}
	if isTxt {
		fi.special = "Text File"
	} else {
		fi.special = "Binary File"
	}
	updated <- FileListUpdate{i, &fi, true}
}

// CheckIfTextFile tests if the file is text or binary
// using the bash's diff tool method:
// by reading the first numberOfReadTestBytes bytes
// and looking for NULL byte. If there is one encountered,
// it is probably a binary.
func CheckIfTextFile(file FileInfo) (bool, error) {
	var bytesToRead int64 = numberOfReadTestBytes
	if file.f.Size() < numberOfReadTestBytes {
		bytesToRead = file.f.Size()
	}

	fi, err := os.Open(mode.targetPath + "/" + file.f.Name())
	if err != nil {
		return false, err
	}
	defer fi.Close()

	buf := make([]byte, bytesToRead)
	_, err = fi.Read(buf)
	if err != nil {
		return false, err
	}
	for _, b := range buf {
		if b == byte(0) {
			return false, nil
		}
	}
	return true, nil
}
<commit_msg>Read /git/config and show remote repo details for dirs<commit_after>// investigate.go contains functions to "investigate" individual files for type, size, binary/text
// directories for character of their content and so on
package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	c "github.com/mitchellh/colorstring"
)

const (
	numberOfReadTestBytes = 1024
)

// InvestigateFile prepares detailed file/directory summary
func (fi FileInfo) InvestigateFile(i int, updated chan FileListUpdate) {
	m := fi.f.Mode()
	done := true
	switch {
	case m&os.ModeSymlink != 0:
		fi.special = "symlink"
		link, err := filepath.EvalSymlinks(mode.targetPath + "/" + fi.f.Name())
		if err == nil {
			fi.description = "link: [green]" + presentPath(link) // will eventually use strings.TrimPrefix to shorten for things like homepath
		} else {
			fi.description = "got error trying to resolve symlink"
		}
	case m&os.ModeDevice != 0:
		fi.special = "device"
	case m&os.ModeNamedPipe != 0:
		fi.special = "unix named pipe"
	case m&os.ModeSocket != 0:
		fi.special = "unix domain socket"
	case m&os.ModeAppend != 0:
		fi.special = "append-only file"
	case m&os.ModeExclusive != 0:
		fi.special = "exclusive-use file"
	case fi.f.Name() == ".git":
		fi.hidden = true
		remote := investigateGit(mode.targetPath)
		if remote != "" {
			mode.comments = append(mode.comments, "git repo (remote at "+remote+")")
		} else {
			mode.comments = append(mode.comments, "git repo")
		}
	case m&os.ModeDir != 0:
		fi.special = "dir"
		go fi.investigateDir(i, updated)
		done = false

	default:
		fi.special = "regular"
		go fi.investigateRegFile(i, updated)
		done = false
	}

	updated <- FileListUpdate{i, &fi, done}
}

func (fi FileInfo) investigateRegFile(i int, updated chan FileListUpdate) {
	if fi.f.Size() == 0 {
		fi.description = "Empty File"
		fi.special = "Empty File"
		updated <- FileListUpdate{i, &fi, true}
		return
	}
	isTxt, err := CheckIfTextFile(fi)
	if err != nil {
		updated <- FileListUpdate{i, nil, true}
		return
	}
	if isTxt {
		fi.special = "Text File"
	} else {
		fi.special = "Binary File"
	}
	updated <- FileListUpdate{i, &fi, true}
}

func (fi FileInfo) investigateDir(i int, updated chan FileListUpdate) {
	files, err := ioutil.ReadDir(mode.targetPath + "/" + fi.f.Name())
	if err != nil {
		updated <- FileListUpdate{i, &fi, true}
		return
	}
	fi.description = fmt.Sprintf(c.Color("[red]%v[white] files inside"), len(files))
	isgit := investigateGit(mode.targetPath + "/" + fi.f.Name())
	if isgit != "" {
		fi.description = isgit
	}
	updated <- FileListUpdate{i, &fi, true}
}

// CheckIfTextFile tests if the file is text or binary
// using the bash's diff tool method:
// by reading the first numberOfReadTestBytes bytes
// and looking for NULL byte. If there is one encountered,
// it is probably a binary.
func CheckIfTextFile(file FileInfo) (bool, error) {
	var bytesToRead int64 = numberOfReadTestBytes
	if file.f.Size() < numberOfReadTestBytes {
		bytesToRead = file.f.Size()
	}

	fi, err := os.Open(mode.targetPath + "/" + file.f.Name())
	if err != nil {
		return false, err
	}
	defer fi.Close()

	buf := make([]byte, bytesToRead)
	_, err = fi.Read(buf)
	if err != nil {
		return false, err
	}
	for _, b := range buf {
		if b == byte(0) {
			return false, nil
		}
	}
	return true, nil
}

func investigateGit(path string) string {
	const UrlLine = "url = "
	buf, err := ioutil.ReadFile(path + "/.git/config")
	if err != nil {
		return ""
	}
	cfg := string(buf)
	i := strings.Index(cfg, UrlLine)
	if i == -1 {
		return ""
	}
	j := strings.Index(cfg[i:], "\n")
	if j == -1 {
		return ""
	}
	return "[green]" + cfg[i+len(UrlLine):i+j] + "[yellow]"
}
<|endoftext|>


<commit_before>// Copyright 2014 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tcp

import (
	"os"
	"strings"
	"time"
	"unsafe"
)

// A CAState represents a state of congestion avoidance.
type CAState int

var caStates = map[CAState]string{
	CAOpen:     "open",
	CADisorder: "disorder",
	CACWR:      "congestion window reduced",
	CARecovery: "recovery",
	CALoss:     "loss",
}

func (st CAState) String() string {
	s, ok := caStates[st]
	if !ok {
		return "<nil>"
	}
	return s
}

// A SysInfo represents platform-specific information.
type SysInfo struct {
	PathMTU         uint       `json:"path mtu"`     // path maximum transmission unit
	AdvertisedMSS   MaxSegSize `json:"adv mss"`      // advertised maximum segment size
	CAState         CAState    `json:"ca state"`     // state of congestion avoidance
	KeepAliveProbes uint       `json:"ka probes"`    // # of keep alive probes sent
	UnackSegs       uint       `json:"unack segs"`   // # of unack'd segments in transmission queue
	SackSegs        uint       `json:"sack segs"`    // # of sack'd segments in tranmission queue
	LostSegs        uint       `json:"lost segs"`    // # of lost segments in transmission queue
	RetransSegs     uint       `json:"retrans segs"` // # of retransmitting segments in transmission queue
	ForwardAckSegs  uint       `json:"fack segs"`    // # of forward ack'd segments in transmission queue
}

func info(s uintptr) (*Info, error) {
	b := make([]byte, sizeofTCPInfo)
	if err := getsockopt(s, ianaProtocolTCP, sysTCP_INFO, b); err != nil {
		return nil, os.NewSyscallError("getsockopt", err)
	}
	cb := make([]byte, sizeofTCPCCInfo)
	if err := getsockopt(s, ianaProtocolTCP, sysTCP_CC_INFO, cb); err != nil {
		return parseInfo(b), nil
	}
	nb := make([]byte, 16) // see TCP_CA_NAME_MAX
	if err := getsockopt(s, ianaProtocolTCP, sysTCP_CONGESTION, nb); err != nil {
		return parseInfo(b), nil
	}
	ti := parseInfo(b)
	i := 0
	for i = 0; i < 16; i++ {
		if nb[i] == 0 {
			break
		}
	}
	scc := parseSysCC(string(nb[:i]), cb)
	if ti != nil && ti.CongestionControl != nil && scc != nil {
		ti.CongestionControl.Sys = scc
	}
	return ti, nil
}

var sysStates = [12]State{Unknown, Established, SynSent, SynReceived, FinWait1, FinWait2, TimeWait, Closed, CloseWait, LastAck, Listen, Closing}

func parseInfo(b []byte) *Info {
	sti := (*sysTCPInfo)(unsafe.Pointer(&b[0]))
	ti := &Info{State: sysStates[sti.State]}
	if sti.Options&sysTCPI_OPT_WSCALE != 0 {
		ti.Options = append(ti.Options, WindowScale(sti.Pad_cgo_0[0]>>4))
		ti.PeerOptions = append(ti.PeerOptions, WindowScale(sti.Pad_cgo_0[0]&0x0f))
	}
	if sti.Options&sysTCPI_OPT_TIMESTAMPS != 0 {
		ti.Options = append(ti.Options, Timestamps(true))
		ti.PeerOptions = append(ti.PeerOptions, Timestamps(true))
	}
	ti.SenderMSS = MaxSegSize(sti.Snd_mss)
	ti.ReceiverMSS = MaxSegSize(sti.Rcv_mss)
	ti.RTT = time.Duration(sti.Rtt) * time.Microsecond
	ti.RTTVar = time.Duration(sti.Rttvar) * time.Microsecond
	ti.RTO = time.Duration(sti.Rto) * time.Microsecond
	ti.ATO = time.Duration(sti.Ato) * time.Microsecond
	ti.LastDataSent = time.Duration(sti.Last_data_sent) * time.Millisecond
	ti.LastDataReceived = time.Duration(sti.Last_data_recv) * time.Millisecond
	ti.LastAckReceived = time.Duration(sti.Last_ack_recv) * time.Millisecond
	ti.FlowControl = &FlowControl{
		ReceiverWindow: uint(sti.Rcv_space),
	}
	ti.CongestionControl = &CongestionControl{
		SenderSSThreshold:   uint(sti.Snd_ssthresh),
		ReceiverSSThreshold: uint(sti.Rcv_ssthresh),
		SenderWindow:        uint(sti.Snd_cwnd),
	}
	ti.Sys = &SysInfo{
		PathMTU:         uint(sti.Pmtu),
		AdvertisedMSS:   MaxSegSize(sti.Advmss),
		CAState:         CAState(sti.Ca_state),
		KeepAliveProbes: uint(sti.Probes),
		UnackSegs:       uint(sti.Unacked),
		SackSegs:        uint(sti.Sacked),
		LostSegs:        uint(sti.Lost),
		RetransSegs:     uint(sti.Retrans),
		ForwardAckSegs:  uint(sti.Fackets),
	}
	return ti
}

// A SysCongestionControl represents platform-specific congestion
// control information.
type SysCongestionControl struct {
	Algo string                   `json:"algo"` // congestion control algorithm name
	Info SysCongestionControlInfo `json:"info,omitempty"`
}

// A SysCongestionControlInfo represents congestion control
// algorithm-specific information.
type SysCongestionControlInfo interface {
	String() string
}

// A VegasInfo represents TCP Vegas congestion control information.
type VegasInfo struct {
	Enabled    bool          `json:"enabled"`
	RoundTrips uint          `json:"rnd trips"` // # of round-trips
	RTT        time.Duration `json:"rtt"`       // round-trip time
	MinRTT     time.Duration `json:"min rtt"`   // minimum round-trip time
}

// String implements the String method of SysCongestionControlInfo
// interface.
func (vi *VegasInfo) String() string { return "vegas" }

// A CEState represents a state of ECN congestion encountered (CE)
// codepoint.
type CEState int

// A DCTCPInfo represents Datacenter TCP congestion control
// information.
type DCTCPInfo struct {
	Enabled         bool    `json:"enabled"`
	CEState         CEState `json:"ce state"`    // state of ECN CE codepoint
	Alpha           uint    `json:"alpha"`       // fraction of bytes sent
	ECNAckedBytes   uint    `json:"ecn acked"`   // # of acked bytes with ECN
	TotalAckedBytes uint    `json:"total acked"` // total # of acked bytes
}

// String implements the String method of SysCongestionControlInfo
// interface.
func (di *DCTCPInfo) String() string { return "dctcp" }

func parseSysCC(name string, b []byte) *SysCongestionControl {
	stcci := (*sysTCPCCInfo)(unsafe.Pointer(&b[0]))
	scc := SysCongestionControl{Algo: name}
	if strings.HasPrefix(name, "dctcp") {
		stdi := (*sysTCPDCTCPInfo)(unsafe.Pointer(stcci))
		if stdi.Enabled == 0 {
			return &scc
		}
		scc.Info = &DCTCPInfo{
			Enabled: true,
			Alpha:   uint(stdi.Alpha),
		}
		return &scc
	}
	stvi := (*sysTCPVegasInfo)(unsafe.Pointer(stcci))
	if stvi.Enabled == 0 {
		return &scc
	}
	scc.Info = &VegasInfo{
		Enabled:    true,
		RoundTrips: uint(stvi.Rttcnt),
		RTT:        time.Duration(stvi.Rtt) * time.Microsecond,
		MinRTT:     time.Duration(stvi.Minrtt) * time.Microsecond,
	}
	return &scc
}
<commit_msg>tcp: rename String method of the SysCongestionControlInfo interface to Algo<commit_after>// Copyright 2014 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tcp

import (
	"os"
	"strings"
	"time"
	"unsafe"
)

// A CAState represents a state of congestion avoidance.
type CAState int

var caStates = map[CAState]string{
	CAOpen:     "open",
	CADisorder: "disorder",
	CACWR:      "congestion window reduced",
	CARecovery: "recovery",
	CALoss:     "loss",
}

func (st CAState) String() string {
	s, ok := caStates[st]
	if !ok {
		return "<nil>"
	}
	return s
}

// A SysInfo represents platform-specific information.
type SysInfo struct {
	PathMTU         uint       `json:"path mtu"`     // path maximum transmission unit
	AdvertisedMSS   MaxSegSize `json:"adv mss"`      // advertised maximum segment size
	CAState         CAState    `json:"ca state"`     // state of congestion avoidance
	KeepAliveProbes uint       `json:"ka probes"`    // # of keep alive probes sent
	UnackSegs       uint       `json:"unack segs"`   // # of unack'd segments in transmission queue
	SackSegs        uint       `json:"sack segs"`    // # of sack'd segments in tranmission queue
	LostSegs        uint       `json:"lost segs"`    // # of lost segments in transmission queue
	RetransSegs     uint       `json:"retrans segs"` // # of retransmitting segments in transmission queue
	ForwardAckSegs  uint       `json:"fack segs"`    // # of forward ack'd segments in transmission queue
}

func info(s uintptr) (*Info, error) {
	b := make([]byte, sizeofTCPInfo)
	if err := getsockopt(s, ianaProtocolTCP, sysTCP_INFO, b); err != nil {
		return nil, os.NewSyscallError("getsockopt", err)
	}
	cb := make([]byte, sizeofTCPCCInfo)
	if err := getsockopt(s, ianaProtocolTCP, sysTCP_CC_INFO, cb); err != nil {
		return parseInfo(b), nil
	}
	nb := make([]byte, 16) // see TCP_CA_NAME_MAX
	if err := getsockopt(s, ianaProtocolTCP, sysTCP_CONGESTION, nb); err != nil {
		return parseInfo(b), nil
	}
	ti := parseInfo(b)
	i := 0
	for i = 0; i < 16; i++ {
		if nb[i] == 0 {
			break
		}
	}
	scc := parseSysCC(string(nb[:i]), cb)
	if ti != nil && ti.CongestionControl != nil && scc != nil {
		ti.CongestionControl.Sys = scc
	}
	return ti, nil
}

var sysStates = [12]State{Unknown, Established, SynSent, SynReceived, FinWait1, FinWait2, TimeWait, Closed, CloseWait, LastAck, Listen, Closing}

func parseInfo(b []byte) *Info {
	sti := (*sysTCPInfo)(unsafe.Pointer(&b[0]))
	ti := &Info{State: sysStates[sti.State]}
	if sti.Options&sysTCPI_OPT_WSCALE != 0 {
		ti.Options = append(ti.Options, WindowScale(sti.Pad_cgo_0[0]>>4))
		ti.PeerOptions = append(ti.PeerOptions, WindowScale(sti.Pad_cgo_0[0]&0x0f))
	}
	if sti.Options&sysTCPI_OPT_TIMESTAMPS != 0 {
		ti.Options = append(ti.Options, Timestamps(true))
		ti.PeerOptions = append(ti.PeerOptions, Timestamps(true))
	}
	ti.SenderMSS = MaxSegSize(sti.Snd_mss)
	ti.ReceiverMSS = MaxSegSize(sti.Rcv_mss)
	ti.RTT = time.Duration(sti.Rtt) * time.Microsecond
	ti.RTTVar = time.Duration(sti.Rttvar) * time.Microsecond
	ti.RTO = time.Duration(sti.Rto) * time.Microsecond
	ti.ATO = time.Duration(sti.Ato) * time.Microsecond
	ti.LastDataSent = time.Duration(sti.Last_data_sent) * time.Millisecond
	ti.LastDataReceived = time.Duration(sti.Last_data_recv) * time.Millisecond
	ti.LastAckReceived = time.Duration(sti.Last_ack_recv) * time.Millisecond
	ti.FlowControl = &FlowControl{
		ReceiverWindow: uint(sti.Rcv_space),
	}
	ti.CongestionControl = &CongestionControl{
		SenderSSThreshold:   uint(sti.Snd_ssthresh),
		ReceiverSSThreshold: uint(sti.Rcv_ssthresh),
		SenderWindow:        uint(sti.Snd_cwnd),
	}
	ti.Sys = &SysInfo{
		PathMTU:         uint(sti.Pmtu),
		AdvertisedMSS:   MaxSegSize(sti.Advmss),
		CAState:         CAState(sti.Ca_state),
		KeepAliveProbes: uint(sti.Probes),
		UnackSegs:       uint(sti.Unacked),
		SackSegs:        uint(sti.Sacked),
		LostSegs:        uint(sti.Lost),
		RetransSegs:     uint(sti.Retrans),
		ForwardAckSegs:  uint(sti.Fackets),
	}
	return ti
}

// A SysCongestionControl represents platform-specific congestion
// control information.
type SysCongestionControl struct {
	Algo string                   `json:"algo"` // congestion control algorithm name
	Info SysCongestionControlInfo `json:"info,omitempty"`
}

// A SysCongestionControlInfo represents congestion control
// algorithm-specific information.
type SysCongestionControlInfo interface {
	// Algo returns the congestion control algorithm name.
	Algo() string
}

// A VegasInfo represents TCP Vegas congestion control information.
type VegasInfo struct {
	Enabled    bool          `json:"enabled"`
	RoundTrips uint          `json:"rnd trips"` // # of round-trips
	RTT        time.Duration `json:"rtt"`       // round-trip time
	MinRTT     time.Duration `json:"min rtt"`   // minimum round-trip time
}

// Algo implements the Algo method of SysCongestionControlInfo
// interface.
func (vi *VegasInfo) Algo() string { return "vegas" }

// A CEState represents a state of ECN congestion encountered (CE)
// codepoint.
type CEState int

// A DCTCPInfo represents Datacenter TCP congestion control
// information.
type DCTCPInfo struct {
	Enabled         bool    `json:"enabled"`
	CEState         CEState `json:"ce state"`    // state of ECN CE codepoint
	Alpha           uint    `json:"alpha"`       // fraction of bytes sent
	ECNAckedBytes   uint    `json:"ecn acked"`   // # of acked bytes with ECN
	TotalAckedBytes uint    `json:"total acked"` // total # of acked bytes
}

// Algo implements the Algo method of SysCongestionControlInfo
// interface.
func (di *DCTCPInfo) Algo() string { return "dctcp" }

func parseSysCC(name string, b []byte) *SysCongestionControl {
	stcci := (*sysTCPCCInfo)(unsafe.Pointer(&b[0]))
	scc := SysCongestionControl{Algo: name}
	if strings.HasPrefix(name, "dctcp") {
		stdi := (*sysTCPDCTCPInfo)(unsafe.Pointer(stcci))
		if stdi.Enabled == 0 {
			return &scc
		}
		scc.Info = &DCTCPInfo{
			Enabled: true,
			Alpha:   uint(stdi.Alpha),
		}
		return &scc
	}
	stvi := (*sysTCPVegasInfo)(unsafe.Pointer(stcci))
	if stvi.Enabled == 0 {
		return &scc
	}
	scc.Info = &VegasInfo{
		Enabled:    true,
		RoundTrips: uint(stvi.Rttcnt),
		RTT:        time.Duration(stvi.Rtt) * time.Microsecond,
		MinRTT:     time.Duration(stvi.Minrtt) * time.Microsecond,
	}
	return &scc
}
<|endoftext|>