<commit_before>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package images

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"path"

	flag "github.com/spf13/pflag"

	"github.com/jetstack/cert-manager/hack/release/pkg/bazel"
	"github.com/jetstack/cert-manager/hack/release/pkg/flags"
	logf "github.com/jetstack/cert-manager/hack/release/pkg/log"
	"github.com/jetstack/cert-manager/hack/release/pkg/util"
)

var (
	Default = &Plugin{}

	supportedGoArch     = []string{"amd64", "arm64", "arm"}
	supportedComponents = []string{"acmesolver", "controller", "webhook", "cainjector"}
	log                 = logf.Log.WithName("images")
)

type Plugin struct {
	// The list of images to build (e.g. acmesolver, controller, webhook)
	Components []string

	// If true, the built images will be exported to the configured docker
	// daemon when the Build() method is called.
	ExportToDocker bool

	// List of architectures to build images for
	GoArch []string

	// DockerConfig is a path to a directory containing a config.json file that
	// is used for Docker authentication
	DockerConfig string

	// TODO: add GOOS support once the build system supports more than linux

	// built is set to true if Build() has completed successfully
	built bool
	// configFileName is computed based on the DockerConfig field
	configFileName string
}

func (g *Plugin) AddFlags(fs *flag.FlagSet) {
	fs.BoolVar(&g.ExportToDocker, "images.export", false, "if true, images will be exported to the currently configured docker daemon")
	fs.StringSliceVar(&g.Components, "images.components", []string{"acmesolver", "controller", "webhook"}, "the list of components to build images for")
	fs.StringSliceVar(&g.GoArch, "images.goarch", []string{"amd64", "arm64", "arm"}, "list of architectures to build images for")
	fs.StringVar(&g.DockerConfig, "images.docker-config", "", "path to a directory containing a docker config.json file used when pushing images")
}

func (g *Plugin) Validate() []error {
	var errs []error

	// validate goarch flag
	for _, a := range g.GoArch {
		valid := false
		for _, sa := range supportedGoArch {
			if a == sa {
				valid = true
				break
			}
		}
		if !valid {
			errs = append(errs, fmt.Errorf("invalid goarch value %q", a))
		}
	}

	// validate components flag
	for _, a := range g.Components {
		valid := false
		for _, sa := range supportedComponents {
			if a == sa {
				valid = true
				break
			}
		}
		if !valid {
			errs = append(errs, fmt.Errorf("invalid component name %q", a))
		}
	}

	return errs
}

func (g *Plugin) InitPublish() []error {
	var errs []error

	if g.DockerConfig != "" {
		// Hardcode the filename to be config.json for backwards compatibility
		// with the old release.sh script.
		configFileName := path.Join(g.DockerConfig, "config.json")
		f, err := os.Stat(configFileName)
		if err != nil {
			return []error{fmt.Errorf("error checking config file: %v", err)}
		}
		if f.IsDir() {
			return []error{fmt.Errorf("docker config.json is not a file")}
		}
		g.configFileName = configFileName
	}

	return errs
}

func (g *Plugin) Build(ctx context.Context) error {
	_, err := g.build(ctx)
	if err != nil {
		return err
	}

	if g.ExportToDocker {
		log.Info("Exporting docker images to local docker daemon")
		if err := g.exportToDocker(ctx); err != nil {
			return err
		}
	} else {
		log.Info("skipping exporting docker images to docker daemon")
	}

	return nil
}

func (g *Plugin) Publish(ctx context.Context) error {
	log.Info("running publish for image plugin")
	// this case should never be reached, but we check it to be safe
	if !g.built {
		if _, err := g.build(ctx); err != nil {
			return err
		}
	}

	log.Info("pushing images")
	targets := g.generateTargets()
	err := g.pushImages(ctx, targets)
	if err != nil {
		return err
	}

	log.Info("published all docker images")

	return nil
}

func (g *Plugin) Complete() error {
	log = log.WithName("default-flags")

	if g.DockerConfig == "" {
		g.DockerConfig = os.Getenv("DOCKER_CONFIG")
		if g.DockerConfig != "" {
			log.Info("set default value", "value", g.DockerConfig)
		}
	}
	return nil
}

func (g *Plugin) build(ctx context.Context) (imageTargets, error) {
	targets := g.generateTargets()

	bazelTargets := targets.bazelTargets()
	log := log.WithValues("images", bazelTargets)
	log.Info("building bazel image targets")

	// set the os and arch to linux/amd64
	// whilst we might be building cross-arch binaries, cgo only depends on
	// particular OS settings and not arch, so just by setting this to 'linux'
	// we can fix cross builds on platforms other than linux
	// if we support alternate OS values in future, this will need updating
	// with a call to BuildPlatformE per *OS*.
	err := bazel.Default.BuildPlatformE(ctx, log, "linux", "amd64", bazelTargets...)
	if err != nil {
		return nil, fmt.Errorf("error building docker images (%v): %v", targets, err)
	}

	g.built = true
	return targets, nil
}

func (g *Plugin) exportToDocker(ctx context.Context) error {
	targets := g.generateTargets()
	log.WithValues("images", targets.bazelExportTargets()).Info("exporting images to docker daemon")
	for _, target := range targets.bazelExportTargets() {
		log := log.WithValues("target", target)
		log.Info("exporting image to docker daemon")
		// set the os and arch to linux/amd64
		// whilst we might be building cross-arch binaries, cgo only depends on
		// particular OS settings and not arch, so just by setting this to 'linux'
		// we can fix cross builds on platforms other than linux
		// if we support alternate OS values in future, this will need updating
		// with a call to BuildPlatformE per *OS*.
		err := bazel.Default.RunPlatformE(ctx, log, "linux", "amd64", target)
		if err != nil {
			return fmt.Errorf("error exporting image %q to docker daemon: %v", target, err)
		}
	}

	log.WithValues("images", targets.exportedImageNames()).Info("exported all docker images")

	return nil
}

// generateTargets generates a list of Bazel target names that must be
// built for this invocation of the image builder
func (g *Plugin) generateTargets() imageTargets {
	var targets []imageTarget
	for _, c := range g.Components {
		for _, a := range g.GoArch {
			targets = append(targets, imageTarget{c, "linux", a})
		}
	}
	return targets
}

// pushImages will push the images built for this release to the registry
// TODO: add support for calling container_push targets instead of just 'docker push'
func (p *Plugin) pushImages(ctx context.Context, targets imageTargets) error {
	err := p.exportToDocker(ctx)
	if err != nil {
		return err
	}

	var images []string
	for _, t := range targets {
		images = append(images, t.exportedImageNames()...)
	}

	log.WithValues("images", images).Info("pushing docker images")
	for _, img := range images {
		log := log.WithValues("image", img)
		log.Info("pushing docker image")
		args := []string{"push", img}
		if p.configFileName != "" {
			args = append(args, "--config", p.configFileName)
		}
		cmd := exec.CommandContext(ctx, "docker", args...)
		err := util.RunE(log, cmd)
		if err != nil {
			return err
		}
	}

	return nil
}

type imageTargets []imageTarget

func (i imageTargets) bazelTargets() []string {
	out := make([]string, len(i))
	for idx, target := range i {
		out[idx] = target.bazelTarget()
	}
	return out
}

func (i imageTargets) bazelExportTargets() []string {
	out := make([]string, len(i))
	for idx, target := range i {
		out[idx] = target.bazelExportTarget()
	}
	return out
}

func (i imageTargets) exportedImageNames() []string {
	out := make([]string, 0)
	for _, target := range i {
		out = append(out, target.exportedImageNames()...)
	}
	return out
}

type imageTarget struct {
	name, os, arch string
}

func (i imageTarget) bazelTarget() string {
	return fmt.Sprintf("//cmd/%s:image.%s-%s", i.name, i.os, i.arch)
}

func (i imageTarget) bazelExportTarget() string {
	return fmt.Sprintf("//cmd/%s:image.%s-%s.export", i.name, i.os, i.arch)
}

func (i imageTarget) exportedImageNames() []string {
	if i.arch == "amd64" {
		return []string{
			fmt.Sprintf("%s/cert-manager-%s:%s", flags.Default.DockerRepo, i.name, flags.Default.AppVersion),
			fmt.Sprintf("%s/cert-manager-%s:%s", flags.Default.DockerRepo, i.name, flags.Default.GitCommitRef),
		}
	}

	return []string{
		fmt.Sprintf("%s/cert-manager-%s-%s:%s", flags.Default.DockerRepo, i.name, i.arch, flags.Default.AppVersion),
		fmt.Sprintf("%s/cert-manager-%s-%s:%s", flags.Default.DockerRepo, i.name, i.arch, flags.Default.GitCommitRef),
	}
}
<commit_msg>Set flag variable on log<commit_after>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package images

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"path"

	flag "github.com/spf13/pflag"

	"github.com/jetstack/cert-manager/hack/release/pkg/bazel"
	"github.com/jetstack/cert-manager/hack/release/pkg/flags"
	logf "github.com/jetstack/cert-manager/hack/release/pkg/log"
	"github.com/jetstack/cert-manager/hack/release/pkg/util"
)

var (
	Default = &Plugin{}

	supportedGoArch     = []string{"amd64", "arm64", "arm"}
	supportedComponents = []string{"acmesolver", "controller", "webhook", "cainjector"}
	log                 = logf.Log.WithName("images")
)

type Plugin struct {
	// The list of images to build (e.g. acmesolver, controller, webhook)
	Components []string

	// If true, the built images will be exported to the configured docker
	// daemon when the Build() method is called.
	ExportToDocker bool

	// List of architectures to build images for
	GoArch []string

	// DockerConfig is a path to a directory containing a config.json file that
	// is used for Docker authentication
	DockerConfig string

	// TODO: add GOOS support once the build system supports more than linux

	// built is set to true if Build() has completed successfully
	built bool
	// configFileName is computed based on the DockerConfig field
	configFileName string
}

func (g *Plugin) AddFlags(fs *flag.FlagSet) {
	fs.BoolVar(&g.ExportToDocker, "images.export", false, "if true, images will be exported to the currently configured docker daemon")
	fs.StringSliceVar(&g.Components, "images.components", []string{"acmesolver", "controller", "webhook"}, "the list of components to build images for")
	fs.StringSliceVar(&g.GoArch, "images.goarch", []string{"amd64", "arm64", "arm"}, "list of architectures to build images for")
	fs.StringVar(&g.DockerConfig, "images.docker-config", "", "path to a directory containing a docker config.json file used when pushing images")
}

func (g *Plugin) Validate() []error {
	var errs []error

	// validate goarch flag
	for _, a := range g.GoArch {
		valid := false
		for _, sa := range supportedGoArch {
			if a == sa {
				valid = true
				break
			}
		}
		if !valid {
			errs = append(errs, fmt.Errorf("invalid goarch value %q", a))
		}
	}

	// validate components flag
	for _, a := range g.Components {
		valid := false
		for _, sa := range supportedComponents {
			if a == sa {
				valid = true
				break
			}
		}
		if !valid {
			errs = append(errs, fmt.Errorf("invalid component name %q", a))
		}
	}

	return errs
}

func (g *Plugin) InitPublish() []error {
	var errs []error

	if g.DockerConfig != "" {
		// Hardcode the filename to be config.json for backwards compatibility
		// with the old release.sh script.
		configFileName := path.Join(g.DockerConfig, "config.json")
		f, err := os.Stat(configFileName)
		if err != nil {
			return []error{fmt.Errorf("error checking config file: %v", err)}
		}
		if f.IsDir() {
			return []error{fmt.Errorf("docker config.json is not a file")}
		}
		g.configFileName = configFileName
	}

	return errs
}

func (g *Plugin) Build(ctx context.Context) error {
	_, err := g.build(ctx)
	if err != nil {
		return err
	}

	if g.ExportToDocker {
		log.Info("Exporting docker images to local docker daemon")
		if err := g.exportToDocker(ctx); err != nil {
			return err
		}
	} else {
		log.Info("skipping exporting docker images to docker daemon")
	}

	return nil
}

func (g *Plugin) Publish(ctx context.Context) error {
	log.Info("running publish for image plugin")
	// this case should never be reached, but we check it to be safe
	if !g.built {
		if _, err := g.build(ctx); err != nil {
			return err
		}
	}

	log.Info("pushing images")
	targets := g.generateTargets()
	err := g.pushImages(ctx, targets)
	if err != nil {
		return err
	}

	log.Info("published all docker images")

	return nil
}

func (g *Plugin) Complete() error {
	log = log.WithName("default-flags")

	if g.DockerConfig == "" {
		g.DockerConfig = os.Getenv("DOCKER_CONFIG")
		if g.DockerConfig != "" {
			log.Info("set default value", "flag", "images.docker-config", "value", g.DockerConfig)
		}
	}
	return nil
}

func (g *Plugin) build(ctx context.Context) (imageTargets, error) {
	targets := g.generateTargets()

	bazelTargets := targets.bazelTargets()
	log := log.WithValues("images", bazelTargets)
	log.Info("building bazel image targets")

	// set the os and arch to linux/amd64
	// whilst we might be building cross-arch binaries, cgo only depends on
	// particular OS settings and not arch, so just by setting this to 'linux'
	// we can fix cross builds on platforms other than linux
	// if we support alternate OS values in future, this will need updating
	// with a call to BuildPlatformE per *OS*.
	err := bazel.Default.BuildPlatformE(ctx, log, "linux", "amd64", bazelTargets...)
	if err != nil {
		return nil, fmt.Errorf("error building docker images (%v): %v", targets, err)
	}

	g.built = true
	return targets, nil
}

func (g *Plugin) exportToDocker(ctx context.Context) error {
	targets := g.generateTargets()
	log.WithValues("images", targets.bazelExportTargets()).Info("exporting images to docker daemon")
	for _, target := range targets.bazelExportTargets() {
		log := log.WithValues("target", target)
		log.Info("exporting image to docker daemon")
		// set the os and arch to linux/amd64
		// whilst we might be building cross-arch binaries, cgo only depends on
		// particular OS settings and not arch, so just by setting this to 'linux'
		// we can fix cross builds on platforms other than linux
		// if we support alternate OS values in future, this will need updating
		// with a call to BuildPlatformE per *OS*.
		err := bazel.Default.RunPlatformE(ctx, log, "linux", "amd64", target)
		if err != nil {
			return fmt.Errorf("error exporting image %q to docker daemon: %v", target, err)
		}
	}

	log.WithValues("images", targets.exportedImageNames()).Info("exported all docker images")

	return nil
}

// generateTargets generates a list of Bazel target names that must be
// built for this invocation of the image builder
func (g *Plugin) generateTargets() imageTargets {
	var targets []imageTarget
	for _, c := range g.Components {
		for _, a := range g.GoArch {
			targets = append(targets, imageTarget{c, "linux", a})
		}
	}
	return targets
}

// pushImages will push the images built for this release to the registry
// TODO: add support for calling container_push targets instead of just 'docker push'
func (p *Plugin) pushImages(ctx context.Context, targets imageTargets) error {
	err := p.exportToDocker(ctx)
	if err != nil {
		return err
	}

	var images []string
	for _, t := range targets {
		images = append(images, t.exportedImageNames()...)
	}

	log.WithValues("images", images).Info("pushing docker images")
	for _, img := range images {
		log := log.WithValues("image", img)
		log.Info("pushing docker image")
		args := []string{"push", img}
		if p.configFileName != "" {
			args = append(args, "--config", p.configFileName)
		}
		cmd := exec.CommandContext(ctx, "docker", args...)
		err := util.RunE(log, cmd)
		if err != nil {
			return err
		}
	}

	return nil
}

type imageTargets []imageTarget

func (i imageTargets) bazelTargets() []string {
	out := make([]string, len(i))
	for idx, target := range i {
		out[idx] = target.bazelTarget()
	}
	return out
}

func (i imageTargets) bazelExportTargets() []string {
	out := make([]string, len(i))
	for idx, target := range i {
		out[idx] = target.bazelExportTarget()
	}
	return out
}

func (i imageTargets) exportedImageNames() []string {
	out := make([]string, 0)
	for _, target := range i {
		out = append(out, target.exportedImageNames()...)
	}
	return out
}

type imageTarget struct {
	name, os, arch string
}

func (i imageTarget) bazelTarget() string {
	return fmt.Sprintf("//cmd/%s:image.%s-%s", i.name, i.os, i.arch)
}

func (i imageTarget) bazelExportTarget() string {
	return fmt.Sprintf("//cmd/%s:image.%s-%s.export", i.name, i.os, i.arch)
}

func (i imageTarget) exportedImageNames() []string {
	if i.arch == "amd64" {
		return []string{
			fmt.Sprintf("%s/cert-manager-%s:%s", flags.Default.DockerRepo, i.name, flags.Default.AppVersion),
			fmt.Sprintf("%s/cert-manager-%s:%s", flags.Default.DockerRepo, i.name, flags.Default.GitCommitRef),
		}
	}

	return []string{
		fmt.Sprintf("%s/cert-manager-%s-%s:%s", flags.Default.DockerRepo, i.name, i.arch, flags.Default.AppVersion),
		fmt.Sprintf("%s/cert-manager-%s-%s:%s", flags.Default.DockerRepo, i.name, i.arch, flags.Default.GitCommitRef),
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"os"
	"strings"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	data "github.com/tendermint/go-wire/data"
	"github.com/tendermint/go-wire/data/base58"
)

const (
	HomeFlag     = "home"
	TraceFlag    = "trace"
	OutputFlag   = "output"
	EncodingFlag = "encoding"
)

// Executable is the minimal interface to *corba.Command, so we can
// wrap if desired before the test
type Executable interface {
	Execute() error
}

// PrepareBaseCmd is meant for tendermint and other servers
func PrepareBaseCmd(cmd *cobra.Command, envPrefix, defaultHome string) Executor {
	cobra.OnInitialize(func() { initEnv(envPrefix) })
	cmd.PersistentFlags().StringP(HomeFlag, "", defaultHome, "directory for config and data")
	cmd.PersistentFlags().Bool(TraceFlag, false, "print out full stack trace on errors")
	cmd.PersistentPreRunE = concatCobraCmdFuncs(bindFlagsLoadViper, cmd.PersistentPreRunE)
	return Executor{cmd, os.Exit}
}

// PrepareMainCmd is meant for client side libs that want some more flags
//
// This adds --encoding (hex, btc, base64) and --output (text, json) to
// the command.  These only really make sense in interactive commands.
func PrepareMainCmd(cmd *cobra.Command, envPrefix, defaultHome string) Executor {
	cmd.PersistentFlags().StringP(EncodingFlag, "e", "hex", "Binary encoding (hex|b64|btc)")
	cmd.PersistentFlags().StringP(OutputFlag, "o", "text", "Output format (text|json)")
	cmd.PersistentPreRunE = concatCobraCmdFuncs(setEncoding, validateOutput, cmd.PersistentPreRunE)
	return PrepareBaseCmd(cmd, envPrefix, defaultHome)
}

// initEnv sets to use ENV variables if set.
func initEnv(prefix string) {
	copyEnvVars(prefix)

	// env variables with TM prefix (eg. TM_ROOT)
	viper.SetEnvPrefix(prefix)
	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
	viper.AutomaticEnv()
}

// This copies all variables like TMROOT to TM_ROOT,
// so we can support both formats for the user
func copyEnvVars(prefix string) {
	prefix = strings.ToUpper(prefix)
	ps := prefix + "_"
	for _, e := range os.Environ() {
		kv := strings.SplitN(e, "=", 2)
		if len(kv) == 2 {
			k, v := kv[0], kv[1]
			if strings.HasPrefix(k, prefix) && !strings.HasPrefix(k, ps) {
				k2 := strings.Replace(k, prefix, ps, 1)
				os.Setenv(k2, v)
			}
		}
	}
}

// Executor wraps the cobra Command with a nicer Execute method
type Executor struct {
	*cobra.Command
	Exit func(int) // this is os.Exit by default, override in tests
}

type ExitCoder interface {
	ExitCode() int
}

// execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func (e Executor) Execute() error {
	e.SilenceUsage = true
	e.SilenceErrors = true
	err := e.Command.Execute()
	if err != nil {
		if viper.GetBool(TraceFlag) {
			fmt.Fprintf(os.Stderr, "ERROR: %+v\n", err)
		} else {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		}

		// return error code 1 by default, can override it with a special error type
		exitCode := 1
		if ec, ok := err.(ExitCoder); ok {
			exitCode = ec.ExitCode()
		}
		e.Exit(exitCode)
	}
	return err
}

type cobraCmdFunc func(cmd *cobra.Command, args []string) error

// Returns a single function that calls each argument function in sequence
// RunE, PreRunE, PersistentPreRunE, etc. all have this same signature
func concatCobraCmdFuncs(fs ...cobraCmdFunc) cobraCmdFunc {
	return func(cmd *cobra.Command, args []string) error {
		for _, f := range fs {
			if f != nil {
				if err := f(cmd, args); err != nil {
					return err
				}
			}
		}
		return nil
	}
}

// Bind all flags and read the config into viper
func bindFlagsLoadViper(cmd *cobra.Command, args []string) error {
	// cmd.Flags() includes flags from this command and all persistent flags from the parent
	if err := viper.BindPFlags(cmd.Flags()); err != nil {
		return err
	}

	homeDir := viper.GetString(HomeFlag)
	viper.Set(HomeFlag, homeDir)
	viper.SetConfigName("config") // name of config file (without extension)
	viper.AddConfigPath(homeDir)  // search root directory

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		// stderr, so if we redirect output to json file, this doesn't appear
		// fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed())
	} else if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
		// we ignore not found error, only parse error
		// stderr, so if we redirect output to json file, this doesn't appear
		fmt.Fprintf(os.Stderr, "%#v", err)
	}
	return nil
}

// setEncoding reads the encoding flag
func setEncoding(cmd *cobra.Command, args []string) error {
	// validate and set encoding
	enc := viper.GetString("encoding")
	switch enc {
	case "hex":
		data.Encoder = data.HexEncoder
	case "b64":
		data.Encoder = data.B64Encoder
	case "btc":
		data.Encoder = base58.BTCEncoder
	default:
		return errors.Errorf("Unsupported encoding: %s", enc)
	}
	return nil
}

func validateOutput(cmd *cobra.Command, args []string) error {
	// validate output format
	output := viper.GetString(OutputFlag)
	switch output {
	case "text", "json":
	default:
		return errors.Errorf("Unsupported output format: %s", output)
	}
	return nil
}
<commit_msg>Remove encoding from common cli<commit_after>package cli

import (
	"fmt"
	"os"
	"strings"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

const (
	HomeFlag     = "home"
	TraceFlag    = "trace"
	OutputFlag   = "output"
	EncodingFlag = "encoding"
)

// Executable is the minimal interface to *corba.Command, so we can
// wrap if desired before the test
type Executable interface {
	Execute() error
}

// PrepareBaseCmd is meant for tendermint and other servers
func PrepareBaseCmd(cmd *cobra.Command, envPrefix, defaultHome string) Executor {
	cobra.OnInitialize(func() { initEnv(envPrefix) })
	cmd.PersistentFlags().StringP(HomeFlag, "", defaultHome, "directory for config and data")
	cmd.PersistentFlags().Bool(TraceFlag, false, "print out full stack trace on errors")
	cmd.PersistentPreRunE = concatCobraCmdFuncs(bindFlagsLoadViper, cmd.PersistentPreRunE)
	return Executor{cmd, os.Exit}
}

// PrepareMainCmd is meant for client side libs that want some more flags
//
// This adds --encoding (hex, btc, base64) and --output (text, json) to
// the command.  These only really make sense in interactive commands.
func PrepareMainCmd(cmd *cobra.Command, envPrefix, defaultHome string) Executor {
	cmd.PersistentFlags().StringP(EncodingFlag, "e", "hex", "Binary encoding (hex|b64|btc)")
	cmd.PersistentFlags().StringP(OutputFlag, "o", "text", "Output format (text|json)")
	cmd.PersistentPreRunE = concatCobraCmdFuncs(validateOutput, cmd.PersistentPreRunE)
	return PrepareBaseCmd(cmd, envPrefix, defaultHome)
}

// initEnv sets to use ENV variables if set.
func initEnv(prefix string) {
	copyEnvVars(prefix)

	// env variables with TM prefix (eg. TM_ROOT)
	viper.SetEnvPrefix(prefix)
	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
	viper.AutomaticEnv()
}

// This copies all variables like TMROOT to TM_ROOT,
// so we can support both formats for the user
func copyEnvVars(prefix string) {
	prefix = strings.ToUpper(prefix)
	ps := prefix + "_"
	for _, e := range os.Environ() {
		kv := strings.SplitN(e, "=", 2)
		if len(kv) == 2 {
			k, v := kv[0], kv[1]
			if strings.HasPrefix(k, prefix) && !strings.HasPrefix(k, ps) {
				k2 := strings.Replace(k, prefix, ps, 1)
				os.Setenv(k2, v)
			}
		}
	}
}

// Executor wraps the cobra Command with a nicer Execute method
type Executor struct {
	*cobra.Command
	Exit func(int) // this is os.Exit by default, override in tests
}

type ExitCoder interface {
	ExitCode() int
}

// execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func (e Executor) Execute() error {
	e.SilenceUsage = true
	e.SilenceErrors = true
	err := e.Command.Execute()
	if err != nil {
		if viper.GetBool(TraceFlag) {
			fmt.Fprintf(os.Stderr, "ERROR: %+v\n", err)
		} else {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		}

		// return error code 1 by default, can override it with a special error type
		exitCode := 1
		if ec, ok := err.(ExitCoder); ok {
			exitCode = ec.ExitCode()
		}
		e.Exit(exitCode)
	}
	return err
}

type cobraCmdFunc func(cmd *cobra.Command, args []string) error

// Returns a single function that calls each argument function in sequence
// RunE, PreRunE, PersistentPreRunE, etc. all have this same signature
func concatCobraCmdFuncs(fs ...cobraCmdFunc) cobraCmdFunc {
	return func(cmd *cobra.Command, args []string) error {
		for _, f := range fs {
			if f != nil {
				if err := f(cmd, args); err != nil {
					return err
				}
			}
		}
		return nil
	}
}

// Bind all flags and read the config into viper
func bindFlagsLoadViper(cmd *cobra.Command, args []string) error {
	// cmd.Flags() includes flags from this command and all persistent flags from the parent
	if err := viper.BindPFlags(cmd.Flags()); err != nil {
		return err
	}

	homeDir := viper.GetString(HomeFlag)
	viper.Set(HomeFlag, homeDir)
	viper.SetConfigName("config") // name of config file (without extension)
	viper.AddConfigPath(homeDir)  // search root directory

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		// stderr, so if we redirect output to json file, this doesn't appear
		// fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed())
	} else if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
		// we ignore not found error, only parse error
		// stderr, so if we redirect output to json file, this doesn't appear
		fmt.Fprintf(os.Stderr, "%#v", err)
	}
	return nil
}

func validateOutput(cmd *cobra.Command, args []string) error {
	// validate output format
	output := viper.GetString(OutputFlag)
	switch output {
	case "text", "json":
	default:
		return errors.Errorf("Unsupported output format: %s", output)
	}
	return nil
}
<|endoftext|>


<commit_before>package waddell

import (
	"net"
	"sync"
)

// IdCallback is a callback for id (re)assignments for the client connected to
// the indicated addr.
type IdCallback func(addr string, id PeerId)

// ClientMgr provides a mechanism for managing connections to multiple waddell
// servers.
type ClientMgr struct {
	// Dial is a function that dials the waddell server at the given addr.
	Dial func(addr string) (net.Conn, error)

	// ReconnectAttempts specifies how many consecutive times to try
	// reconnecting in the event of a connection failure. See
	// Client.ReconnectAttempts for more information.
	ReconnectAttempts int

	// IdCallback optional allows specifying a callback that gets notified
	// everytime a client is assigned a new PeerId
	IdCallback IdCallback

	clients      map[string]*Client
	ids          map[string]PeerId
	clientsMutex sync.Mutex
}

func (m *ClientMgr) ClientTo(addr string) (*Client, PeerId, error) {
	m.clientsMutex.Lock()
	defer m.clientsMutex.Unlock()
	if m.clients == nil {
		m.clients = make(map[string]*Client)
		m.ids = make(map[string]PeerId)
	}
	client := m.clients[addr]
	if client == nil {
		client = &Client{
			Dial: func() (net.Conn, error) {
				return m.Dial(addr)
			},
			ReconnectAttempts: m.ReconnectAttempts,
		}
		id, err := client.Connect()
		if err != nil {
			return nil, id, err
		}
		m.clients[addr] = client
		m.ids[addr] = id
		if m.IdCallback != nil {
			m.IdCallback(addr, id)
			go func() {
				for id := range client.UpdatedIdsCh {
					m.IdCallback(addr, id)
				}
			}()
		}
	}
	id := m.ids[addr]
	return client, id, nil
}
<commit_msg>Removed IdCallback<commit_after>package waddell

import (
	"net"
	"sync"
)

// ClientMgr provides a mechanism for managing connections to multiple waddell
// servers.
type ClientMgr struct {
	// Dial is a function that dials the waddell server at the given addr.
	Dial func(addr string) (net.Conn, error)

	// ReconnectAttempts specifies how many consecutive times to try
	// reconnecting in the event of a connection failure. See
	// Client.ReconnectAttempts for more information.
	ReconnectAttempts int

	clients      map[string]*Client
	ids          map[string]PeerId
	clientsMutex sync.Mutex
}

func (m *ClientMgr) ClientTo(addr string) (*Client, PeerId, error) {
	m.clientsMutex.Lock()
	defer m.clientsMutex.Unlock()
	if m.clients == nil {
		m.clients = make(map[string]*Client)
		m.ids = make(map[string]PeerId)
	}
	client := m.clients[addr]
	if client == nil {
		client = &Client{
			Dial: func() (net.Conn, error) {
				return m.Dial(addr)
			},
			ReconnectAttempts: m.ReconnectAttempts,
		}
		id, err := client.Connect()
		if err != nil {
			return nil, id, err
		}
		m.clients[addr] = client
		m.ids[addr] = id
	}
	id := m.ids[addr]
	return client, id, nil
}
<|endoftext|>


<commit_before>package veneur

import (
	"crypto/tls"
	"fmt"
	"net"
	"os"
	"sync"

	"github.com/sirupsen/logrus"
	flock "github.com/theckman/go-flock"
)

// StartStatsd spawns a goroutine that listens for metrics in statsd
// format on the address a. As this is a setup routine, if any error
// occurs, it panics.
func StartStatsd(s *Server, a net.Addr, packetPool *sync.Pool) {
	switch addr := a.(type) {
	case *net.UDPAddr:
		startStatsdUDP(s, addr, packetPool)
	case *net.TCPAddr:
		startStatsdTCP(s, addr, packetPool)
	default:
		panic(fmt.Sprintf("Can't listen on %v: only TCP and UDP are supported", a))
	}
}

func startStatsdUDP(s *Server, addr *net.UDPAddr, packetPool *sync.Pool) {
	for i := 0; i < s.numReaders; i++ {
		go func() {
			defer func() {
				ConsumePanic(s.Sentry, s.Statsd, s.Hostname, recover())
			}()
			// each goroutine gets its own socket
			// if the sockets support SO_REUSEPORT, then this will cause the
			// kernel to distribute datagrams across them, for better read
			// performance
			sock, err := NewSocket(addr, s.RcvbufBytes, s.numReaders != 1)
			if err != nil {
				// if any goroutine fails to create the socket, we can't really
				// recover, so we just blow up
				// this probably indicates a systemic issue, eg lack of
				// SO_REUSEPORT support
				panic(fmt.Sprintf("couldn't listen on UDP socket %v: %v", addr, err))
			}
			log.WithField("address", addr).Info("Listening for statsd metrics on UDP socket")
			s.ReadMetricSocket(sock, packetPool)
		}()
	}
}

func startStatsdTCP(s *Server, addr *net.TCPAddr, packetPool *sync.Pool) {
	var listener net.Listener
	var err error

	listener, err = net.ListenTCP("tcp", addr)
	if err != nil {
		panic(fmt.Sprintf("couldn't listen on TCP socket %v: %v", addr, err))
	}

	go func() {
		<-s.shutdown
		// TODO: the socket is in use until there are no goroutines blocked in Accept
		// we should wait until the accepting goroutine exits
		err := listener.Close()
		if err != nil {
			log.WithError(err).Warn("Ignoring error closing TCP listener")
		}
	}()

	mode := "unencrypted"
	if s.tlsConfig != nil {
		// wrap the listener with TLS
		listener = tls.NewListener(listener, s.tlsConfig)
		if s.tlsConfig.ClientAuth == tls.RequireAndVerifyClientCert {
			mode = "authenticated"
		} else {
			mode = "encrypted"
		}
	}

	log.WithFields(logrus.Fields{
		"address": addr, "mode": mode,
	}).Info("Listening for statsd metrics on TCP socket")

	go func() {
		defer func() {
			ConsumePanic(s.Sentry, s.Statsd, s.Hostname, recover())
		}()
		s.ReadTCPSocket(listener)
	}()
}

func StartSSF(s *Server, a net.Addr, tracePool *sync.Pool) {
	switch addr := a.(type) {
	case *net.UDPAddr:
		startSSFUDP(s, addr, tracePool)
	case *net.UnixAddr:
		startSSFUnix(s, addr)
	default:
		panic(fmt.Sprintf("Can't listen for SSF on %v: only udp:// & unix:// are supported", a))
	}
	log.WithFields(logrus.Fields{
		"address": a.String(),
		"network": a.Network(),
	}).Info("Listening for SSF traces")
}

func startSSFUDP(s *Server, addr *net.UDPAddr, tracePool *sync.Pool) {
	// TODO: Make this actually use readers / add a predicate
	// function for testing if we should SO_REUSEPORT.
	listener, err := NewSocket(addr, s.RcvbufBytes, s.numReaders > 1)
	if err != nil {
		// if any goroutine fails to create the socket, we can't really
		// recover, so we just blow up
		// this probably indicates a systemic issue, eg lack of
		// SO_REUSEPORT support
		panic(fmt.Sprintf("couldn't listen on UDP socket %v: %v", addr, err))
	}
	go func() {
		defer func() {
			ConsumePanic(s.Sentry, s.Statsd, s.Hostname, recover())
		}()
		s.ReadSSFPacketSocket(listener, tracePool)
	}()
}

// startSSFUnix starts listening for connections that send framed SSF
// spans on a UNIX domain socket address. It does so until the
// server's shutdown socket is closed. startSSFUnix returns a channel
// that is closed once the listener has terminated.
func startSSFUnix(s *Server, addr *net.UnixAddr) <-chan struct{} {
	done := make(chan struct{})
	if addr.Network() != "unix" {
		panic(fmt.Sprintf("Can't listen for SSF on %v: only udp:// and unix:// addresses are supported", addr))
	}
	// ensure we are the only ones locking this socket:
	lockname := fmt.Sprintf("%s.lock", addr.String())
	lock := flock.NewFlock(lockname)
	locked, err := lock.TryLock()
	if err != nil {
		panic(fmt.Sprintf("Could not acquire the lock %q to listen on %v: %v", lockname, addr, err))
	}
	if !locked {
		panic(fmt.Sprintf("Lock file %q for %v is in use by another process already", lockname, addr))
	}
	// We have the exclusive use of the socket, clear away any old sockets and listen:
	_ = os.Remove(addr.String())
	listener, err := net.ListenUnix(addr.Network(), addr)
	if err != nil {
		panic(fmt.Sprintf("Couldn't listen on UNIX socket %v: %v", addr, err))
	}

	// Make the socket connectable by everyone with access to the socket pathname:
	err = os.Chmod(addr.String(), 0666)
	if err != nil {
		panic(fmt.Sprintf("Couldn't set permissions on %v: %v", addr, err))
	}

	go func() {
		conns := make(chan net.Conn)
		go func() {
			defer lock.Unlock()
			defer close(done)
			for {
				conn, err := listener.AcceptUnix()
				if err != nil {
					select {
					case <-s.shutdown:
						// occurs when cleanly shutting down the server e.g. in tests; ignore errors
						log.WithError(err).Info("Ignoring Accept error while shutting down")
						return
					default:
						log.WithError(err).Fatal("Unix accept failed")
					}
				}
				conns <- conn
			}
		}()
		for {
			select {
			case conn := <-conns:
				go s.ReadSSFStreamSocket(conn)
			case <-s.shutdown:
				listener.Close()
				return
			}
		}
	}()
	return done
}
<commit_msg>Fix shutdown race in startSSFUnix<commit_after>package veneur

import (
	"crypto/tls"
	"fmt"
	"net"
	"os"
	"sync"

	"github.com/sirupsen/logrus"
	flock "github.com/theckman/go-flock"
)

// StartStatsd spawns a goroutine that listens for metrics in statsd
// format on the address a. As this is a setup routine, if any error
// occurs, it panics.
func StartStatsd(s *Server, a net.Addr, packetPool *sync.Pool) {
	switch addr := a.(type) {
	case *net.UDPAddr:
		startStatsdUDP(s, addr, packetPool)
	case *net.TCPAddr:
		startStatsdTCP(s, addr, packetPool)
	default:
		panic(fmt.Sprintf("Can't listen on %v: only TCP and UDP are supported", a))
	}
}

func startStatsdUDP(s *Server, addr *net.UDPAddr, packetPool *sync.Pool) {
	for i := 0; i < s.numReaders; i++ {
		go func() {
			defer func() {
				ConsumePanic(s.Sentry, s.Statsd, s.Hostname, recover())
			}()
			// each goroutine gets its own socket
			// if the sockets support SO_REUSEPORT, then this will cause the
			// kernel to distribute datagrams across them, for better read
			// performance
			sock, err := NewSocket(addr, s.RcvbufBytes, s.numReaders != 1)
			if err != nil {
				// if any goroutine fails to create the socket, we can't really
				// recover, so we just blow up
				// this probably indicates a systemic issue, eg lack of
				// SO_REUSEPORT support
				panic(fmt.Sprintf("couldn't listen on UDP socket %v: %v", addr, err))
			}
			log.WithField("address", addr).Info("Listening for statsd metrics on UDP socket")
			s.ReadMetricSocket(sock, packetPool)
		}()
	}
}

func startStatsdTCP(s *Server, addr *net.TCPAddr, packetPool *sync.Pool) {
	var listener net.Listener
	var err error

	listener, err = net.ListenTCP("tcp", addr)
	if err != nil {
		panic(fmt.Sprintf("couldn't listen on TCP socket %v: %v", addr, err))
	}

	go func() {
		<-s.shutdown
		// TODO: the socket is in use until there are no goroutines blocked in Accept
		// we should wait until the accepting goroutine exits
		err := listener.Close()
		if err != nil {
			log.WithError(err).Warn("Ignoring error closing TCP listener")
		}
	}()

	mode := "unencrypted"
	if s.tlsConfig != nil {
		// wrap the listener with TLS
		listener = tls.NewListener(listener, s.tlsConfig)
		if s.tlsConfig.ClientAuth == tls.RequireAndVerifyClientCert {
			mode = "authenticated"
		} else {
			mode = "encrypted"
		}
	}

	log.WithFields(logrus.Fields{
		"address": addr, "mode": mode,
	}).Info("Listening for statsd metrics on TCP socket")

	go func() {
		defer func() {
			ConsumePanic(s.Sentry, s.Statsd, s.Hostname, recover())
		}()
		s.ReadTCPSocket(listener)
	}()
}

func StartSSF(s *Server, a net.Addr, tracePool *sync.Pool) {
	switch addr := a.(type) {
	case *net.UDPAddr:
		startSSFUDP(s, addr, tracePool)
	case *net.UnixAddr:
		startSSFUnix(s, addr)
	default:
		panic(fmt.Sprintf("Can't listen for SSF on %v: only udp:// & unix:// are supported", a))
	}
	log.WithFields(logrus.Fields{
		"address": a.String(),
		"network": a.Network(),
	}).Info("Listening for SSF traces")
}

func startSSFUDP(s *Server, addr *net.UDPAddr, tracePool *sync.Pool) {
	// TODO: Make this actually use readers / add a predicate
	// function for testing if we should SO_REUSEPORT.
	listener, err := NewSocket(addr, s.RcvbufBytes, s.numReaders > 1)
	if err != nil {
		// if any goroutine fails to create the socket, we can't really
		// recover, so we just blow up
		// this probably indicates a systemic issue, eg lack of
		// SO_REUSEPORT support
		panic(fmt.Sprintf("couldn't listen on UDP socket %v: %v", addr, err))
	}
	go func() {
		defer func() {
			ConsumePanic(s.Sentry, s.Statsd, s.Hostname, recover())
		}()
		s.ReadSSFPacketSocket(listener, tracePool)
	}()
}

// startSSFUnix starts listening for connections that send framed SSF
// spans on a UNIX domain socket address. It does so until the
// server's shutdown socket is closed. startSSFUnix returns a channel
// that is closed once the listener has terminated.
func startSSFUnix(s *Server, addr *net.UnixAddr) <-chan struct{} {
	done := make(chan struct{})
	if addr.Network() != "unix" {
		panic(fmt.Sprintf("Can't listen for SSF on %v: only udp:// and unix:// addresses are supported", addr))
	}
	// ensure we are the only ones locking this socket:
	lockname := fmt.Sprintf("%s.lock", addr.String())
	lock := flock.NewFlock(lockname)
	locked, err := lock.TryLock()
	if err != nil {
		panic(fmt.Sprintf("Could not acquire the lock %q to listen on %v: %v", lockname, addr, err))
	}
	if !locked {
		panic(fmt.Sprintf("Lock file %q for %v is in use by another process already", lockname, addr))
	}
	// We have the exclusive use of the socket, clear away any old sockets and listen:
	_ = os.Remove(addr.String())
	listener, err := net.ListenUnix(addr.Network(), addr)
	if err != nil {
		panic(fmt.Sprintf("Couldn't listen on UNIX socket %v: %v", addr, err))
	}

	// Make the socket connectable by everyone with access to the socket pathname:
	err = os.Chmod(addr.String(), 0666)
	if err != nil {
		panic(fmt.Sprintf("Couldn't set permissions on %v: %v", addr, err))
	}

	go func() {
		conns := make(chan net.Conn)
		go func() {
			defer func() {
				lock.Unlock()
				close(done)
			}()
			for {
				conn, err := listener.AcceptUnix()
				if err != nil {
					select {
					case <-s.shutdown:
						// occurs when cleanly shutting down the server e.g. in tests; ignore errors
						log.WithError(err).Info("Ignoring Accept error while shutting down")
						return
					default:
						log.WithError(err).Fatal("Unix accept failed")
					}
				}
				conns <- conn
			}
		}()
		for {
			select {
			case conn := <-conns:
				go s.ReadSSFStreamSocket(conn)
			case <-s.shutdown:
				listener.Close()
				return
			}
		}
	}()
	return done
}
<|endoftext|>


<commit_before>package itchfs

import (
	"fmt"
	"path"
	"strconv"
	"strings"

	"github.com/go-errors/errors"
	itchio "github.com/itchio/go-itchio"
	"github.com/itchio/httpfile"
)

// over-engineering follows

type SourceType int

const (
	// 691 is the earliest Bernoulli number that's also an irregular prime.
	// It's also sufficiently different from zero that it should surface
	// coding errors more easily
	SourceType_DownloadBuild SourceType = 691 + iota
	SourceType_KeyDownloadBuild
	SourceType_WharfDownloadBuild
	SourceType_DownloadUpload
)

type Source struct {
	Type       SourceType
	ItchClient *itchio.Client
	Path       string
}

var patterns = map[string]SourceType{
	"/upload/*/download":                  SourceType_DownloadUpload,
	"/upload/*/download/builds/*":         SourceType_DownloadBuild,
	"/download-key/*/download/*/builds/*": SourceType_KeyDownloadBuild,
	"/wharf/builds/*/files/*/download":    SourceType_WharfDownloadBuild,
}

func ObtainSource(itchClient *itchio.Client, itchPath string) (*Source, error) {
	var matches bool
	var err error

	for pattern, sourceType := range patterns {
		matches, err = path.Match(pattern, itchPath)
		if err != nil {
			return nil, err
		}

		if matches {
			return &Source{
				Type:       sourceType,
				ItchClient: itchClient,
				Path:       itchPath,
			}, nil
		}
	}

	return nil, fmt.Errorf("unrecognized itchfs pattern: %s", itchPath)
}

func (s *Source) makeGetURL() (httpfile.GetURLFunc, error) {
	tokens := strings.Split(s.Path, "/")

	var getter httpfile.GetURLFunc

	switch s.Type {
	case SourceType_DownloadBuild:
		uploadID, err := strconv.ParseInt(tokens[2], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		buildID, err := strconv.ParseInt(tokens[5], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		getter = func() (string, error) {
			r, err := s.ItchClient.DownloadUploadBuild(uploadID, buildID)
			if err != nil {
				return "", err
			}

			return r.Archive.URL, nil
		}
	case SourceType_KeyDownloadBuild:
		downloadKey := tokens[2]

		uploadID, err := strconv.ParseInt(tokens[4], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		buildID, err := strconv.ParseInt(tokens[6], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		getter = func() (string, error) {
			r, err := s.ItchClient.DownloadUploadBuildWithKey(downloadKey, uploadID, buildID)
			if err != nil {
				return "", err
			}

			return r.Archive.URL, nil
		}
	case SourceType_WharfDownloadBuild:
		buildID, err := strconv.ParseInt(tokens[3], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		buildFileID, err := strconv.ParseInt(tokens[5], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		getter = func() (string, error) {
			r, err := s.ItchClient.GetBuildFileDownloadURL(buildID, buildFileID)
			if err != nil {
				return "", err
			}

			return r.URL, nil
		}
	case SourceType_DownloadUpload:
		uploadID, err := strconv.ParseInt(tokens[2], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		getter = func() (string, error) {
			r, err := s.ItchClient.UploadDownload(uploadID)
			if err != nil {
				return "", err
			}

			return r.URL, nil
		}
	default:
		return nil, fmt.Errorf("unsupported source type: %d", s.Type)
	}

	return getter, nil
}
<commit_msg>Allow downloading signature/patch/manifest/etc.<commit_after>package itchfs

import (
	"fmt"
	"path"
	"strconv"
	"strings"

	"github.com/go-errors/errors"
	itchio "github.com/itchio/go-itchio"
	"github.com/itchio/httpfile"
)

// over-engineering follows

type SourceType int

const (
	// 691 is the earliest Bernoulli number that's also an irregular prime.
	// It's also sufficiently different from zero that it should surface
	// coding errors more easily
	SourceType_DownloadBuild SourceType = 691 + iota
	SourceType_KeyDownloadBuild
	SourceType_WharfDownloadBuild
	SourceType_DownloadUpload
)

type Source struct {
	Type       SourceType
	ItchClient *itchio.Client
	Path       string
}

var patterns = map[string]SourceType{
	"/upload/*/download":                    SourceType_DownloadUpload,
	"/upload/*/download/builds/*/*":         SourceType_DownloadBuild,
	"/download-key/*/download/*/builds/*/*": SourceType_KeyDownloadBuild,
	"/wharf/builds/*/files/*/download":      SourceType_WharfDownloadBuild,
}

func ObtainSource(itchClient *itchio.Client, itchPath string) (*Source, error) {
	var matches bool
	var err error

	for pattern, sourceType := range patterns {
		matches, err = path.Match(pattern, itchPath)
		if err != nil {
			return nil, err
		}

		if matches {
			return &Source{
				Type:       sourceType,
				ItchClient: itchClient,
				Path:       itchPath,
			}, nil
		}
	}

	return nil, fmt.Errorf("unrecognized itchfs pattern: %s", itchPath)
}

func serveBuildFile(r itchio.DownloadUploadBuildResponse, fileType string) (string, error) {
	switch fileType {
	case "archive":
		return r.Archive.URL, nil
	case "patch":
		return r.Patch.URL, nil
	case "signature":
		return r.Signature.URL, nil
	case "manifest":
		return r.Manifest.URL, nil
	}

	return "", fmt.Errorf("unknown file type %s", fileType)
}

func (s *Source) makeGetURL() (httpfile.GetURLFunc, error) {
	tokens := strings.Split(s.Path, "/")

	var getter httpfile.GetURLFunc

	switch s.Type {
	case SourceType_DownloadBuild:
		uploadID, err := strconv.ParseInt(tokens[2], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		buildID, err := strconv.ParseInt(tokens[5], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		fileType := tokens[6]

		getter = func() (string, error) {
			r, err := s.ItchClient.DownloadUploadBuild(uploadID, buildID)
			if err != nil {
				return "", err
			}

			return serveBuildFile(r, fileType)
		}
	case SourceType_KeyDownloadBuild:
		downloadKey := tokens[2]

		uploadID, err := strconv.ParseInt(tokens[4], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		buildID, err := strconv.ParseInt(tokens[6], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		fileType := tokens[7]

		getter = func() (string, error) {
			r, err := s.ItchClient.DownloadUploadBuildWithKey(downloadKey, uploadID, buildID)
			if err != nil {
				return "", err
			}

			return serveBuildFile(r, fileType)
		}
	case SourceType_WharfDownloadBuild:
		buildID, err := strconv.ParseInt(tokens[3], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		buildFileID, err := strconv.ParseInt(tokens[5], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		getter = func() (string, error) {
			r, err := s.ItchClient.GetBuildFileDownloadURL(buildID, buildFileID)
			if err != nil {
				return "", err
			}

			return r.URL, nil
		}
	case SourceType_DownloadUpload:
		uploadID, err := strconv.ParseInt(tokens[2], 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, 1)
		}

		getter = func() (string, error) {
			r, err := s.ItchClient.UploadDownload(uploadID)
			if err != nil {
				return "", err
			}

			return r.URL, nil
		}
	default:
		return nil, fmt.Errorf("unsupported source type: %d", s.Type)
	}

	return getter, nil
}
<|endoftext|>


<commit_before>/*
 * config.go - Actions for creating a new config file, which includes new
 * hashing costs and the config file's location.
 *
 * Copyright 2017 Google Inc.
 * Author: Joe Richey (joerichey@google.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package actions

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"runtime"
	"time"

	"golang.org/x/sys/unix"

	"github.com/google/fscrypt/crypto"
	"github.com/google/fscrypt/filesystem"
	"github.com/google/fscrypt/metadata"
	"github.com/google/fscrypt/util"
)

// ConfigFileLocation is the location of fscrypt's global settings. This can be
// overridden by the user of this package.
var ConfigFileLocation = "/etc/fscrypt.conf"

// ErrBadConfig is an internal error that indicates that the config struct is invalid.
type ErrBadConfig struct {
	Config          *metadata.Config
	UnderlyingError error
}

func (err *ErrBadConfig) Error() string {
	return fmt.Sprintf(`internal error: config is invalid: %s

	The invalid config is %s`, err.UnderlyingError, err.Config)
}

// ErrBadConfigFile indicates that the config file is invalid.
type ErrBadConfigFile struct {
	Path            string
	UnderlyingError error
}

func (err *ErrBadConfigFile) Error() string {
	return fmt.Sprintf("%q is invalid: %s", err.Path, err.UnderlyingError)
}

// ErrConfigFileExists indicates that the config file already exists.
type ErrConfigFileExists struct {
	Path string
}

func (err *ErrConfigFileExists) Error() string {
	return fmt.Sprintf("%q already exists", err.Path)
}

// ErrNoConfigFile indicates that the config file doesn't exist.
type ErrNoConfigFile struct {
	Path string
}

func (err *ErrNoConfigFile) Error() string {
	return fmt.Sprintf("%q doesn't exist", err.Path)
}

const (
	// Permissions of the config file (global readable)
	configPermissions = 0644
	// Config file should be created for writing and not already exist
	createFlags = os.O_CREATE | os.O_WRONLY | os.O_EXCL
	// 128 MiB is a large enough amount of memory to make the password hash
	// very difficult to brute force on specialized hardware, but small
	// enough to work on most GNU/Linux systems.
	maxMemoryBytes = 128 * 1024 * 1024
)

var (
	timingPassphrase = []byte("I am a fake passphrase")
	timingSalt       = bytes.Repeat([]byte{42}, metadata.SaltLen)
)

// CreateConfigFile creates a new config file at the appropriate location with
// the appropriate hashing costs and encryption parameters. The hashing will be
// configured to take as long as the specified time target. In addition, the
// version of encryption policy to use may be overridden from the default of v1.
func CreateConfigFile(target time.Duration, policyVersion int64) error {
	// Create the config file before computing the hashing costs, so we fail
	// immediately if the program has insufficient permissions.
	configFile, err := filesystem.OpenFileOverridingUmask(ConfigFileLocation,
		createFlags, configPermissions)
	switch {
	case os.IsExist(err):
		return &ErrConfigFileExists{ConfigFileLocation}
	case err != nil:
		return err
	}
	defer configFile.Close()

	config := &metadata.Config{
		Source:  metadata.DefaultSource,
		Options: metadata.DefaultOptions,
	}

	if policyVersion != 0 {
		config.Options.PolicyVersion = policyVersion
	}

	if config.HashCosts, err = getHashingCosts(target); err != nil {
		return err
	}

	log.Printf("Creating config at %q with %v\n", ConfigFileLocation, config)
	return metadata.WriteConfig(config, configFile)
}

// getConfig returns the current configuration struct. Any fields not specified
// in the config file use the system defaults. An error is returned if the
// config file hasn't been setup with CreateConfigFile yet or the config
// contains invalid data.
func getConfig() (*metadata.Config, error) {
	configFile, err := os.Open(ConfigFileLocation)
	switch {
	case os.IsNotExist(err):
		return nil, &ErrNoConfigFile{ConfigFileLocation}
	case err != nil:
		return nil, err
	}
	defer configFile.Close()

	log.Printf("Reading config from %q\n", ConfigFileLocation)
	config, err := metadata.ReadConfig(configFile)
	if err != nil {
		return nil, &ErrBadConfigFile{ConfigFileLocation, err}
	}

	// Use system defaults if not specified
	if config.Source == metadata.SourceType_default {
		config.Source = metadata.DefaultSource
		log.Printf("Falling back to source of %q", config.Source.String())
	}
	if config.Options.Padding == 0 {
		config.Options.Padding = metadata.DefaultOptions.Padding
		log.Printf("Falling back to padding of %d", config.Options.Padding)
	}
	if config.Options.Contents == metadata.EncryptionOptions_default {
		config.Options.Contents = metadata.DefaultOptions.Contents
		log.Printf("Falling back to contents mode of %q", config.Options.Contents)
	}
	if config.Options.Filenames == metadata.EncryptionOptions_default {
		config.Options.Filenames = metadata.DefaultOptions.Filenames
		log.Printf("Falling back to filenames mode of %q", config.Options.Filenames)
	}
	if config.Options.PolicyVersion == 0 {
		config.Options.PolicyVersion = metadata.DefaultOptions.PolicyVersion
		log.Printf("Falling back to policy version of %d", config.Options.PolicyVersion)
	}

	if err := config.CheckValidity(); err != nil {
		return nil, &ErrBadConfigFile{ConfigFileLocation, err}
	}

	return config, nil
}

// getHashingCosts returns hashing costs so that hashing a password will take
// approximately the target time. This is done using the total amount of RAM,
// the number of CPUs present, and by running the passphrase hash many times.
func getHashingCosts(target time.Duration) (*metadata.HashingCosts, error) {
	log.Printf("Finding hashing costs that take %v\n", target)

	// Start out with the minimal possible costs that use all the CPUs.
	nCPUs := int64(runtime.NumCPU())
	costs := &metadata.HashingCosts{
		Time:        1,
		Memory:      8 * nCPUs,
		Parallelism: nCPUs,
	}

	// If even the minimal costs are not fast enough, just return the
	// minimal costs and log a warning.
	t, err := timeHashingCosts(costs)
	if err != nil {
		return nil, err
	}
	log.Printf("Min Costs={%v}\t-> %v\n", costs, t)

	if t > target {
		log.Printf("time exceeded the target of %v.\n", target)
		return costs, nil
	}

	// Now we start doubling the costs until we reach the target.
	memoryKiBLimit := memoryBytesLimit() / 1024
	for {
		// Store a copy of the previous costs
		costsPrev := *costs
		tPrev := t

		// Double the memory up to the max, then double the time.
		if costs.Memory < memoryKiBLimit {
			costs.Memory = util.MinInt64(2*costs.Memory, memoryKiBLimit)
		} else {
			costs.Time *= 2
		}

		// If our hashing failed, return the last good set of costs.
		if t, err = timeHashingCosts(costs); err != nil {
			log.Printf("Hashing with costs={%v} failed: %v\n", costs, err)
			return &costsPrev, nil
		}
		log.Printf("Costs={%v}\t-> %v\n", costs, t)

		// If we have reached the target time, we return a set of costs
		// based on the linear interpolation between the last two times.
		if t >= target {
			f := float64(target-tPrev) / float64(t-tPrev)
			return &metadata.HashingCosts{
				Time:        betweenCosts(costsPrev.Time, costs.Time, f),
				Memory:      betweenCosts(costsPrev.Memory, costs.Memory, f),
				Parallelism: costs.Parallelism,
			}, nil
		}
	}
}

// memoryBytesLimit returns the maximum amount of memory we will use for
// passphrase hashing. This will never be more than a reasonable maximum (for
// compatibility) or half the available system RAM.
func memoryBytesLimit() int64 {
	// The sysinfo syscall only fails if given a bad address
	var info unix.Sysinfo_t
	err := unix.Sysinfo(&info)
	util.NeverError(err)

	totalRAMBytes := int64(info.Totalram)
	return util.MinInt64(totalRAMBytes/2, maxMemoryBytes)
}

// betweenCosts returns a cost between a and b. Specifically, it returns the
// floor of a + f*(b-a). This way, f=0 returns a and f=1 returns b.
func betweenCosts(a, b int64, f float64) int64 {
	return a + int64(f*float64(b-a))
}

// timeHashingCosts runs the passphrase hash with the specified costs and
// returns the time it takes to hash the passphrase.
func timeHashingCosts(costs *metadata.HashingCosts) (time.Duration, error) {
	passphrase, err := crypto.NewKeyFromReader(bytes.NewReader(timingPassphrase))
	if err != nil {
		return 0, err
	}
	defer passphrase.Wipe()

	// Be sure to measure CPU time, not wall time (time.Now)
	begin := cpuTimeInNanoseconds()
	hash, err := crypto.PassphraseHash(passphrase, timingSalt, costs)
	if err == nil {
		hash.Wipe()
	}
	end := cpuTimeInNanoseconds()

	return time.Duration((end - begin) / costs.Parallelism), nil
}

// cpuTimeInNanoseconds returns the nanosecond count based on the process's CPU usage.
// This number has no absolute meaning, only relative meaning to other calls.
func cpuTimeInNanoseconds() int64 {
	var ts unix.Timespec
	err := unix.ClockGettime(unix.CLOCK_PROCESS_CPUTIME_ID, &ts)
	// ClockGettime fails if given a bad address or on a VERY old system.
	util.NeverError(err)
	return unix.TimespecToNsec(ts)
}
<commit_msg>Only use 1/8 of the system RAM<commit_after>/*
 * config.go - Actions for creating a new config file, which includes new
 * hashing costs and the config file's location.
 *
 * Copyright 2017 Google Inc.
 * Author: Joe Richey (joerichey@google.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package actions

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"runtime"
	"time"

	"golang.org/x/sys/unix"

	"github.com/google/fscrypt/crypto"
	"github.com/google/fscrypt/filesystem"
	"github.com/google/fscrypt/metadata"
	"github.com/google/fscrypt/util"
)

// ConfigFileLocation is the location of fscrypt's global settings. This can be
// overridden by the user of this package.
var ConfigFileLocation = "/etc/fscrypt.conf"

// ErrBadConfig is an internal error that indicates that the config struct is invalid.
type ErrBadConfig struct {
	Config          *metadata.Config
	UnderlyingError error
}

func (err *ErrBadConfig) Error() string {
	return fmt.Sprintf(`internal error: config is invalid: %s

	The invalid config is %s`, err.UnderlyingError, err.Config)
}

// ErrBadConfigFile indicates that the config file is invalid.
type ErrBadConfigFile struct {
	Path            string
	UnderlyingError error
}

func (err *ErrBadConfigFile) Error() string {
	return fmt.Sprintf("%q is invalid: %s", err.Path, err.UnderlyingError)
}

// ErrConfigFileExists indicates that the config file already exists.
type ErrConfigFileExists struct {
	Path string
}

func (err *ErrConfigFileExists) Error() string {
	return fmt.Sprintf("%q already exists", err.Path)
}

// ErrNoConfigFile indicates that the config file doesn't exist.
type ErrNoConfigFile struct {
	Path string
}

func (err *ErrNoConfigFile) Error() string {
	return fmt.Sprintf("%q doesn't exist", err.Path)
}

const (
	// Permissions of the config file (global readable)
	configPermissions = 0644
	// Config file should be created for writing and not already exist
	createFlags = os.O_CREATE | os.O_WRONLY | os.O_EXCL
	// 128 MiB is a large enough amount of memory to make the password hash
	// very difficult to brute force on specialized hardware, but small
	// enough to work on most GNU/Linux systems.
	maxMemoryBytes = 128 * 1024 * 1024
)

var (
	timingPassphrase = []byte("I am a fake passphrase")
	timingSalt       = bytes.Repeat([]byte{42}, metadata.SaltLen)
)

// CreateConfigFile creates a new config file at the appropriate location with
// the appropriate hashing costs and encryption parameters. The hashing will be
// configured to take as long as the specified time target. In addition, the
// version of encryption policy to use may be overridden from the default of v1.
func CreateConfigFile(target time.Duration, policyVersion int64) error {
	// Create the config file before computing the hashing costs, so we fail
	// immediately if the program has insufficient permissions.
	configFile, err := filesystem.OpenFileOverridingUmask(ConfigFileLocation,
		createFlags, configPermissions)
	switch {
	case os.IsExist(err):
		return &ErrConfigFileExists{ConfigFileLocation}
	case err != nil:
		return err
	}
	defer configFile.Close()

	config := &metadata.Config{
		Source:  metadata.DefaultSource,
		Options: metadata.DefaultOptions,
	}

	if policyVersion != 0 {
		config.Options.PolicyVersion = policyVersion
	}

	if config.HashCosts, err = getHashingCosts(target); err != nil {
		return err
	}

	log.Printf("Creating config at %q with %v\n", ConfigFileLocation, config)
	return metadata.WriteConfig(config, configFile)
}

// getConfig returns the current configuration struct. Any fields not specified
// in the config file use the system defaults. An error is returned if the
// config file hasn't been setup with CreateConfigFile yet or the config
// contains invalid data.
func getConfig() (*metadata.Config, error) {
	configFile, err := os.Open(ConfigFileLocation)
	switch {
	case os.IsNotExist(err):
		return nil, &ErrNoConfigFile{ConfigFileLocation}
	case err != nil:
		return nil, err
	}
	defer configFile.Close()

	log.Printf("Reading config from %q\n", ConfigFileLocation)
	config, err := metadata.ReadConfig(configFile)
	if err != nil {
		return nil, &ErrBadConfigFile{ConfigFileLocation, err}
	}

	// Use system defaults if not specified
	if config.Source == metadata.SourceType_default {
		config.Source = metadata.DefaultSource
		log.Printf("Falling back to source of %q", config.Source.String())
	}
	if config.Options.Padding == 0 {
		config.Options.Padding = metadata.DefaultOptions.Padding
		log.Printf("Falling back to padding of %d", config.Options.Padding)
	}
	if config.Options.Contents == metadata.EncryptionOptions_default {
		config.Options.Contents = metadata.DefaultOptions.Contents
		log.Printf("Falling back to contents mode of %q", config.Options.Contents)
	}
	if config.Options.Filenames == metadata.EncryptionOptions_default {
		config.Options.Filenames = metadata.DefaultOptions.Filenames
		log.Printf("Falling back to filenames mode of %q", config.Options.Filenames)
	}
	if config.Options.PolicyVersion == 0 {
		config.Options.PolicyVersion = metadata.DefaultOptions.PolicyVersion
		log.Printf("Falling back to policy version of %d", config.Options.PolicyVersion)
	}

	if err := config.CheckValidity(); err != nil {
		return nil, &ErrBadConfigFile{ConfigFileLocation, err}
	}

	return config, nil
}

// getHashingCosts returns hashing costs so that hashing a password will take
// approximately the target time. This is done using the total amount of RAM,
// the number of CPUs present, and by running the passphrase hash many times.
func getHashingCosts(target time.Duration) (*metadata.HashingCosts, error) {
	log.Printf("Finding hashing costs that take %v\n", target)

	// Start out with the minimal possible costs that use all the CPUs.
	nCPUs := int64(runtime.NumCPU())
	costs := &metadata.HashingCosts{
		Time:        1,
		Memory:      8 * nCPUs,
		Parallelism: nCPUs,
	}

	// If even the minimal costs are not fast enough, just return the
	// minimal costs and log a warning.
	t, err := timeHashingCosts(costs)
	if err != nil {
		return nil, err
	}
	log.Printf("Min Costs={%v}\t-> %v\n", costs, t)

	if t > target {
		log.Printf("time exceeded the target of %v.\n", target)
		return costs, nil
	}

	// Now we start doubling the costs until we reach the target.
	memoryKiBLimit := memoryBytesLimit() / 1024
	for {
		// Store a copy of the previous costs
		costsPrev := *costs
		tPrev := t

		// Double the memory up to the max, then double the time.
		if costs.Memory < memoryKiBLimit {
			costs.Memory = util.MinInt64(2*costs.Memory, memoryKiBLimit)
		} else {
			costs.Time *= 2
		}

		// If our hashing failed, return the last good set of costs.
		if t, err = timeHashingCosts(costs); err != nil {
			log.Printf("Hashing with costs={%v} failed: %v\n", costs, err)
			return &costsPrev, nil
		}
		log.Printf("Costs={%v}\t-> %v\n", costs, t)

		// If we have reached the target time, we return a set of costs
		// based on the linear interpolation between the last two times.
		if t >= target {
			f := float64(target-tPrev) / float64(t-tPrev)
			return &metadata.HashingCosts{
				Time:        betweenCosts(costsPrev.Time, costs.Time, f),
				Memory:      betweenCosts(costsPrev.Memory, costs.Memory, f),
				Parallelism: costs.Parallelism,
			}, nil
		}
	}
}

// memoryBytesLimit returns the maximum amount of memory we will use for
// passphrase hashing. This will never be more than a reasonable maximum (for
// compatibility) or an 8th the available system RAM.
func memoryBytesLimit() int64 {
	// The sysinfo syscall only fails if given a bad address
	var info unix.Sysinfo_t
	err := unix.Sysinfo(&info)
	util.NeverError(err)

	totalRAMBytes := int64(info.Totalram)
	return util.MinInt64(totalRAMBytes/8, maxMemoryBytes)
}

// betweenCosts returns a cost between a and b. Specifically, it returns the
// floor of a + f*(b-a). This way, f=0 returns a and f=1 returns b.
func betweenCosts(a, b int64, f float64) int64 {
	return a + int64(f*float64(b-a))
}

// timeHashingCosts runs the passphrase hash with the specified costs and
// returns the time it takes to hash the passphrase.
func timeHashingCosts(costs *metadata.HashingCosts) (time.Duration, error) {
	passphrase, err := crypto.NewKeyFromReader(bytes.NewReader(timingPassphrase))
	if err != nil {
		return 0, err
	}
	defer passphrase.Wipe()

	// Be sure to measure CPU time, not wall time (time.Now)
	begin := cpuTimeInNanoseconds()
	hash, err := crypto.PassphraseHash(passphrase, timingSalt, costs)
	if err == nil {
		hash.Wipe()
	}
	end := cpuTimeInNanoseconds()

	return time.Duration((end - begin) / costs.Parallelism), nil
}

// cpuTimeInNanoseconds returns the nanosecond count based on the process's CPU usage.
// This number has no absolute meaning, only relative meaning to other calls.
func cpuTimeInNanoseconds() int64 {
	var ts unix.Timespec
	err := unix.ClockGettime(unix.CLOCK_PROCESS_CPUTIME_ID, &ts)
	// ClockGettime fails if given a bad address or on a VERY old system.
	util.NeverError(err)
	return unix.TimespecToNsec(ts)
}
<|endoftext|>


<commit_before>package schema

import (
	"reflect"
	"testing"

	"github.com/hashicorp/terraform/terraform"
)

func TestDiffFieldReader_impl(t *testing.T) {
	var _ FieldReader = new(DiffFieldReader)
}

func TestDiffFieldReader_ASDF(t *testing.T) {
	schema := map[string]*Schema{
		"list_of_sets_1": &Schema{
			Type: TypeList,
			Elem: &Resource{
				Schema: map[string]*Schema{
					"nested_set": &Schema{
						Type: TypeSet,
						Elem: &Resource{
							Schema: map[string]*Schema{
								"val": &Schema{
									Type: TypeInt,
								},
							},
						},
						Set: func(a interface{}) int {
							m := a.(map[string]interface{})
							return m["val"].(int)
						},
					},
				},
			},
		},
		"list_of_sets_2": &Schema{
			Type: TypeList,
			Elem: &Resource{
				Schema: map[string]*Schema{
					"nested_set": &Schema{
						Type: TypeSet,
						Elem: &Resource{
							Schema: map[string]*Schema{
								"val": &Schema{
									Type: TypeInt,
								},
							},
						},
						Set: func(a interface{}) int {
							m := a.(map[string]interface{})
							return m["val"].(int)
						},
					},
				},
			},
		},
	}

	r := &DiffFieldReader{
		Schema: schema,
		Diff: &terraform.InstanceDiff{
			Attributes: map[string]*terraform.ResourceAttrDiff{
				"list_of_sets_1.0.nested_set.1.val": &terraform.ResourceAttrDiff{
					Old:        "1",
					New:        "0",
					NewRemoved: true,
				},
				"list_of_sets_1.0.nested_set.2.val": &terraform.ResourceAttrDiff{
					New: "2",
				},
			},
		},
	}

	r.Source = &MultiLevelFieldReader{
		Readers: map[string]FieldReader{
			"diff": r,
			"set":  &MapFieldReader{Schema: schema},
			"state": &MapFieldReader{
				Map: &BasicMapReader{
					"list_of_sets_1.#":                  "1",
					"list_of_sets_1.0.nested_set.#":     "1",
					"list_of_sets_1.0.nested_set.1.val": "1",
					"list_of_sets_2.#":                  "1",
					"list_of_sets_2.0.nested_set.#":     "1",
					"list_of_sets_2.0.nested_set.1.val": "1",
				},
				Schema: schema,
			},
		},
		Levels: []string{"state", "config"},
	}

	out, err := r.ReadField([]string{"list_of_sets_2"})
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	list, ok := out.Value.([]interface{})
	if !ok {
		t.Fatalf("nestedSetUpdate: bad: Value type\n\nexpected: []interface{}\n\ngot: %v", reflect.TypeOf(out.Value))
	}
	if len(list) != 1 {
		t.Fatalf("nestedSetUpdate: bad: list length\n\nexpected: 1\n\ngot: %v\n\n", len(list))
	}

	var m map[string]interface{}
	m, ok = list[0].(map[string]interface{})
	if !ok {
		t.Fatalf("nestedSetUpdate: bad: list item type\n\nexpected: map[string]interface{}\n\ngot: %v\n\n", reflect.TypeOf(list[0]))
	}

	rawSet := m["nested_set"]
	if rawSet == nil {
		t.Fatalf("nestedSetUpdate: bad: nested_set\n\nexpected: not nil\n\ngot: nil\n\n")
	}

	var set *Set
	set, ok = rawSet.(*Set)
	if !ok {
		t.Fatalf("nestedSetUpdate: bad: nested set type\n\nexpected: *Set\n\ngot: %v\n\n", reflect.TypeOf(m["nested_set"]))
	}
	if set.Len() != 1 {
		t.Fatalf("nestedSetUpdate: bad: nested set length\n\nexpected: 1\n\ngot: %v\n\n", set.Len())
	}

	setList := set.List()
	var item map[string]interface{}
	item, ok = setList[0].(map[string]interface{})
	if !ok {
		t.Fatalf("nestedSetUpdate: bad: nested set item type\n\nexpected: map[string]interface{}\n\ngot: %v\n\n", reflect.TypeOf(setList[0]))
	}

	val := item["val"].(int)
	if val != 1 {
		t.Fatalf("nestedSetUpdate: bad: val\n\nexpected: 1\n\ngot: %v\n\n", val)
	}
}

// https://github.com/hashicorp/terraform/issues/914
func TestDiffFieldReader_MapHandling(t *testing.T) {
	schema := map[string]*Schema{
		"tags": &Schema{
			Type: TypeMap,
		},
	}
	r := &DiffFieldReader{
		Schema: schema,
		Diff: &terraform.InstanceDiff{
			Attributes: map[string]*terraform.ResourceAttrDiff{
				"tags.%": &terraform.ResourceAttrDiff{
					Old: "1",
					New: "2",
				},
				"tags.baz": &terraform.ResourceAttrDiff{
					Old: "",
					New: "qux",
				},
			},
		},
		Source: &MapFieldReader{
			Schema: schema,
			Map: BasicMapReader(map[string]string{
				"tags.%":   "1",
				"tags.foo": "bar",
			}),
		},
	}

	result, err := r.ReadField([]string{"tags"})
	if err != nil {
		t.Fatalf("ReadField failed: %#v", err)
	}

	expected := map[string]interface{}{
		"foo": "bar",
		"baz": "qux",
	}

	if !reflect.DeepEqual(expected, result.Value) {
		t.Fatalf("bad: DiffHandling\n\nexpected: %#v\n\ngot: %#v\n\n", expected, result.Value)
	}
}

func TestDiffFieldReader_extra(t *testing.T) {
	schema := map[string]*Schema{
		"stringComputed": &Schema{Type: TypeString},

		"listMap": &Schema{
			Type: TypeList,
			Elem: &Schema{
				Type: TypeMap,
			},
		},

		"mapRemove": &Schema{Type: TypeMap},

		"setChange": &Schema{
			Type:     TypeSet,
			Optional: true,
			Elem: &Resource{
				Schema: map[string]*Schema{
					"index": &Schema{
						Type:     TypeInt,
						Required: true,
					},

					"value": &Schema{
						Type:     TypeString,
						Required: true,
					},
				},
			},
			Set: func(a interface{}) int {
				m := a.(map[string]interface{})
				return m["index"].(int)
			},
		},

		"setEmpty": &Schema{
			Type:     TypeSet,
			Optional: true,
			Elem: &Resource{
				Schema: map[string]*Schema{
					"index": &Schema{
						Type:     TypeInt,
						Required: true,
					},

					"value": &Schema{
						Type:     TypeString,
						Required: true,
					},
				},
			},
			Set: func(a interface{}) int {
				m := a.(map[string]interface{})
				return m["index"].(int)
			},
		},
	}

	r := &DiffFieldReader{
		Schema: schema,
		Diff: &terraform.InstanceDiff{
			Attributes: map[string]*terraform.ResourceAttrDiff{
				"stringComputed": &terraform.ResourceAttrDiff{
					Old:         "foo",
					New:         "bar",
					NewComputed: true,
				},

				"listMap.0.bar": &terraform.ResourceAttrDiff{
					NewRemoved: true,
				},

				"mapRemove.bar": &terraform.ResourceAttrDiff{
					NewRemoved: true,
				},

				"setChange.10.value": &terraform.ResourceAttrDiff{
					Old: "50",
					New: "80",
				},

				"setEmpty.#": &terraform.ResourceAttrDiff{
					Old: "2",
					New: "0",
				},
			},
		},

		Source: &MapFieldReader{
			Schema: schema,
			Map: BasicMapReader(map[string]string{
				"listMap.#":     "2",
				"listMap.0.foo": "bar",
				"listMap.0.bar": "baz",
				"listMap.1.baz": "baz",

				"mapRemove.foo": "bar",
				"mapRemove.bar": "bar",

				"setChange.#":        "1",
				"setChange.10.index": "10",
				"setChange.10.value": "50",

				"setEmpty.#":        "2",
				"setEmpty.10.index": "10",
				"setEmpty.10.value": "50",
				"setEmpty.20.index": "20",
				"setEmpty.20.value": "50",
			}),
		},
	}

	cases := map[string]struct {
		Addr   []string
		Result FieldReadResult
		Err    bool
	}{
		"stringComputed": {
			[]string{"stringComputed"},
			FieldReadResult{
				Value:    "",
				Exists:   true,
				Computed: true,
			},
			false,
		},

		"listMapRemoval": {
			[]string{"listMap"},
			FieldReadResult{
				Value: []interface{}{
					map[string]interface{}{
						"foo": "bar",
					},
					map[string]interface{}{
						"baz": "baz",
					},
				},
				Exists: true,
			},
			false,
		},

		"mapRemove": {
			[]string{"mapRemove"},
			FieldReadResult{
				Value: map[string]interface{}{
					"foo": "bar",
				},
				Exists:   true,
				Computed: false,
			},
			false,
		},

		"setChange": {
			[]string{"setChange"},
			FieldReadResult{
				Value: []interface{}{
					map[string]interface{}{
						"index": 10,
						"value": "80",
					},
				},
				Exists: true,
			},
			false,
		},

		"setEmpty": {
			[]string{"setEmpty"},
			FieldReadResult{
				Value:  []interface{}{},
				Exists: true,
			},
			false,
		},
	}

	for name, tc := range cases {
		out, err := r.ReadField(tc.Addr)
		if err != nil != tc.Err {
			t.Fatalf("%s: err: %s", name, err)
		}
		if s, ok := out.Value.(*Set); ok {
			// If it is a set, convert to a list so its more easily checked.
			out.Value = s.List()
		}
		if !reflect.DeepEqual(tc.Result, out) {
			t.Fatalf("%s: bad: %#v", name, out)
		}
	}
}

func TestDiffFieldReader(t *testing.T) {
	testFieldReader(t, func(s map[string]*Schema) FieldReader {
		return &DiffFieldReader{
			Schema: s,
			Diff: &terraform.InstanceDiff{
				Attributes: map[string]*terraform.ResourceAttrDiff{
					"bool": &terraform.ResourceAttrDiff{
						Old: "",
						New: "true",
					},

					"int": &terraform.ResourceAttrDiff{
						Old: "",
						New: "42",
					},

					"float": &terraform.ResourceAttrDiff{
						Old: "",
						New: "3.1415",
					},

					"string": &terraform.ResourceAttrDiff{
						Old: "",
						New: "string",
					},

					"stringComputed": &terraform.ResourceAttrDiff{
						Old:         "foo",
						New:         "bar",
						NewComputed: true,
					},

					"list.#": &terraform.ResourceAttrDiff{
						Old: "0",
						New: "2",
					},

					"list.0": &terraform.ResourceAttrDiff{
						Old: "",
						New: "foo",
					},

					"list.1": &terraform.ResourceAttrDiff{
						Old: "",
						New: "bar",
					},

					"listInt.#": &terraform.ResourceAttrDiff{
						Old: "0",
						New: "2",
					},

					"listInt.0": &terraform.ResourceAttrDiff{
						Old: "",
						New: "21",
					},

					"listInt.1": &terraform.ResourceAttrDiff{
						Old: "",
						New: "42",
					},

					"map.foo": &terraform.ResourceAttrDiff{
						Old: "",
						New: "bar",
					},

					"map.bar": &terraform.ResourceAttrDiff{
						Old: "",
						New: "baz",
					},

					"set.#": &terraform.ResourceAttrDiff{
						Old: "0",
						New: "2",
					},

					"set.10": &terraform.ResourceAttrDiff{
						Old: "",
						New: "10",
					},

					"set.50": &terraform.ResourceAttrDiff{
						Old: "",
						New: "50",
					},

					"setDeep.#": &terraform.ResourceAttrDiff{
						Old: "0",
						New: "2",
					},

					"setDeep.10.index": &terraform.ResourceAttrDiff{
						Old: "",
						New: "10",
					},

					"setDeep.10.value": &terraform.ResourceAttrDiff{
						Old: "",
						New: "foo",
					},

					"setDeep.50.index": &terraform.ResourceAttrDiff{
						Old: "",
						New: "50",
					},

					"setDeep.50.value": &terraform.ResourceAttrDiff{
						Old: "",
						New: "bar",
					},
				},
			},

			Source: &MapFieldReader{
				Schema: s,
				Map: BasicMapReader(map[string]string{
					"listMap.#":     "2",
					"listMap.0.foo": "bar",
					"listMap.0.bar": "baz",
					"listMap.1.baz": "baz",
				}),
			},
		}
	})
}
<commit_msg>Name test appropriately.<commit_after>package schema

import (
	"reflect"
	"testing"

	"github.com/hashicorp/terraform/terraform"
)

func TestDiffFieldReader_impl(t *testing.T) {
	var _ FieldReader = new(DiffFieldReader)
}

func TestDiffFieldReader_NestedSetUpdate(t *testing.T) {
	schema := map[string]*Schema{
		"list_of_sets_1": &Schema{
			Type: TypeList,
			Elem: &Resource{
				Schema: map[string]*Schema{
					"nested_set": &Schema{
						Type: TypeSet,
						Elem: &Resource{
							Schema: map[string]*Schema{
								"val": &Schema{
									Type: TypeInt,
								},
							},
						},
						Set: func(a interface{}) int {
							m := a.(map[string]interface{})
							return m["val"].(int)
						},
					},
				},
			},
		},
		"list_of_sets_2": &Schema{
			Type: TypeList,
			Elem: &Resource{
				Schema: map[string]*Schema{
					"nested_set": &Schema{
						Type: TypeSet,
						Elem: &Resource{
							Schema: map[string]*Schema{
								"val": &Schema{
									Type: TypeInt,
								},
							},
						},
						Set: func(a interface{}) int {
							m := a.(map[string]interface{})
							return m["val"].(int)
						},
					},
				},
			},
		},
	}

	r := &DiffFieldReader{
		Schema: schema,
		Diff: &terraform.InstanceDiff{
			Attributes: map[string]*terraform.ResourceAttrDiff{
				"list_of_sets_1.0.nested_set.1.val": &terraform.ResourceAttrDiff{
					Old:        "1",
					New:        "0",
					NewRemoved: true,
				},
				"list_of_sets_1.0.nested_set.2.val": &terraform.ResourceAttrDiff{
					New: "2",
				},
			},
		},
	}

	r.Source = &MultiLevelFieldReader{
		Readers: map[string]FieldReader{
			"diff": r,
			"set":  &MapFieldReader{Schema: schema},
			"state": &MapFieldReader{
				Map: &BasicMapReader{
					"list_of_sets_1.#":                  "1",
					"list_of_sets_1.0.nested_set.#":     "1",
					"list_of_sets_1.0.nested_set.1.val": "1",
					"list_of_sets_2.#":                  "1",
					"list_of_sets_2.0.nested_set.#":     "1",
					"list_of_sets_2.0.nested_set.1.val": "1",
				},
				Schema: schema,
			},
		},
		Levels: []string{"state", "config"},
	}

	out, err := r.ReadField([]string{"list_of_sets_2"})
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	list, ok := out.Value.([]interface{})
	if !ok {
		t.Fatalf("nestedSetUpdate: bad: Value type\n\nexpected: []interface{}\n\ngot: %v", reflect.TypeOf(out.Value))
	}
	if len(list) != 1 {
		t.Fatalf("nestedSetUpdate: bad: list length\n\nexpected: 1\n\ngot: %v\n\n", len(list))
	}

	var m map[string]interface{}
	m, ok = list[0].(map[string]interface{})
	if !ok {
		t.Fatalf("nestedSetUpdate: bad: list item type\n\nexpected: map[string]interface{}\n\ngot: %v\n\n", reflect.TypeOf(list[0]))
	}

	rawSet := m["nested_set"]
	if rawSet == nil {
		t.Fatalf("nestedSetUpdate: bad: nested_set\n\nexpected: not nil\n\ngot: nil\n\n")
	}

	var set *Set
	set, ok = rawSet.(*Set)
	if !ok {
		t.Fatalf("nestedSetUpdate: bad: nested set type\n\nexpected: *Set\n\ngot: %v\n\n", reflect.TypeOf(m["nested_set"]))
	}
	if set.Len() != 1 {
		t.Fatalf("nestedSetUpdate: bad: nested set length\n\nexpected: 1\n\ngot: %v\n\n", set.Len())
	}

	setList := set.List()
	var item map[string]interface{}
	item, ok = setList[0].(map[string]interface{})
	if !ok {
		t.Fatalf("nestedSetUpdate: bad: nested set item type\n\nexpected: map[string]interface{}\n\ngot: %v\n\n", reflect.TypeOf(setList[0]))
	}

	val := item["val"].(int)
	if val != 1 {
		t.Fatalf("nestedSetUpdate: bad: val\n\nexpected: 1\n\ngot: %v\n\n", val)
	}
}

// https://github.com/hashicorp/terraform/issues/914
func TestDiffFieldReader_MapHandling(t *testing.T) {
	schema := map[string]*Schema{
		"tags": &Schema{
			Type: TypeMap,
		},
	}
	r := &DiffFieldReader{
		Schema: schema,
		Diff: &terraform.InstanceDiff{
			Attributes: map[string]*terraform.ResourceAttrDiff{
				"tags.%": &terraform.ResourceAttrDiff{
					Old: "1",
					New: "2",
				},
				"tags.baz": &terraform.ResourceAttrDiff{
					Old: "",
					New: "qux",
				},
			},
		},
		Source: &MapFieldReader{
			Schema: schema,
			Map: BasicMapReader(map[string]string{
				"tags.%":   "1",
				"tags.foo": "bar",
			}),
		},
	}

	result, err := r.ReadField([]string{"tags"})
	if err != nil {
		t.Fatalf("ReadField failed: %#v", err)
	}

	expected := map[string]interface{}{
		"foo": "bar",
		"baz": "qux",
	}

	if !reflect.DeepEqual(expected, result.Value) {
		t.Fatalf("bad: DiffHandling\n\nexpected: %#v\n\ngot: %#v\n\n", expected, result.Value)
	}
}

func TestDiffFieldReader_extra(t *testing.T) {
	schema := map[string]*Schema{
		"stringComputed": &Schema{Type: TypeString},

		"listMap": &Schema{
			Type: TypeList,
			Elem: &Schema{
				Type: TypeMap,
			},
		},

		"mapRemove": &Schema{Type: TypeMap},

		"setChange": &Schema{
			Type:     TypeSet,
			Optional: true,
			Elem: &Resource{
				Schema: map[string]*Schema{
					"index": &Schema{
						Type:     TypeInt,
						Required: true,
					},

					"value": &Schema{
						Type:     TypeString,
						Required: true,
					},
				},
			},
			Set: func(a interface{}) int {
				m := a.(map[string]interface{})
				return m["index"].(int)
			},
		},

		"setEmpty": &Schema{
			Type:     TypeSet,
			Optional: true,
			Elem: &Resource{
				Schema: map[string]*Schema{
					"index": &Schema{
						Type:     TypeInt,
						Required: true,
					},

					"value": &Schema{
						Type:     TypeString,
						Required: true,
					},
				},
			},
			Set: func(a interface{}) int {
				m := a.(map[string]interface{})
				return m["index"].(int)
			},
		},
	}

	r := &DiffFieldReader{
		Schema: schema,
		Diff: &terraform.InstanceDiff{
			Attributes: map[string]*terraform.ResourceAttrDiff{
				"stringComputed": &terraform.ResourceAttrDiff{
					Old:         "foo",
					New:         "bar",
					NewComputed: true,
				},

				"listMap.0.bar": &terraform.ResourceAttrDiff{
					NewRemoved: true,
				},

				"mapRemove.bar": &terraform.ResourceAttrDiff{
					NewRemoved: true,
				},

				"setChange.10.value": &terraform.ResourceAttrDiff{
					Old: "50",
					New: "80",
				},

				"setEmpty.#": &terraform.ResourceAttrDiff{
					Old: "2",
					New: "0",
				},
			},
		},

		Source: &MapFieldReader{
			Schema: schema,
			Map: BasicMapReader(map[string]string{
				"listMap.#":     "2",
				"listMap.0.foo": "bar",
				"listMap.0.bar": "baz",
				"listMap.1.baz": "baz",

				"mapRemove.foo": "bar",
				"mapRemove.bar": "bar",

				"setChange.#":        "1",
				"setChange.10.index": "10",
				"setChange.10.value": "50",

				"setEmpty.#":        "2",
				"setEmpty.10.index": "10",
				"setEmpty.10.value": "50",
				"setEmpty.20.index": "20",
				"setEmpty.20.value": "50",
			}),
		},
	}

	cases := map[string]struct {
		Addr   []string
		Result FieldReadResult
		Err    bool
	}{
		"stringComputed": {
			[]string{"stringComputed"},
			FieldReadResult{
				Value:    "",
				Exists:   true,
				Computed: true,
			},
			false,
		},

		"listMapRemoval": {
			[]string{"listMap"},
			FieldReadResult{
				Value: []interface{}{
					map[string]interface{}{
						"foo": "bar",
					},
					map[string]interface{}{
						"baz": "baz",
					},
				},
				Exists: true,
			},
			false,
		},

		"mapRemove": {
			[]string{"mapRemove"},
			FieldReadResult{
				Value: map[string]interface{}{
					"foo": "bar",
				},
				Exists:   true,
				Computed: false,
			},
			false,
		},

		"setChange": {
			[]string{"setChange"},
			FieldReadResult{
				Value: []interface{}{
					map[string]interface{}{
						"index": 10,
						"value": "80",
					},
				},
				Exists: true,
			},
			false,
		},

		"setEmpty": {
			[]string{"setEmpty"},
			FieldReadResult{
				Value:  []interface{}{},
				Exists: true,
			},
			false,
		},
	}

	for name, tc := range cases {
		out, err := r.ReadField(tc.Addr)
		if err != nil != tc.Err {
			t.Fatalf("%s: err: %s", name, err)
		}
		if s, ok := out.Value.(*Set); ok {
			// If it is a set, convert to a list so its more easily checked.
			out.Value = s.List()
		}
		if !reflect.DeepEqual(tc.Result, out) {
			t.Fatalf("%s: bad: %#v", name, out)
		}
	}
}

func TestDiffFieldReader(t *testing.T) {
	testFieldReader(t, func(s map[string]*Schema) FieldReader {
		return &DiffFieldReader{
			Schema: s,
			Diff: &terraform.InstanceDiff{
				Attributes: map[string]*terraform.ResourceAttrDiff{
					"bool": &terraform.ResourceAttrDiff{
						Old: "",
						New: "true",
					},

					"int": &terraform.ResourceAttrDiff{
						Old: "",
						New: "42",
					},

					"float": &terraform.ResourceAttrDiff{
						Old: "",
						New: "3.1415",
					},

					"string": &terraform.ResourceAttrDiff{
						Old: "",
						New: "string",
					},

					"stringComputed": &terraform.ResourceAttrDiff{
						Old:         "foo",
						New:         "bar",
						NewComputed: true,
					},

					"list.#": &terraform.ResourceAttrDiff{
						Old: "0",
						New: "2",
					},

					"list.0": &terraform.ResourceAttrDiff{
						Old: "",
						New: "foo",
					},

					"list.1": &terraform.ResourceAttrDiff{
						Old: "",
						New: "bar",
					},

					"listInt.#": &terraform.ResourceAttrDiff{
						Old: "0",
						New: "2",
					},

					"listInt.0": &terraform.ResourceAttrDiff{
						Old: "",
						New: "21",
					},

					"listInt.1": &terraform.ResourceAttrDiff{
						Old: "",
						New: "42",
					},

					"map.foo": &terraform.ResourceAttrDiff{
						Old: "",
						New: "bar",
					},

					"map.bar": &terraform.ResourceAttrDiff{
						Old: "",
						New: "baz",
					},

					"set.#": &terraform.ResourceAttrDiff{
						Old: "0",
						New: "2",
					},

					"set.10": &terraform.ResourceAttrDiff{
						Old: "",
						New: "10",
					},

					"set.50": &terraform.ResourceAttrDiff{
						Old: "",
						New: "50",
					},

					"setDeep.#": &terraform.ResourceAttrDiff{
						Old: "0",
						New: "2",
					},

					"setDeep.10.index": &terraform.ResourceAttrDiff{
						Old: "",
						New: "10",
					},

					"setDeep.10.value": &terraform.ResourceAttrDiff{
						Old: "",
						New: "foo",
					},

					"setDeep.50.index": &terraform.ResourceAttrDiff{
						Old: "",
						New: "50",
					},

					"setDeep.50.value": &terraform.ResourceAttrDiff{
						Old: "",
						New: "bar",
					},
				},
			},

			Source: &MapFieldReader{
				Schema: s,
				Map: BasicMapReader(map[string]string{
					"listMap.#":     "2",
					"listMap.0.foo": "bar",
					"listMap.0.bar": "baz",
					"listMap.1.baz": "baz",
				}),
			},
		}
	})
}
<|endoftext|>


<commit_before>package integrationtests

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"time"

	_ "github.com/lucas-clemente/quic-clients" // download clients
	"github.com/lucas-clemente/quic-go/protocol"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("Drop Proxy", func() {
	var proxy *UDPProxy
	clientPath := fmt.Sprintf(
		"%s/src/github.com/lucas-clemente/quic-clients/client-%s-debug",
		os.Getenv("GOPATH"),
		runtime.GOOS,
	)
	proxyPort := 10001

	AfterEach(func() {
		proxy.Stop()
		time.Sleep(time.Millisecond)
	})

	for i := range protocol.SupportedVersions {
		version := protocol.SupportedVersions[i]

		Context(fmt.Sprintf("with quic version %d", version), func() {
			It("gets a file when many outgoing packets are dropped", func() {
				dropper := func(p PacketNumber) bool {
					if p <= 5 { // don't interfere with the crypto handshake
						return false
					}
					return p%4 == 0
				}

				iPort, _ := strconv.Atoi(port)
				var err error
				proxy, err = NewUDPProxy(proxyPort, "localhost", iPort, nil, dropper)
				Expect(err).ToNot(HaveOccurred())

				command := exec.Command(
					clientPath,
					"--quic-version="+strconv.Itoa(int(version)),
					"--host=127.0.0.1",
					"--port="+strconv.Itoa(proxyPort),
					"https://quic.clemente.io/data",
				)
				session, err := Start(command, nil, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())
				defer session.Kill()
				Eventually(session, 4).Should(Exit(0))
				Expect(bytes.Contains(session.Out.Contents(), data)).To(BeTrue())
			})

			It("gets a file when many incoming packets are dropped", func() {
				dropper := func(p PacketNumber) bool {
					if p <= 5 { // don't interfere with the crypto handshake
						return false
					}
					return p%4 == 0
				}

				iPort, _ := strconv.Atoi(port)
				var err error
				proxy, err = NewUDPProxy(proxyPort, "localhost", iPort, dropper, nil)
				Expect(err).ToNot(HaveOccurred())

				command := exec.Command(
					clientPath,
					"--quic-version="+strconv.Itoa(int(version)),
					"--host=127.0.0.1",
					"--port="+strconv.Itoa(proxyPort),
					"https://quic.clemente.io/data",
				)
				session, err := Start(command, nil, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())
				defer session.Kill()
				Eventually(session, 4).Should(Exit(0))
				Expect(bytes.Contains(session.Out.Contents(), data)).To(BeTrue())
			})

		})
	}
})
<commit_msg>cleanup drop test<commit_after>package integrationtests

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"time"

	_ "github.com/lucas-clemente/quic-clients" // download clients
	"github.com/lucas-clemente/quic-go/protocol"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"
)

var proxy *UDPProxy

func runDropTest(incomingPacketDropper, outgoingPacketDropper dropCallback, version protocol.VersionNumber) {
	proxyPort := 12345

	clientPath := fmt.Sprintf(
		"%s/src/github.com/lucas-clemente/quic-clients/client-%s-debug",
		os.Getenv("GOPATH"),
		runtime.GOOS,
	)

	iPort, _ := strconv.Atoi(port)
	var err error
	proxy, err = NewUDPProxy(proxyPort, "localhost", iPort, incomingPacketDropper, outgoingPacketDropper)
	Expect(err).ToNot(HaveOccurred())

	command := exec.Command(
		clientPath,
		"--quic-version="+strconv.Itoa(int(version)),
		"--host=127.0.0.1",
		"--port="+strconv.Itoa(proxyPort),
		"https://quic.clemente.io/data",
	)

	session, err := Start(command, nil, GinkgoWriter)
	Expect(err).NotTo(HaveOccurred())
	defer session.Kill()
	Eventually(session, 4).Should(Exit(0))
	Expect(bytes.Contains(session.Out.Contents(), data)).To(BeTrue())
}

var _ = Describe("Drop Proxy", func() {
	AfterEach(func() {
		proxy.Stop()
		time.Sleep(time.Millisecond)
	})

	for i := range protocol.SupportedVersions {
		version := protocol.SupportedVersions[i]

		Context(fmt.Sprintf("with quic version %d", version), func() {
			Context("dropping every 4th packet after the crypto handshake", func() {
				dropper := func(p PacketNumber) bool {
					if p <= 5 { // don't interfere with the crypto handshake
						return false
					}
					return p%4 == 0
				}

				It("gets a file when many outgoing packets are dropped", func() {
					runDropTest(nil, dropper, version)
				})

				It("gets a file when many incoming packets are dropped", func() {
					runDropTest(dropper, nil, version)
				})
			})
		})
	}
})
<|endoftext|>


<commit_before>package geocodio_test

import (
	"os"
	"testing"

	"github.com/stevepartridge/geocodio"
)

const (
	DefaultAPIKey = "YOUR_API_KEY"

	// AddressTestOneFull       = "42370 Bob Hope Dr, Rancho Mirage, CA 92270"
	// AddressTestOneWithoutZip = "42370 Bob Hope Dr, Rancho Mirage, CA"
	// AddressTestOneNumber     = "42370"
	// AddressTestOneStreet     = "Bob Hope Dr"
	// AddressTestOneCity       = "Rancho Mirage"
	// AddressTestOneState      = "CA"
	// AddressTestOneLatitude   = 33.738645
	// AddressTestOneLongitude  = -116.407141

	AddressTestOneFull       = "1109 N Highland St, Arlington, VA 22201"
	AddressTestOneWithoutZip = "1109 N Highland St, Arlington, VA"
	AddressTestOneNumber     = "1109"
	AddressTestOneStreet     = "N Highland St"
	AddressTestOneCity       = "Arlington"
	AddressTestOneState      = "VA"
	AddressTestOneLatitude   = 38.886672
	AddressTestOneLongitude  = -77.094735

	AddressTestTwoFull       = "500 H St NE, Washington, DC 20002"
	AddressTestTwoWithoutZip = "500 H St NE, Washington, DC"
	AddressTestTwoNumber     = "500"
	AddressTestTwoStreet     = "H St NE"
	AddressTestTwoCity       = "Washington"
	AddressTestTwoState      = "DC"
	AddressTestTwoLatitude   = 35.9746000
	AddressTestTwoLongitude  = -77.9658000
)

func APIKey() string {
	if DefaultAPIKey != "YOUR_API_KEY" {
		return DefaultAPIKey
	}
	return os.Getenv("API_KEY")
}

func TestGeocodioWithApiKey(t *testing.T) {
	_, err := geocodio.NewGeocodio(APIKey())
	if err != nil {
		t.Error("Failed with API KEY set.", err)
	}
}

func TestGeocodioWithoutApiKey(t *testing.T) {
	_, err := geocodio.NewGeocodio("")
	if err == nil {
		t.Error("Did not through error when omitting API KEY")
	}
}

// 1109+N+Highland+St%2c+Arlington+VA
<commit_msg>Change test address two<commit_after>package geocodio_test

import (
	"os"
	"testing"

	"github.com/stevepartridge/geocodio"
)

const (
	DefaultAPIKey = "YOUR_API_KEY"

	// AddressTestOneFull       = "42370 Bob Hope Dr, Rancho Mirage, CA 92270"
	// AddressTestOneWithoutZip = "42370 Bob Hope Dr, Rancho Mirage, CA"
	// AddressTestOneNumber     = "42370"
	// AddressTestOneStreet     = "Bob Hope Dr"
	// AddressTestOneCity       = "Rancho Mirage"
	// AddressTestOneState      = "CA"
	// AddressTestOneLatitude   = 33.738645
	// AddressTestOneLongitude  = -116.407141

	AddressTestOneFull       = "1109 N Highland St, Arlington, VA 22201"
	AddressTestOneWithoutZip = "1109 N Highland St, Arlington, VA"
	AddressTestOneNumber     = "1109"
	AddressTestOneStreet     = "N Highland St"
	AddressTestOneCity       = "Arlington"
	AddressTestOneState      = "VA"
	AddressTestOneLatitude   = 38.886672
	AddressTestOneLongitude  = -77.094735

	AddressTestTwoFull       = "100 Legends Way, Boston, MA 02114"
	AddressTestTwoWithoutZip = "100 Legends Way, Boston, MA"
	AddressTestTwoNumber     = "100"
	AddressTestTwoStreet     = "Legends Way"
	AddressTestTwoCity       = "Boston"
	AddressTestTwoState      = "MA"
	AddressTestTwoLatitude   = 42.36629
	AddressTestTwoLongitude  = -71.0622
)

func APIKey() string {
	if DefaultAPIKey != "YOUR_API_KEY" {
		return DefaultAPIKey
	}
	return os.Getenv("API_KEY")
}

func TestGeocodioWithApiKey(t *testing.T) {
	_, err := geocodio.NewGeocodio(APIKey())
	if err != nil {
		t.Error("Failed with API KEY set.", err)
	}
}

func TestGeocodioWithoutApiKey(t *testing.T) {
	_, err := geocodio.NewGeocodio("")
	if err == nil {
		t.Error("Did not through error when omitting API KEY")
	}
}

// 1109+N+Highland+St%2c+Arlington+VA
<|endoftext|>