<commit_before>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package compose

import (
	"context"
	"fmt"
	"os"
	"strings"

	cgo "github.com/compose-spec/compose-go/cli"
	"github.com/compose-spec/compose-go/loader"
	"github.com/compose-spec/compose-go/types"
	"github.com/mattn/go-shellwords"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"github.com/docker/cli/cli"
	"github.com/docker/compose/v2/pkg/api"
	"github.com/docker/compose/v2/pkg/progress"
)

type runOptions struct {
	*composeOptions
	Service       string
	Command       []string
	environment   []string
	Detach        bool
	Remove        bool
	noTty         bool
	user          string
	workdir       string
	entrypoint    string
	entrypointCmd []string
	labels        []string
	volumes       []string
	publish       []string
	useAliases    bool
	servicePorts  bool
	name          string
	noDeps        bool
	ignoreOrphans bool
	quietPull     bool
}

func (opts runOptions) apply(project *types.Project) error {
	target, err := project.GetService(opts.Service)
	if err != nil {
		return err
	}
	if !opts.servicePorts {
		target.Ports = []types.ServicePortConfig{}
	}
	if len(opts.publish) > 0 {
		target.Ports = []types.ServicePortConfig{}
		for _, p := range opts.publish {
			config, err := types.ParsePortConfig(p)
			if err != nil {
				return err
			}
			target.Ports = append(target.Ports, config...)
		}
	}
	if len(opts.volumes) > 0 {
		for _, v := range opts.volumes {
			volume, err := loader.ParseVolume(v)
			if err != nil {
				return err
			}
			target.Volumes = append(target.Volumes, volume)
		}
	}

	if opts.noDeps {
		for _, s := range project.Services {
			if s.Name != opts.Service {
				project.DisabledServices = append(project.DisabledServices, s)
			}
		}
		project.Services = types.Services{target}
	}

	for i, s := range project.Services {
		if s.Name == opts.Service {
			project.Services[i] = target
			break
		}
	}
	return nil
}

func runCommand(p *projectOptions, backend api.Service) *cobra.Command {
	opts := runOptions{
		composeOptions: &composeOptions{
			projectOptions: p,
		},
	}
	cmd := &cobra.Command{
		Use:   "run [options] [-v VOLUME...] [-p PORT...] [-e KEY=VAL...] [-l KEY=VALUE...] SERVICE [COMMAND] [ARGS...]",
		Short: "Run a one-off command on a service.",
		Args:  cobra.MinimumNArgs(1),
		PreRunE: AdaptCmd(func(ctx context.Context, cmd *cobra.Command, args []string) error {
			opts.Service = args[0]
			if len(args) > 1 {
				opts.Command = args[1:]
			}
			if len(opts.publish) > 0 && opts.servicePorts {
				return fmt.Errorf("--service-ports and --publish are incompatible")
			}
			if cmd.Flags().Changed("entrypoint") {
				command, err := shellwords.Parse(opts.entrypoint)
				if err != nil {
					return err
				}
				opts.entrypointCmd = command
			}
			return nil
		}),
		RunE: Adapt(func(ctx context.Context, args []string) error {
			project, err := p.toProject([]string{opts.Service}, cgo.WithResolvedPaths(true))
			if err != nil {
				return err
			}
			ignore := project.Environment["COMPOSE_IGNORE_ORPHANS"]
			opts.ignoreOrphans = strings.ToLower(ignore) == "true"
			return runRun(ctx, backend, project, opts)
		}),
		ValidArgsFunction: serviceCompletion(p),
	}
	flags := cmd.Flags()
	flags.BoolVarP(&opts.Detach, "detach", "d", false, "Run container in background and print container ID")
	flags.StringArrayVarP(&opts.environment, "env", "e", []string{}, "Set environment variables")
	flags.StringArrayVarP(&opts.labels, "label", "l", []string{}, "Add or override a label")
	flags.BoolVar(&opts.Remove, "rm", false, "Automatically remove the container when it exits")
	flags.BoolVarP(&opts.noTty, "no-TTY", "T", false, "Disable pseudo-noTty allocation. By default docker compose run allocates a TTY")
	flags.StringVar(&opts.name, "name", "", " Assign a name to the container")
	flags.StringVarP(&opts.user, "user", "u", "", "Run as specified username or uid")
	flags.StringVarP(&opts.workdir, "workdir", "w", "", "Working directory inside the container")
	flags.StringVar(&opts.entrypoint, "entrypoint", "", "Override the entrypoint of the image")
	flags.BoolVar(&opts.noDeps, "no-deps", false, "Don't start linked services.")
	flags.StringArrayVarP(&opts.volumes, "volume", "v", []string{}, "Bind mount a volume.")
	flags.StringArrayVarP(&opts.publish, "publish", "p", []string{}, "Publish a container's port(s) to the host.")
	flags.BoolVar(&opts.useAliases, "use-aliases", false, "Use the service's network useAliases in the network(s) the container connects to.")
	flags.BoolVar(&opts.servicePorts, "service-ports", false, "Run command with the service's ports enabled and mapped to the host.")
	flags.BoolVar(&opts.quietPull, "quiet-pull", false, "Pull without printing progress information.")

	cmd.Flags().BoolP("interactive", "i", true, "Keep STDIN open even if not attached. DEPRECATED")
	cmd.Flags().MarkHidden("interactive") //nolint:errcheck
	cmd.Flags().BoolP("tty", "t", true, "Allocate a pseudo-TTY. DEPRECATED")
	cmd.Flags().MarkHidden("tty") //nolint:errcheck

	flags.SetNormalizeFunc(normalizeRunFlags)
	flags.SetInterspersed(false)
	return cmd
}

func normalizeRunFlags(f *pflag.FlagSet, name string) pflag.NormalizedName {
	switch name {
	case "volumes":
		name = "volume"
	case "labels":
		name = "label"
	}
	return pflag.NormalizedName(name)
}

func runRun(ctx context.Context, backend api.Service, project *types.Project, opts runOptions) error {
	err := opts.apply(project)
	if err != nil {
		return err
	}

	err = progress.Run(ctx, func(ctx context.Context) error {
		return startDependencies(ctx, backend, *project, opts.Service, opts.ignoreOrphans)
	})
	if err != nil {
		return err
	}

	labels := types.Labels{}
	for _, s := range opts.labels {
		parts := strings.SplitN(s, "=", 2)
		if len(parts) != 2 {
			return fmt.Errorf("label must be set as KEY=VALUE")
		}
		labels[parts[0]] = parts[1]
	}

	// start container and attach to container streams
	runOpts := api.RunOptions{
		Name:              opts.name,
		Service:           opts.Service,
		Command:           opts.Command,
		Detach:            opts.Detach,
		AutoRemove:        opts.Remove,
		Stdin:             os.Stdin,
		Stdout:            os.Stdout,
		Stderr:            os.Stderr,
		Tty:               !opts.noTty,
		WorkingDir:        opts.workdir,
		User:              opts.user,
		Environment:       opts.environment,
		Entrypoint:        opts.entrypointCmd,
		Labels:            labels,
		UseNetworkAliases: opts.useAliases,
		NoDeps:            opts.noDeps,
		Index:             0,
		QuietPull:         opts.quietPull,
	}
	exitCode, err := backend.RunOneOffContainer(ctx, project, runOpts)
	if exitCode != 0 {
		errMsg := ""
		if err != nil {
			errMsg = err.Error()
		}
		return cli.StatusError{StatusCode: exitCode, Status: errMsg}
	}
	return err
}

func startDependencies(ctx context.Context, backend api.Service, project types.Project, requestedServiceName string, ignoreOrphans bool) error {
	dependencies := types.Services{}
	var requestedService types.ServiceConfig
	for _, service := range project.Services {
		if service.Name != requestedServiceName {
			dependencies = append(dependencies, service)
		} else {
			requestedService = service
		}
	}

	project.Services = dependencies
	project.DisabledServices = append(project.DisabledServices, requestedService)
	err := backend.Create(ctx, &project, api.CreateOptions{
		IgnoreOrphans: ignoreOrphans,
	})
	if err != nil {
		return err
	}

	if len(dependencies) > 0 {
		return backend.Start(ctx, project.Name, api.StartOptions{})
	}
	return nil
}
<commit_msg>map --interactive to StdinOpen<commit_after>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package compose

import (
	"context"
	"fmt"
	"os"
	"strings"

	cgo "github.com/compose-spec/compose-go/cli"
	"github.com/compose-spec/compose-go/loader"
	"github.com/compose-spec/compose-go/types"
	"github.com/mattn/go-shellwords"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"github.com/docker/cli/cli"
	"github.com/docker/compose/v2/pkg/api"
	"github.com/docker/compose/v2/pkg/progress"
)

type runOptions struct {
	*composeOptions
	Service       string
	Command       []string
	environment   []string
	Detach        bool
	Remove        bool
	noTty         bool
	interactive   bool
	user          string
	workdir       string
	entrypoint    string
	entrypointCmd []string
	labels        []string
	volumes       []string
	publish       []string
	useAliases    bool
	servicePorts  bool
	name          string
	noDeps        bool
	ignoreOrphans bool
	quietPull     bool
}

func (opts runOptions) apply(project *types.Project) error {
	target, err := project.GetService(opts.Service)
	if err != nil {
		return err
	}
	if !opts.servicePorts {
		target.Ports = []types.ServicePortConfig{}
	}
	if len(opts.publish) > 0 {
		target.Ports = []types.ServicePortConfig{}
		for _, p := range opts.publish {
			config, err := types.ParsePortConfig(p)
			if err != nil {
				return err
			}
			target.Ports = append(target.Ports, config...)
		}
	}
	if len(opts.volumes) > 0 {
		for _, v := range opts.volumes {
			volume, err := loader.ParseVolume(v)
			if err != nil {
				return err
			}
			target.Volumes = append(target.Volumes, volume)
		}
	}

	if opts.noDeps {
		for _, s := range project.Services {
			if s.Name != opts.Service {
				project.DisabledServices = append(project.DisabledServices, s)
			}
		}
		project.Services = types.Services{target}
	}

	for i, s := range project.Services {
		if s.Name == opts.Service {
			project.Services[i] = target
			break
		}
	}
	return nil
}

func runCommand(p *projectOptions, backend api.Service) *cobra.Command {
	opts := runOptions{
		composeOptions: &composeOptions{
			projectOptions: p,
		},
	}
	cmd := &cobra.Command{
		Use:   "run [options] [-v VOLUME...] [-p PORT...] [-e KEY=VAL...] [-l KEY=VALUE...] SERVICE [COMMAND] [ARGS...]",
		Short: "Run a one-off command on a service.",
		Args:  cobra.MinimumNArgs(1),
		PreRunE: AdaptCmd(func(ctx context.Context, cmd *cobra.Command, args []string) error {
			opts.Service = args[0]
			if len(args) > 1 {
				opts.Command = args[1:]
			}
			if len(opts.publish) > 0 && opts.servicePorts {
				return fmt.Errorf("--service-ports and --publish are incompatible")
			}
			if cmd.Flags().Changed("entrypoint") {
				command, err := shellwords.Parse(opts.entrypoint)
				if err != nil {
					return err
				}
				opts.entrypointCmd = command
			}
			return nil
		}),
		RunE: Adapt(func(ctx context.Context, args []string) error {
			project, err := p.toProject([]string{opts.Service}, cgo.WithResolvedPaths(true))
			if err != nil {
				return err
			}
			ignore := project.Environment["COMPOSE_IGNORE_ORPHANS"]
			opts.ignoreOrphans = strings.ToLower(ignore) == "true"
			return runRun(ctx, backend, project, opts)
		}),
		ValidArgsFunction: serviceCompletion(p),
	}
	flags := cmd.Flags()
	flags.BoolVarP(&opts.Detach, "detach", "d", false, "Run container in background and print container ID")
	flags.StringArrayVarP(&opts.environment, "env", "e", []string{}, "Set environment variables")
	flags.StringArrayVarP(&opts.labels, "label", "l", []string{}, "Add or override a label")
	flags.BoolVar(&opts.Remove, "rm", false, "Automatically remove the container when it exits")
	flags.BoolVarP(&opts.noTty, "no-TTY", "T", false, "Disable pseudo-noTty allocation. By default docker compose run allocates a TTY")
	flags.StringVar(&opts.name, "name", "", " Assign a name to the container")
	flags.StringVarP(&opts.user, "user", "u", "", "Run as specified username or uid")
	flags.StringVarP(&opts.workdir, "workdir", "w", "", "Working directory inside the container")
	flags.StringVar(&opts.entrypoint, "entrypoint", "", "Override the entrypoint of the image")
	flags.BoolVar(&opts.noDeps, "no-deps", false, "Don't start linked services.")
	flags.StringArrayVarP(&opts.volumes, "volume", "v", []string{}, "Bind mount a volume.")
	flags.StringArrayVarP(&opts.publish, "publish", "p", []string{}, "Publish a container's port(s) to the host.")
	flags.BoolVar(&opts.useAliases, "use-aliases", false, "Use the service's network useAliases in the network(s) the container connects to.")
	flags.BoolVar(&opts.servicePorts, "service-ports", false, "Run command with the service's ports enabled and mapped to the host.")
	flags.BoolVar(&opts.quietPull, "quiet-pull", false, "Pull without printing progress information.")

	cmd.Flags().BoolVarP(&opts.interactive, "interactive", "i", true, "Keep STDIN open even if not attached.")
	cmd.Flags().BoolP("tty", "t", true, "Allocate a pseudo-TTY.")
	cmd.Flags().MarkHidden("tty") //nolint:errcheck

	flags.SetNormalizeFunc(normalizeRunFlags)
	flags.SetInterspersed(false)
	return cmd
}

func normalizeRunFlags(f *pflag.FlagSet, name string) pflag.NormalizedName {
	switch name {
	case "volumes":
		name = "volume"
	case "labels":
		name = "label"
	}
	return pflag.NormalizedName(name)
}

func runRun(ctx context.Context, backend api.Service, project *types.Project, opts runOptions) error {
	err := opts.apply(project)
	if err != nil {
		return err
	}

	err = progress.Run(ctx, func(ctx context.Context) error {
		return startDependencies(ctx, backend, *project, opts.Service, opts.ignoreOrphans)
	})
	if err != nil {
		return err
	}

	labels := types.Labels{}
	for _, s := range opts.labels {
		parts := strings.SplitN(s, "=", 2)
		if len(parts) != 2 {
			return fmt.Errorf("label must be set as KEY=VALUE")
		}
		labels[parts[0]] = parts[1]
	}

	// start container and attach to container streams
	runOpts := api.RunOptions{
		Name:              opts.name,
		Service:           opts.Service,
		Command:           opts.Command,
		Detach:            opts.Detach,
		AutoRemove:        opts.Remove,
		Stdin:             os.Stdin,
		Stdout:            os.Stdout,
		Stderr:            os.Stderr,
		Tty:               !opts.noTty,
		WorkingDir:        opts.workdir,
		User:              opts.user,
		Environment:       opts.environment,
		Entrypoint:        opts.entrypointCmd,
		Labels:            labels,
		UseNetworkAliases: opts.useAliases,
		NoDeps:            opts.noDeps,
		Index:             0,
		QuietPull:         opts.quietPull,
	}

	for i, service := range project.Services {
		if service.Name == opts.Service {
			service.StdinOpen = opts.interactive
			project.Services[i] = service
		}
	}

	exitCode, err := backend.RunOneOffContainer(ctx, project, runOpts)
	if exitCode != 0 {
		errMsg := ""
		if err != nil {
			errMsg = err.Error()
		}
		return cli.StatusError{StatusCode: exitCode, Status: errMsg}
	}
	return err
}

func startDependencies(ctx context.Context, backend api.Service, project types.Project, requestedServiceName string, ignoreOrphans bool) error {
	dependencies := types.Services{}
	var requestedService types.ServiceConfig
	for _, service := range project.Services {
		if service.Name != requestedServiceName {
			dependencies = append(dependencies, service)
		} else {
			requestedService = service
		}
	}

	project.Services = dependencies
	project.DisabledServices = append(project.DisabledServices, requestedService)
	err := backend.Create(ctx, &project, api.CreateOptions{
		IgnoreOrphans: ignoreOrphans,
	})
	if err != nil {
		return err
	}

	if len(dependencies) > 0 {
		return backend.Start(ctx, project.Name, api.StartOptions{})
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"

	"github.com/docker/go-units"
	"github.com/go-debos/debos"
	"github.com/go-debos/debos/recipe"
	"github.com/go-debos/fakemachine"
	"github.com/jessevdk/go-flags"
)

func checkError(context debos.DebosContext, err error, a debos.Action, stage string) int {
	if err == nil {
		return 0
	}

	log.Printf("Action `%s` failed at stage %s, error: %s", a, stage, err)
	debos.DebugShell(context)
	return 1
}

func main() {
	var context debos.DebosContext
	var options struct {
		ArtifactDir   string            `long:"artifactdir"`
		InternalImage string            `long:"internal-image" hidden:"true"`
		TemplateVars  map[string]string `short:"t" long:"template-var" description:"Template variables"`
		DebugShell    bool              `long:"debug-shell" description:"Fall into interactive shell on error"`
		Shell         string            `short:"s" long:"shell" description:"Redefine interactive shell binary (default: bash)" optionsl:"" default:"/bin/bash"`
		ScratchSize   string            `long:"scratchsize" description:"Size of disk backed scratch space"`
	}

	var exitcode int = 0
	// Allow to run all deferred calls prior to os.Exit()
	defer func() {
		os.Exit(exitcode)
	}()

	parser := flags.NewParser(&options, flags.Default)
	args, err := parser.Parse()

	if err != nil {
		flagsErr, ok := err.(*flags.Error)
		if ok && flagsErr.Type == flags.ErrHelp {
			return
		} else {
			fmt.Printf("%v\n", flagsErr)
			exitcode = 1
			return
		}
	}

	if len(args) != 1 {
		log.Println("No recipe given!")
		exitcode = 1
		return
	}

	// Set interactive shell binary only if '--debug-shell' options passed
	if options.DebugShell {
		context.DebugShell = options.Shell
	}

	file := args[0]
	file = debos.CleanPath(file)

	r := recipe.Recipe{}
	if _, err := os.Stat(file); os.IsNotExist(err) {
		log.Println(err)
		exitcode = 1
		return
	}
	if err := r.Parse(file, options.TemplateVars); err != nil {
		log.Println(err)
		exitcode = 1
		return
	}

	/* If fakemachine is supported the outer fake machine will never use the
	 * scratchdir, so just set it to /scratch as a dummy to prevent the
	 * outer debos creating a temporary direction */
	if fakemachine.InMachine() || fakemachine.Supported() {
		context.Scratchdir = "/scratch"
	} else {
		log.Printf("fakemachine not supported, running on the host!")
		cwd, _ := os.Getwd()
		context.Scratchdir, err = ioutil.TempDir(cwd, ".debos-")
		defer os.RemoveAll(context.Scratchdir)
	}

	context.Rootdir = path.Join(context.Scratchdir, "root")
	context.Image = options.InternalImage
	context.RecipeDir = path.Dir(file)

	context.Artifactdir = options.ArtifactDir
	if context.Artifactdir == "" {
		context.Artifactdir, _ = os.Getwd()
	}
	context.Artifactdir = debos.CleanPath(context.Artifactdir)

	// Initialise origins map
	context.Origins = make(map[string]string)
	context.Origins["artifacts"] = context.Artifactdir
	context.Origins["filesystem"] = context.Rootdir
	context.Origins["recipe"] = context.RecipeDir

	context.Architecture = r.Architecture

	for _, a := range r.Actions {
		err = a.Verify(&context)
		if exitcode = checkError(context, err, a, "Verify"); exitcode != 0 {
			return
		}
	}

	if !fakemachine.InMachine() && fakemachine.Supported() {
		m := fakemachine.NewMachine()
		var args []string

		if options.ScratchSize != "" {
			size, err := units.FromHumanSize(options.ScratchSize)
			if err != nil {
				fmt.Printf("Couldn't parse scratch size: %v\n", err)
				exitcode = 1
				return
			}
			m.SetScratch(size, "")
		}

		m.AddVolume(context.Artifactdir)
		args = append(args, "--artifactdir", context.Artifactdir)

		for k, v := range options.TemplateVars {
			args = append(args, "--template-var", fmt.Sprintf("%s:\"%s\"", k, v))
		}

		m.AddVolume(context.RecipeDir)
		args = append(args, file)

		if options.DebugShell {
			args = append(args, "--debug-shell")
			args = append(args, "--shell", fmt.Sprintf("%s", options.Shell))
		}

		for _, a := range r.Actions {
			err = a.PreMachine(&context, m, &args)
			if exitcode = checkError(context, err, a, "PreMachine"); exitcode != 0 {
				return
			}
		}

		exitcode, err = m.RunInMachineWithArgs(args)
		if err != nil {
			fmt.Println(err)
			return
		}

		if exitcode != 0 {
			return
		}

		for _, a := range r.Actions {
			err = a.PostMachine(context)
			if exitcode = checkError(context, err, a, "Postmachine"); exitcode != 0 {
				return
			}
		}

		log.Printf("==== Recipe done ====")
		return
	}

	if !fakemachine.InMachine() {
		for _, a := range r.Actions {
			err = a.PreNoMachine(&context)
			if exitcode = checkError(context, err, a, "PreNoMachine"); exitcode != 0 {
				return
			}
		}
	}

	// Create Rootdir
	if _, err = os.Stat(context.Rootdir); os.IsNotExist(err) {
		err = os.Mkdir(context.Rootdir, 0755)
		if err != nil && os.IsNotExist(err) {
			exitcode = 1
			return
		}
	}

	for _, a := range r.Actions {
		err = a.Run(&context)
		if exitcode = checkError(context, err, a, "Run"); exitcode != 0 {
			return
		}
	}

	for _, a := range r.Actions {
		err = a.Cleanup(context)
		if exitcode = checkError(context, err, a, "Cleanup"); exitcode != 0 {
			return
		}
	}

	if !fakemachine.InMachine() {
		for _, a := range r.Actions {
			err = a.PostMachine(context)
			if exitcode = checkError(context, err, a, "PostMachine"); exitcode != 0 {
				return
			}
		}
		log.Printf("==== Recipe done ====")
	}
}
<commit_msg>Add CPU and memory configuration for VM<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"

	"github.com/docker/go-units"
	"github.com/go-debos/debos"
	"github.com/go-debos/debos/recipe"
	"github.com/go-debos/fakemachine"
	"github.com/jessevdk/go-flags"
)

func checkError(context debos.DebosContext, err error, a debos.Action, stage string) int {
	if err == nil {
		return 0
	}

	log.Printf("Action `%s` failed at stage %s, error: %s", a, stage, err)
	debos.DebugShell(context)
	return 1
}

func main() {
	var context debos.DebosContext
	var options struct {
		ArtifactDir   string            `long:"artifactdir"`
		InternalImage string            `long:"internal-image" hidden:"true"`
		TemplateVars  map[string]string `short:"t" long:"template-var" description:"Template variables"`
		DebugShell    bool              `long:"debug-shell" description:"Fall into interactive shell on error"`
		Shell         string            `short:"s" long:"shell" description:"Redefine interactive shell binary (default: bash)" optionsl:"" default:"/bin/bash"`
		ScratchSize   string            `long:"scratchsize" description:"Size of disk backed scratch space"`
		CPUs          int               `short:"c" long:"cpus" description:"Number of CPUs to use for build VM (default: 2)"`
		Memory        string            `short:"m" long:"memory" description:"Amount of memory for build VM (default: 2048MB)"`
	}

	var exitcode int = 0
	// Allow to run all deferred calls prior to os.Exit()
	defer func() {
		os.Exit(exitcode)
	}()

	parser := flags.NewParser(&options, flags.Default)
	args, err := parser.Parse()

	if err != nil {
		flagsErr, ok := err.(*flags.Error)
		if ok && flagsErr.Type == flags.ErrHelp {
			return
		} else {
			fmt.Printf("%v\n", flagsErr)
			exitcode = 1
			return
		}
	}

	if len(args) != 1 {
		log.Println("No recipe given!")
		exitcode = 1
		return
	}

	// Set interactive shell binary only if '--debug-shell' options passed
	if options.DebugShell {
		context.DebugShell = options.Shell
	}

	file := args[0]
	file = debos.CleanPath(file)

	r := recipe.Recipe{}
	if _, err := os.Stat(file); os.IsNotExist(err) {
		log.Println(err)
		exitcode = 1
		return
	}
	if err := r.Parse(file, options.TemplateVars); err != nil {
		log.Println(err)
		exitcode = 1
		return
	}

	/* If fakemachine is supported the outer fake machine will never use the
	 * scratchdir, so just set it to /scratch as a dummy to prevent the
	 * outer debos creating a temporary direction */
	if fakemachine.InMachine() || fakemachine.Supported() {
		context.Scratchdir = "/scratch"
	} else {
		log.Printf("fakemachine not supported, running on the host!")
		cwd, _ := os.Getwd()
		context.Scratchdir, err = ioutil.TempDir(cwd, ".debos-")
		defer os.RemoveAll(context.Scratchdir)
	}

	context.Rootdir = path.Join(context.Scratchdir, "root")
	context.Image = options.InternalImage
	context.RecipeDir = path.Dir(file)

	context.Artifactdir = options.ArtifactDir
	if context.Artifactdir == "" {
		context.Artifactdir, _ = os.Getwd()
	}
	context.Artifactdir = debos.CleanPath(context.Artifactdir)

	// Initialise origins map
	context.Origins = make(map[string]string)
	context.Origins["artifacts"] = context.Artifactdir
	context.Origins["filesystem"] = context.Rootdir
	context.Origins["recipe"] = context.RecipeDir

	context.Architecture = r.Architecture

	for _, a := range r.Actions {
		err = a.Verify(&context)
		if exitcode = checkError(context, err, a, "Verify"); exitcode != 0 {
			return
		}
	}

	if !fakemachine.InMachine() && fakemachine.Supported() {
		m := fakemachine.NewMachine()
		var args []string

		if options.Memory == "" {
			// Set default memory size for fakemachine
			options.Memory = "2Gb"
		}
		memsize, err := units.RAMInBytes(options.Memory)
		if err != nil {
			fmt.Printf("Couldn't parse memory size: %v\n", err)
			exitcode = 1
			return
		}
		m.SetMemory(int(memsize / 1024 / 1024))

		if options.CPUs == 0 {
			// Set default CPU count for fakemachine
			options.CPUs = 2
		}
		m.SetNumCPUs(options.CPUs)

		if options.ScratchSize != "" {
			size, err := units.FromHumanSize(options.ScratchSize)
			if err != nil {
				fmt.Printf("Couldn't parse scratch size: %v\n", err)
				exitcode = 1
				return
			}
			m.SetScratch(size, "")
		}

		m.AddVolume(context.Artifactdir)
		args = append(args, "--artifactdir", context.Artifactdir)

		for k, v := range options.TemplateVars {
			args = append(args, "--template-var", fmt.Sprintf("%s:\"%s\"", k, v))
		}

		m.AddVolume(context.RecipeDir)
		args = append(args, file)

		if options.DebugShell {
			args = append(args, "--debug-shell")
			args = append(args, "--shell", fmt.Sprintf("%s", options.Shell))
		}

		for _, a := range r.Actions {
			err = a.PreMachine(&context, m, &args)
			if exitcode = checkError(context, err, a, "PreMachine"); exitcode != 0 {
				return
			}
		}

		exitcode, err = m.RunInMachineWithArgs(args)
		if err != nil {
			fmt.Println(err)
			return
		}

		if exitcode != 0 {
			return
		}

		for _, a := range r.Actions {
			err = a.PostMachine(context)
			if exitcode = checkError(context, err, a, "Postmachine"); exitcode != 0 {
				return
			}
		}

		log.Printf("==== Recipe done ====")
		return
	}

	if !fakemachine.InMachine() {
		for _, a := range r.Actions {
			err = a.PreNoMachine(&context)
			if exitcode = checkError(context, err, a, "PreNoMachine"); exitcode != 0 {
				return
			}
		}
	}

	// Create Rootdir
	if _, err = os.Stat(context.Rootdir); os.IsNotExist(err) {
		err = os.Mkdir(context.Rootdir, 0755)
		if err != nil && os.IsNotExist(err) {
			exitcode = 1
			return
		}
	}

	for _, a := range r.Actions {
		err = a.Run(&context)
		if exitcode = checkError(context, err, a, "Run"); exitcode != 0 {
			return
		}
	}

	for _, a := range r.Actions {
		err = a.Cleanup(context)
		if exitcode = checkError(context, err, a, "Cleanup"); exitcode != 0 {
			return
		}
	}

	if !fakemachine.InMachine() {
		for _, a := range r.Actions {
			err = a.PostMachine(context)
			if exitcode = checkError(context, err, a, "PostMachine"); exitcode != 0 {
				return
			}
		}
		log.Printf("==== Recipe done ====")
	}
}
<|endoftext|>


<commit_before><commit_msg>almost done<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
)

type (
	Config struct {
		// host to listen on
		Host string `json:"host"`
		// port to listen on
		Port int `json:"port"`
		// list of repos to check
		Repos []repo `json:"repos"`
	}

	repo struct {
		Type   string `json:"type"` //github or gitlab
		Name   string `json:"name"`
		Branch string `json:"branch"`
		Tasks  []task `json:"tasks"`
		Notify notify `json:"notify"`
	}

	// notify may contain alternative notifications
	notify struct {
		Telegram telegramBot `json:"telegram"`
	}

	// task is used to specify the commands to execute
	task struct {
		Args []string `json:"args"`
		Cmd  string   `json:"cmd"`
		Cwd  string   `json:"cwd"`
	}

	// telegramBot holds info to connect to telegram bot API
	telegramBot struct {
		ChatID int    `json:"chat_id"`
		Token  string `json:"token"`
	}
)

const (
	cfgName     = `config.json`
	defaultPort = 8080
	PS          = string(os.PathSeparator)
)

var (
	path    = flag.String(`path`, `.`, `where to look for config file`)
	config  Config
	mux     = http.DefaultServeMux
	sprintf = fmt.Sprintf //share sprintf across files
)

func main() {
	flag.Parse()
	loadConf(*path)

	// Add handlers
	mux.HandleFunc(`/gitlab`, gitHandler(`gitlab`))
	mux.HandleFunc(`/github`, gitHandler(`github`))

	address := sprintf("%s:%d", config.Host, config.Port)
	log.Printf("Listening on %s/gitlab and %[1]s/github", address)
	log.Fatal("ERROR SERVING: %+v", http.ListenAndServe(address, nil))
}

// Run method for task
func (t task) Run() {
	var (
		out []byte
		err error
	)

	cmd := exec.Command(t.Cmd, t.Args...)
	if cmd.Dir = t.Cwd; cmd.Dir == `` {
		cmd.Dir, _ = os.Getwd() //fallback to current working dir
	}

	if out, err = cmd.CombinedOutput(); err != nil {
		logf("ERROR: %s -- RUNNING: %s %s WITH ARGS: %+v", err, t.Cwd, t.Cmd, t.Args)
	}
	logf("OUTPUT: %v", out)
}

func logf(f string, args ...interface{}) {
	log.Printf("%s%s%[1]s", "\n-----\n", sprintf(f, args...))
}
<|endoftext|>


<commit_before>package cmd

import (
	"bytes"
	"fmt"
	"os"
	"unicode"

	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/rancher/go-rancher/v2"
	"github.com/urfave/cli"
)

func toEnv(name string) string {
	buf := bytes.Buffer{}
	for _, c := range name {
		if unicode.IsUpper(c) {
			buf.WriteRune('_')
			buf.WriteRune(unicode.ToLower(c))
		} else if c == '-' {
			buf.WriteRune('_')
		} else {
			buf.WriteRune(c)
		}
	}
	return strings.ToUpper(buf.String())
}

func toAPI(name string) string {
	buf := bytes.Buffer{}
	upper := false
	for _, c := range name {
		if c == '-' {
			upper = true
		} else if upper {
			upper = false
			buf.WriteRune(unicode.ToUpper(c))
		} else {
			buf.WriteRune(c)
		}
	}
	return buf.String()
}

func toArg(name string) string {
	buf := bytes.Buffer{}
	for _, c := range name {
		if unicode.IsUpper(c) {
			buf.WriteRune('-')
			buf.WriteRune(unicode.ToLower(c))
		} else {
			buf.WriteRune(c)
		}
	}
	return buf.String()
}

func buildFlag(name string, field client.Field) cli.Flag {
	var flag cli.Flag
	switch field.Type {
	case "bool":
		flag = cli.BoolFlag{
			Name:   toArg(name),
			EnvVar: toEnv(name),
			Usage:  field.Description,
		}
	case "array[string]":
		fallthrough
	case "map[string]":
		flag = cli.StringSliceFlag{
			Name:   toArg(name),
			EnvVar: toEnv(name),
			Usage:  field.Description,
		}
	default:
		sflag := cli.StringFlag{
			Name:   toArg(name),
			EnvVar: toEnv(name),
			Usage:  field.Description,
		}
		flag = sflag
		if field.Default != nil {
			sflag.Value = fmt.Sprint(field.Default)
		}
	}

	return flag
}

func buildFlags(prefix string, schema client.Schema, schemas *client.Schemas) []cli.Flag {
	flags := []cli.Flag{}
	for name, field := range schema.ResourceFields {
		if !field.Create || name == "name" {
			continue
		}

		if strings.HasSuffix(name, "Config") {
			subSchema := schemas.Schema(name)
			driver := strings.TrimSuffix(name, "Config")
			flags = append(flags, buildFlags(driver+"-", subSchema, schemas)...)
		} else {
			if prefix != "" {
				name = prefix + name
			}
			flags = append(flags, buildFlag(name, field))
		}
	}

	return flags
}

func hostCreate(ctx *cli.Context) error {
	c, err := GetClient(ctx)
	if err != nil {
		return err
	}

	hostSchema := c.GetSchemas().Schema("host")
	flags := buildFlags("", hostSchema, c.GetSchemas())
	drivers := []string{}

	for name := range hostSchema.ResourceFields {
		if strings.HasSuffix(name, "Config") {
			drivers = append(drivers, strings.TrimSuffix(name, "Config"))
		}
	}

	hostCommand := HostCommand()

	for i := range hostCommand.Subcommands {
		if hostCommand.Subcommands[i].Name == "create" {
			hostCommand.Subcommands[i].Flags = append(flags, cli.StringFlag{
				Name:   "driver,d",
				Usage:  "Driver to use: " + strings.Join(drivers, ", "),
				EnvVar: "MACHINE_DRIVER",
			})
			hostCommand.Subcommands[i].Action = func(ctx *cli.Context) error {
				return hostCreateRun(ctx, c, hostSchema, c.GetSchemas())
			}
			hostCommand.Subcommands[i].SkipFlagParsing = false
		}
	}

	app := cli.NewApp()
	app.Flags = []cli.Flag{
		//TODO: remove duplication here
		cli.BoolFlag{
			Name:  "debug",
			Usage: "Debug logging",
		},
		cli.StringFlag{
			Name:   "config,c",
			Usage:  "Client configuration file (default ${HOME}/.rancher/cli.json)",
			EnvVar: "RANCHER_CLIENT_CONFIG",
		},
		cli.StringFlag{
			Name:   "environment,env",
			Usage:  "Environment name or ID",
			EnvVar: "RANCHER_ENVIRONMENT",
		},
		cli.StringFlag{
			Name:   "url",
			Usage:  "Specify the Rancher API endpoint URL",
			EnvVar: "RANCHER_URL",
		},
		cli.StringFlag{
			Name:   "access-key",
			Usage:  "Specify Rancher API access key",
			EnvVar: "RANCHER_ACCESS_KEY",
		},
		cli.StringFlag{
			Name:   "secret-key",
			Usage:  "Specify Rancher API secret key",
			EnvVar: "RANCHER_SECRET_KEY",
		},
		cli.StringFlag{
			Name:   "host",
			Usage:  "Host used for docker command",
			EnvVar: "RANCHER_DOCKER_HOST",
		},
		cli.StringFlag{
			Name:  "rancher-file,r",
			Usage: "Specify an alternate Rancher compose file (default: rancher-compose.yml)",
		},
		cli.StringFlag{
			Name:  "env-file,e",
			Usage: "Specify a file from which to read environment variables",
		},
		cli.StringSliceFlag{
			Name:   "file,f",
			Usage:  "Specify one or more alternate compose files (default: docker-compose.yml)",
			Value:  &cli.StringSlice{},
			EnvVar: "COMPOSE_FILE",
		},
		cli.StringFlag{
			Name:  "stack,s",
			Usage: "Specify an alternate project name (default: directory name)",
		},
		cli.BoolFlag{
			Name:  "wait,w",
			Usage: "Wait for resource to reach resting state",
		},
		cli.IntFlag{
			Name:  "wait-timeout",
			Usage: "Timeout in seconds to wait",
			Value: 600,
		},
		cli.StringFlag{
			Name:  "wait-state",
			Usage: "State to wait for (active, healthy, etc)",
		},
	}
	app.Commands = []cli.Command{
		hostCommand,
	}
	return app.Run(os.Args)
}

func hostCreateRun(ctx *cli.Context, c *client.RancherClient, machineSchema client.Schema, schemas *client.Schemas) error {
	args := map[string]interface{}{}
	driverArgs := map[string]interface{}{}
	driver := ctx.String("driver")

	if driver == "" {
		return fmt.Errorf("--driver is required")
	}

	driverSchema, ok := schemas.CheckSchema(driver + "Config")
	if !ok {
		return fmt.Errorf("Invalid driver: %s", driver)
	}

	for _, name := range ctx.FlagNames() {
		schema := machineSchema
		destArgs := args
		key := name
		value := ctx.Generic(name)

		// really dumb way to detect empty values
		if str := fmt.Sprint(value); str == "" || str == "[]" {
			continue
		}

		if strings.HasPrefix(name, driver+"-") {
			key = toAPI(strings.TrimPrefix(name, driver+"-"))
			schema = driverSchema
			destArgs = driverArgs
		}

		fieldType := schema.ResourceFields[key].Type
		if fieldType == "map[string]" {
			mapValue := map[string]string{}
			for _, val := range ctx.StringSlice(name) {
				parts := strings.SplitN(val, "=", 2)
				if len(parts) == 1 {
					mapValue[parts[0]] = ""
				} else {
					mapValue[parts[0]] = parts[1]
				}
			}
			value = mapValue
		}

		destArgs[key] = value
	}

	args[driver+"Config"] = driverArgs

	names := ctx.Args()
	if len(names) == 0 {
		names = []string{RandomName()}
	}

	w, err := NewWaiter(ctx)
	if err != nil {
		return err
	}

	var lastErr error
	for _, name := range names {
		args["name"] = name
		var machine client.Machine
		if err := c.Create("machine", args, &machine); err != nil {
			lastErr = err
			logrus.Error(err)
		} else {
			w.Add(machine.Id)
		}
	}

	if lastErr != nil {
		return lastErr
	}

	return w.Wait()
}
<commit_msg>Switch to host API<commit_after>package cmd

import (
	"bytes"
	"fmt"
	"os"
	"unicode"

	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/rancher/go-rancher/v2"
	"github.com/urfave/cli"
)

func toEnv(name string) string {
	buf := bytes.Buffer{}
	for _, c := range name {
		if unicode.IsUpper(c) {
			buf.WriteRune('_')
			buf.WriteRune(unicode.ToLower(c))
		} else if c == '-' {
			buf.WriteRune('_')
		} else {
			buf.WriteRune(c)
		}
	}
	return strings.ToUpper(buf.String())
}

func toAPI(name string) string {
	buf := bytes.Buffer{}
	upper := false
	for _, c := range name {
		if c == '-' {
			upper = true
		} else if upper {
			upper = false
			buf.WriteRune(unicode.ToUpper(c))
		} else {
			buf.WriteRune(c)
		}
	}
	return buf.String()
}

func toArg(name string) string {
	buf := bytes.Buffer{}
	for _, c := range name {
		if unicode.IsUpper(c) {
			buf.WriteRune('-')
			buf.WriteRune(unicode.ToLower(c))
		} else {
			buf.WriteRune(c)
		}
	}
	return buf.String()
}

func buildFlag(name string, field client.Field) cli.Flag {
	var flag cli.Flag
	switch field.Type {
	case "bool":
		flag = cli.BoolFlag{
			Name:   toArg(name),
			EnvVar: toEnv(name),
			Usage:  field.Description,
		}
	case "array[string]":
		fallthrough
	case "map[string]":
		flag = cli.StringSliceFlag{
			Name:   toArg(name),
			EnvVar: toEnv(name),
			Usage:  field.Description,
		}
	default:
		sflag := cli.StringFlag{
			Name:   toArg(name),
			EnvVar: toEnv(name),
			Usage:  field.Description,
		}
		flag = sflag
		if field.Default != nil {
			sflag.Value = fmt.Sprint(field.Default)
		}
	}

	return flag
}

func buildFlags(prefix string, schema client.Schema, schemas *client.Schemas) []cli.Flag {
	flags := []cli.Flag{}
	for name, field := range schema.ResourceFields {
		if !field.Create || name == "hostname" {
			continue
		}

		if strings.HasSuffix(name, "Config") {
			subSchema := schemas.Schema(name)
			driver := strings.TrimSuffix(name, "Config")
			flags = append(flags, buildFlags(driver+"-", subSchema, schemas)...)
		} else {
			if prefix != "" {
				name = prefix + name
			}
			flags = append(flags, buildFlag(name, field))
		}
	}

	return flags
}

func hostCreate(ctx *cli.Context) error {
	c, err := GetClient(ctx)
	if err != nil {
		return err
	}

	hostSchema := c.GetSchemas().Schema("host")
	flags := buildFlags("", hostSchema, c.GetSchemas())
	drivers := []string{}

	for name := range hostSchema.ResourceFields {
		if strings.HasSuffix(name, "Config") {
			drivers = append(drivers, strings.TrimSuffix(name, "Config"))
		}
	}

	hostCommand := HostCommand()

	for i := range hostCommand.Subcommands {
		if hostCommand.Subcommands[i].Name == "create" {
			hostCommand.Subcommands[i].Flags = append(flags, cli.StringFlag{
				Name:   "driver,d",
				Usage:  "Driver to use: " + strings.Join(drivers, ", "),
				EnvVar: "MACHINE_DRIVER",
			})
			hostCommand.Subcommands[i].Action = func(ctx *cli.Context) error {
				return hostCreateRun(ctx, c, hostSchema, c.GetSchemas())
			}
			hostCommand.Subcommands[i].SkipFlagParsing = false
		}
	}

	app := cli.NewApp()
	app.Flags = []cli.Flag{
		//TODO: remove duplication here
		cli.BoolFlag{
			Name:  "debug",
			Usage: "Debug logging",
		},
		cli.StringFlag{
			Name:   "config,c",
			Usage:  "Client configuration file (default ${HOME}/.rancher/cli.json)",
			EnvVar: "RANCHER_CLIENT_CONFIG",
		},
		cli.StringFlag{
			Name:   "environment,env",
			Usage:  "Environment name or ID",
			EnvVar: "RANCHER_ENVIRONMENT",
		},
		cli.StringFlag{
			Name:   "url",
			Usage:  "Specify the Rancher API endpoint URL",
			EnvVar: "RANCHER_URL",
		},
		cli.StringFlag{
			Name:   "access-key",
			Usage:  "Specify Rancher API access key",
			EnvVar: "RANCHER_ACCESS_KEY",
		},
		cli.StringFlag{
			Name:   "secret-key",
			Usage:  "Specify Rancher API secret key",
			EnvVar: "RANCHER_SECRET_KEY",
		},
		cli.StringFlag{
			Name:   "host",
			Usage:  "Host used for docker command",
			EnvVar: "RANCHER_DOCKER_HOST",
		},
		cli.StringFlag{
			Name:  "rancher-file,r",
			Usage: "Specify an alternate Rancher compose file (default: rancher-compose.yml)",
		},
		cli.StringFlag{
			Name:  "env-file,e",
			Usage: "Specify a file from which to read environment variables",
		},
		cli.StringSliceFlag{
			Name:   "file,f",
			Usage:  "Specify one or more alternate compose files (default: docker-compose.yml)",
			Value:  &cli.StringSlice{},
			EnvVar: "COMPOSE_FILE",
		},
		cli.StringFlag{
			Name:  "stack,s",
			Usage: "Specify an alternate project name (default: directory name)",
		},
		cli.BoolFlag{
			Name:  "wait,w",
			Usage: "Wait for resource to reach resting state",
		},
		cli.IntFlag{
			Name:  "wait-timeout",
			Usage: "Timeout in seconds to wait",
			Value: 600,
		},
		cli.StringFlag{
			Name:  "wait-state",
			Usage: "State to wait for (active, healthy, etc)",
		},
	}
	app.Commands = []cli.Command{
		hostCommand,
	}
	return app.Run(os.Args)
}

func hostCreateRun(ctx *cli.Context, c *client.RancherClient, machineSchema client.Schema, schemas *client.Schemas) error {
	args := map[string]interface{}{}
	driverArgs := map[string]interface{}{}
	driver := ctx.String("driver")

	if driver == "" {
		return fmt.Errorf("--driver is required")
	}

	driverSchema, ok := schemas.CheckSchema(driver + "Config")
	if !ok {
		return fmt.Errorf("Invalid driver: %s", driver)
	}

	for _, name := range ctx.FlagNames() {
		schema := machineSchema
		destArgs := args
		key := name
		value := ctx.Generic(name)

		// really dumb way to detect empty values
		if str := fmt.Sprint(value); str == "" || str == "[]" {
			continue
		}

		if strings.HasPrefix(name, driver+"-") {
			key = toAPI(strings.TrimPrefix(name, driver+"-"))
			schema = driverSchema
			destArgs = driverArgs
		}

		fieldType := schema.ResourceFields[key].Type
		if fieldType == "map[string]" {
			mapValue := map[string]string{}
			for _, val := range ctx.StringSlice(name) {
				parts := strings.SplitN(val, "=", 2)
				if len(parts) == 1 {
					mapValue[parts[0]] = ""
				} else {
					mapValue[parts[0]] = parts[1]
				}
			}
			value = mapValue
		}

		destArgs[key] = value
	}

	args[driver+"Config"] = driverArgs

	names := ctx.Args()
	if len(names) == 0 {
		names = []string{RandomName()}
	}

	w, err := NewWaiter(ctx)
	if err != nil {
		return err
	}

	var lastErr error
	for _, name := range names {
		args["hostname"] = name
		var machine client.Machine
		if err := c.Create("host", args, &machine); err != nil {
			lastErr = err
			logrus.Error(err)
		} else {
			w.Add(machine.Id)
		}
	}

	if lastErr != nil {
		return lastErr
	}

	return w.Wait()
}
<|endoftext|>


<commit_before><commit_msg>Propagate Score setting to child search request during MultiSearch(..)<commit_after><|endoftext|>


<commit_before>package main

import (
	"crypto/sha256"
	"crypto/tls"
	"expvar"
	"flag"
	"fmt"
	"github.com/valyala/fasthttp"
	"github.com/valyala/httpteleport"
	"github.com/valyala/tcplisten"
	"log"
	"net"
	"os"
	"strings"
	"time"
)

var (
	reusePort = flag.Bool("reusePort", false, "Whether to enable SO_REUSEPORT on -in if -inType is http or httptp")

	in     = flag.String("in", "127.0.0.1:8080", "-inType address to listen to for incoming requests")
	inType = flag.String("inType", "http", "Type of -in address. Supported values:\n"+
		"\thttp - accept HTTP requests over TCP, e.g. -in=127.0.0.1:8080\n"+
		"\thttps - accept HTTPS requests over TCP, e.g. -in=127.0.0.1:443\n"+
		"\tunix - accept HTTP requests over unix socket, e.g. -in=/var/httptp/sock.unix\n"+
		"\thttptp - accept httptp connections over TCP, e.g. -in=127.0.0.1:8043")
	inDelay    = flag.Duration("inDelay", 0, "How long to wait before sending batched responses back if -inType=httptp")
	inCompress = flag.String("inCompress", "flate", "Which compression to use for responses if -inType=httptp.\n"+
		"\tSupported values:\n"+
		"\tnone - responses aren't compressed. Low CPU usage at the cost of high network bandwidth\n"+
		"\tflate - responses are compressed using flate algorithm. Low network bandwidth at the cost of high CPU usage\n"+
		"\tsnappy - responses are compressed using snappy algorithm. Balance between network bandwidth and CPU usage")

	inTLSCert             = flag.String("inTLSCert", "/etc/ssl/certs/ssl-cert-snakeoil.pem", "Path to TLS certificate file if -inType=https")
	inTLSKey              = flag.String("inTLSKey", "/etc/ssl/private/ssl-cert-snakeoil.key", "Path to TLS key file if -inType=https")
	inTLSSessionTicketKey = flag.String("inTLSSessionTicketKey", "", "TLS sesssion ticket key if -inType=https. Automatically generated if empty.\n"+
		"\tSee https://blog.cloudflare.com/tls-session-resumption-full-speed-and-secure/ for details")

	out = flag.String("out", "127.0.0.1:8043", "Comma-separated list of -outType addresses to forward requests to.\n"+
		"\tEach request is forwarded to the least loaded address")
	outType = flag.String("outType", "httptp", "Type of -out address. Supported values:\n"+
		"\thttp - forward requests to HTTP servers on TCP, e.g. -out=127.0.0.1:80\n"+
		"\tunix - forward requests to HTTP servers on unix socket, e.g. -out=/var/nginx/sock.unix\n"+
		"\thttptp - forward requests to httptp servers over TCP, e.g. -out=127.0.0.1:8043")
	outDelay    = flag.Duration("outDelay", 0, "How long to wait before forwarding incoming requests to -out if -outType=httptp")
	outCompress = flag.String("outCompress", "flate", "Which compression to use for requests if -outType=httptp.\n"+
		"\tSupported values:\n"+
		"\tnone - requests aren't compressed. Low CPU usage at the cost of high network bandwidth\n"+
		"\tflate - requests are compressed using flate algorithm. Low network bandwidth at the cost of high CPU usage\n"+
		"\tsnappy - requests are compressed using snappy algorithm. Balance between network bandwidth and CPU usage")

	outConnsPerAddr = flag.Int("outConnsPerAddr", 1, "How many connections must be established per each -out server if -outType=httptp.\n"+
		"\tUsually a single connection is enough. Increase this value if the compression\n"+
		"\ton the connection occupies 100% of a single CPU core.\n"+
		"\tAlternatively, -inCompress and/or -outCompress may be set to snappy or none in order to reduce CPU load")

	concurrency   = flag.Int("concurrency", 100000, "The maximum number of concurrent requests httptp may process")
	timeout       = flag.Duration("timeout", 3*time.Second, "The maximum duration for waiting responses from -out server")
	xForwardedFor = flag.Bool("xForwardedFor", false, "Whether to set client's ip in X-Forwarded-For request header for outgoing requests")
)

func main() {
	flag.Parse()

	initExpvarServer()

	outs := strings.Split(*out, ",")
	switch *outType {
	case "http":
		initHTTPClients(outs)
	case "unix":
		initUnixClients(outs)
	case "httptp":
		initHTTPTPClients(outs)
	default:
		log.Fatalf("unknown -outType=%q. Supported values are: http, unix, httptp", *outType)
	}

	switch *inType {
	case "http":
		serveHTTP()
	case "https":
		serveHTTPS()
	case "unix":
		serveUnix()
	case "httptp":
		serveHTTPTP()
	default:
		log.Fatalf("unknown -inType=%q. Supported values are: http, https, unix and httptp", *inType)
	}
}

func initHTTPClients(outs []string) {
	connsPerAddr := (*concurrency + len(outs) - 1) / len(outs)
	for _, addr := range outs {
		c := newHTTPClient(fasthttp.Dial, addr, connsPerAddr)
		upstreamClients = append(upstreamClients, c)
	}
	log.Printf("forwarding requests to HTTP servers at %q", outs)
}

func initUnixClients(outs []string) {
	connsPerAddr := (*concurrency + len(outs) - 1) / len(outs)
	for _, addr := range outs {
		verifyUnixAddr(addr)
		c := newHTTPClient(dialUnix, addr, connsPerAddr)
		upstreamClients = append(upstreamClients, c)
	}
	log.Printf("forwarding requests to HTTP servers at unix:%q", outs)
}

func verifyUnixAddr(addr string) {
	fi, err := os.Stat(addr)
	if err != nil {
		log.Fatalf("error when accessing unix:%q: %s", addr, err)
	}
	mode := fi.Mode()
	if (mode & os.ModeSocket) == 0 {
		log.Fatalf("the %q must be unix socket", addr)
	}
}

func initHTTPTPClients(outs []string) {
	concurrencyPerAddr := (*concurrency + len(outs) - 1) / len(outs)
	concurrencyPerAddr = (concurrencyPerAddr + *outConnsPerAddr - 1) / *outConnsPerAddr
	outCompressType := compressType(*outCompress, "outCompress")
	var cs []client
	for _, addr := range outs {
		c := &httpteleport.Client{
			Addr:               addr,
			Dial:               newExpvarDial(fasthttp.Dial),
			MaxBatchDelay:      *outDelay,
			MaxPendingRequests: concurrencyPerAddr,
			ReadTimeout:        120 * time.Second,
			WriteTimeout:       5 * time.Second,
			CompressType:       outCompressType,
		}
		cs = append(cs, c)
	}
	for i := 0; i < *outConnsPerAddr; i++ {
		upstreamClients = append(upstreamClients, cs...)
	}
	log.Printf("forwarding requests to httptp servers at %q", outs)
}

func compressType(ct, name string) httpteleport.CompressType {
	switch ct {
	case "none":
		return httpteleport.CompressNone
	case "flate":
		return httpteleport.CompressFlate
	case "snappy":
		return httpteleport.CompressSnappy
	default:
		log.Fatalf("unknown -%s: %q. Supported values: none, flate, snappy", name, ct)
	}
	panic("unreached")
}

func newHTTPClient(dial fasthttp.DialFunc, addr string, connsPerAddr int) client {
	return &fasthttp.HostClient{
		Addr:         addr,
		Dial:         newExpvarDial(dial),
		MaxConns:     connsPerAddr,
		ReadTimeout:  *timeout * 5,
		WriteTimeout: *timeout,
	}
}

func dialUnix(addr string) (net.Conn, error) {
	return net.Dial("unix", addr)
}

func serveHTTP() {
	ln := newTCPListener()
	s := newHTTPServer()

	log.Printf("listening for HTTP requests on %q", *in)
	if err := s.Serve(ln); err != nil {
		log.Fatalf("error in fasthttp server: %s", err)
	}
}

func serveHTTPS() {
	ln := newTCPListener()

	cert, err := tls.LoadX509KeyPair(*inTLSCert, *inTLSKey)
	if err != nil {
		log.Fatalf("cannot load TLS certificate from -inTLSCert=%q and -inTLSKey=%q: %s", *inTLSCert, *inTLSKey, err)
	}
	tlsConfig := &tls.Config{
		Certificates:             []tls.Certificate{cert},
		PreferServerCipherSuites: true,
	}
	if len(*inTLSSessionTicketKey) > 0 {
		tlsConfig.SessionTicketKey = sha256.Sum256([]byte(*inTLSSessionTicketKey))
	}

	lnTLS := tls.NewListener(ln, tlsConfig)
	s := newHTTPServer()

	log.Printf("listening for HTTPS requests on %q", *in)
	if err := s.Serve(lnTLS); err != nil {
		log.Fatalf("error in fasthttp server: %s", err)
	}
}

func serveUnix() {
	addr := *in
	if _, err := os.Stat(addr); err == nil {
		verifyUnixAddr(addr)
		if err := os.Remove(addr); err != nil {
			log.Fatalf("cannot remove %q: %s", addr, err)
		}
	}

	ln, err := net.Listen("unix", addr)
	if err != nil {
		log.Fatalf("cannot listen to -in=%q: %s", addr, err)
	}
	s := newHTTPServer()

	log.Printf("listening for HTTP requests on unix:%q", addr)
	if err := s.Serve(ln); err != nil {
		log.Fatalf("error in fasthttp server: %s", err)
	}
}

func serveHTTPTP() {
	ln := newTCPListener()
	inCompressType := compressType(*inCompress, "inCompress")
	s := httpteleport.Server{
		Handler:           httptpRequestHandler,
		Concurrency:       *concurrency,
		MaxBatchDelay:     *inDelay,
		ReduceMemoryUsage: true,
		ReadTimeout:       120 * time.Second,
		WriteTimeout:      5 * time.Second,
		CompressType:      inCompressType,
	}

	log.Printf("listening for httptp connections on %q", *in)
	if err := s.Serve(ln); err != nil {
		log.Fatalf("error in fasthttp server: %s", err)
	}
}

func newTCPListener() net.Listener {
	cfg := tcplisten.Config{
		ReusePort: *reusePort,
	}
	ln, err := cfg.NewListener("tcp4", *in)
	if err != nil {
		log.Fatalf("cannot listen to -in=%q: %s", *in, err)
	}
	return &expvarListener{
		Listener: ln,
	}
}

func newHTTPServer() *fasthttp.Server {
	return &fasthttp.Server{
		Handler:           httpRequestHandler,
		Concurrency:       *concurrency,
		ReduceMemoryUsage: true,
		ReadTimeout:       120 * time.Second,
		WriteTimeout:      5 * time.Second,
	}
}

var (
	inRequestStart        = expvar.NewInt("inRequestStart")
	inRequestSuccess      = expvar.NewInt("inRequestSuccess")
	inRequestNon200       = expvar.NewInt("inRequestNon200")
	inRequestTimeoutError = expvar.NewInt("inRequestTimeoutError")
	inRequestOtherError   = expvar.NewInt("inRequestOtherError")
)

func httpRequestHandler(ctx *fasthttp.RequestCtx) {
	inRequestStart.Add(1)
	if *xForwardedFor {
		var buf [16]byte
		ip := fasthttp.AppendIPv4(buf[:0], ctx.RemoteIP())
		ctx.Request.Header.SetBytesV("X-Forwarded-For", ip)
	}

	c := leastLoadedClient()
	err := c.DoTimeout(&ctx.Request, &ctx.Response, *timeout)
	if err == nil {
		inRequestSuccess.Add(1)
		if ctx.Response.StatusCode() != fasthttp.StatusOK {
			inRequestNon200.Add(1)
		}
		return
	}

	ctx.ResetBody()
	fmt.Fprintf(ctx, "HTTP proxying error: %s", err)
	if err == fasthttp.ErrTimeout {
		inRequestTimeoutError.Add(1)
		ctx.SetStatusCode(fasthttp.StatusGatewayTimeout)
	} else {
		inRequestOtherError.Add(1)
		ctx.SetStatusCode(fasthttp.StatusBadGateway)
	}
}

func httptpRequestHandler(ctx *fasthttp.RequestCtx) {
	inRequestStart.Add(1)
	// Reset 'Connection: close' request header in order to prevent
	// from closing keep-alive connections to -out servers.
	ctx.Request.Header.ResetConnectionClose()

	c := leastLoadedClient()
	err := c.DoTimeout(&ctx.Request, &ctx.Response, *timeout)
	if err == nil {
		inRequestSuccess.Add(1)
		if ctx.Response.StatusCode() != fasthttp.StatusOK {
			inRequestNon200.Add(1)
		}
		return
	}

	ctx.ResetBody()
	fmt.Fprintf(ctx, "httptp proxying error: %s", err)
	if err == httpteleport.ErrTimeout {
		inRequestTimeoutError.Add(1)
		ctx.SetStatusCode(fasthttp.StatusGatewayTimeout)
	} else {
		inRequestOtherError.Add(1)
		ctx.SetStatusCode(fasthttp.StatusBadGateway)
	}
}

type client interface {
	DoTimeout(req *fasthttp.Request, resp *fasthttp.Response, timeout time.Duration) error
	PendingRequests() int
}

var upstreamClients []client

func leastLoadedClient() client {
	minC := upstreamClients[0]
	minN := minC.PendingRequests()
	if minN == 0 {
		return minC
	}
	for _, c := range upstreamClients[1:] {
		n := c.PendingRequests()
		if n == 0 {
			return c
		}
		if n < minN {
			minC = c
			minN = n
		}
	}
	return minC
}
<commit_msg>renamed "httptp" type to more clear "teleport" type<commit_after>package main

import (
	"crypto/sha256"
	"crypto/tls"
	"expvar"
	"flag"
	"fmt"
	"github.com/valyala/fasthttp"
	"github.com/valyala/httpteleport"
	"github.com/valyala/tcplisten"
	"log"
	"net"
	"os"
	"strings"
	"time"
)

var (
	reusePort = flag.Bool("reusePort", false, "Whether to enable SO_REUSEPORT on -in if -inType is http or teleport")

	in     = flag.String("in", "127.0.0.1:8080", "-inType address to listen to for incoming requests")
	inType = flag.String("inType", "http", "Type of -in address. Supported values:\n"+
		"\thttp - accept HTTP requests over TCP, e.g. -in=127.0.0.1:8080\n"+
		"\thttps - accept HTTPS requests over TCP, e.g. -in=127.0.0.1:443\n"+
		"\tunix - accept HTTP requests over unix socket, e.g. -in=/var/httptp/sock.unix\n"+
		"\tteleport - accept httpteleport connections over TCP, e.g. -in=127.0.0.1:8043")
	inDelay    = flag.Duration("inDelay", 0, "How long to wait before sending batched responses back if -inType=teleport")
	inCompress = flag.String("inCompress", "flate", "Which compression to use for responses if -inType=teleport.\n"+
		"\tSupported values:\n"+
		"\tnone - responses aren't compressed. Low CPU usage at the cost of high network bandwidth\n"+
		"\tflate - responses are compressed using flate algorithm. Low network bandwidth at the cost of high CPU usage\n"+
		"\tsnappy - responses are compressed using snappy algorithm. Balance between network bandwidth and CPU usage")

	inTLSCert             = flag.String("inTLSCert", "/etc/ssl/certs/ssl-cert-snakeoil.pem", "Path to TLS certificate file if -inType=https")
	inTLSKey              = flag.String("inTLSKey", "/etc/ssl/private/ssl-cert-snakeoil.key", "Path to TLS key file if -inType=https")
	inTLSSessionTicketKey = flag.String("inTLSSessionTicketKey", "", "TLS sesssion ticket key if -inType=https. Automatically generated if empty.\n"+
		"\tSee https://blog.cloudflare.com/tls-session-resumption-full-speed-and-secure/ for details")

	out = flag.String("out", "127.0.0.1:8043", "Comma-separated list of -outType addresses to forward requests to.\n"+
		"\tEach request is forwarded to the least loaded address")
	outType = flag.String("outType", "teleport", "Type of -out address. Supported values:\n"+
		"\thttp - forward requests to HTTP servers on TCP, e.g. -out=127.0.0.1:80\n"+
		"\tunix - forward requests to HTTP servers on unix socket, e.g. -out=/var/nginx/sock.unix\n"+
		"\tteleport - forward requests to httpteleport servers over TCP, e.g. -out=127.0.0.1:8043")
	outDelay    = flag.Duration("outDelay", 0, "How long to wait before forwarding incoming requests to -out if -outType=teleport")
	outCompress = flag.String("outCompress", "flate", "Which compression to use for requests if -outType=teleport.\n"+
		"\tSupported values:\n"+
		"\tnone - requests aren't compressed. Low CPU usage at the cost of high network bandwidth\n"+
		"\tflate - requests are compressed using flate algorithm. Low network bandwidth at the cost of high CPU usage\n"+
		"\tsnappy - requests are compressed using snappy algorithm. Balance between network bandwidth and CPU usage")

	outConnsPerAddr = flag.Int("outConnsPerAddr", 1, "How many connections must be established per each -out server if -outType=teleport.\n"+
		"\tUsually a single connection is enough. Increase this value if the compression\n"+
		"\ton the connection occupies 100% of a single CPU core.\n"+
		"\tAlternatively, -inCompress and/or -outCompress may be set to snappy or none in order to reduce CPU load")

	concurrency   = flag.Int("concurrency", 100000, "The maximum number of concurrent requests httptp may process")
	timeout       = flag.Duration("timeout", 3*time.Second, "The maximum duration for waiting responses from -out server")
	xForwardedFor = flag.Bool("xForwardedFor", false, "Whether to set client's ip in X-Forwarded-For request header for outgoing requests")
)

func main() {
	flag.Parse()

	initExpvarServer()

	outs := strings.Split(*out, ",")
	switch *outType {
	case "http":
		initHTTPClients(outs)
	case "unix":
		initUnixClients(outs)
	case "teleport":
		initHTTPTPClients(outs)
	default:
		log.Fatalf("unknown -outType=%q. Supported values are: http, unix, teleport", *outType)
	}

	switch *inType {
	case "http":
		serveHTTP()
	case "https":
		serveHTTPS()
	case "unix":
		serveUnix()
	case "teleport":
		serveHTTPTP()
	default:
		log.Fatalf("unknown -inType=%q. Supported values are: http, https, unix and teleport", *inType)
	}
}

func initHTTPClients(outs []string) {
	connsPerAddr := (*concurrency + len(outs) - 1) / len(outs)
	for _, addr := range outs {
		c := newHTTPClient(fasthttp.Dial, addr, connsPerAddr)
		upstreamClients = append(upstreamClients, c)
	}
	log.Printf("forwarding requests to HTTP servers at %q", outs)
}

func initUnixClients(outs []string) {
	connsPerAddr := (*concurrency + len(outs) - 1) / len(outs)
	for _, addr := range outs {
		verifyUnixAddr(addr)
		c := newHTTPClient(dialUnix, addr, connsPerAddr)
		upstreamClients = append(upstreamClients, c)
	}
	log.Printf("forwarding requests to HTTP servers at unix:%q", outs)
}

func verifyUnixAddr(addr string) {
	fi, err := os.Stat(addr)
	if err != nil {
		log.Fatalf("error when accessing unix:%q: %s", addr, err)
	}
	mode := fi.Mode()
	if (mode & os.ModeSocket) == 0 {
		log.Fatalf("the %q must be unix socket", addr)
	}
}

func initHTTPTPClients(outs []string) {
	concurrencyPerAddr := (*concurrency + len(outs) - 1) / len(outs)
	concurrencyPerAddr = (concurrencyPerAddr + *outConnsPerAddr - 1) / *outConnsPerAddr
	outCompressType := compressType(*outCompress, "outCompress")
	var cs []client
	for _, addr := range outs {
		c := &httpteleport.Client{
			Addr:               addr,
			Dial:               newExpvarDial(fasthttp.Dial),
			MaxBatchDelay:      *outDelay,
			MaxPendingRequests: concurrencyPerAddr,
			ReadTimeout:        120 * time.Second,
			WriteTimeout:       5 * time.Second,
			CompressType:       outCompressType,
		}
		cs = append(cs, c)
	}
	for i := 0; i < *outConnsPerAddr; i++ {
		upstreamClients = append(upstreamClients, cs...)
	}
	log.Printf("forwarding requests to httpteleport servers at %q", outs)
}

func compressType(ct, name string) httpteleport.CompressType {
	switch ct {
	case "none":
		return httpteleport.CompressNone
	case "flate":
		return httpteleport.CompressFlate
	case "snappy":
		return httpteleport.CompressSnappy
	default:
		log.Fatalf("unknown -%s: %q. Supported values: none, flate, snappy", name, ct)
	}
	panic("unreached")
}

func newHTTPClient(dial fasthttp.DialFunc, addr string, connsPerAddr int) client {
	return &fasthttp.HostClient{
		Addr:         addr,
		Dial:         newExpvarDial(dial),
		MaxConns:     connsPerAddr,
		ReadTimeout:  *timeout * 5,
		WriteTimeout: *timeout,
	}
}

func dialUnix(addr string) (net.Conn, error) {
	return net.Dial("unix", addr)
}

func serveHTTP() {
	ln := newTCPListener()
	s := newHTTPServer()

	log.Printf("listening for HTTP requests on %q", *in)
	if err := s.Serve(ln); err != nil {
		log.Fatalf("error in fasthttp server: %s", err)
	}
}

func serveHTTPS() {
	ln := newTCPListener()

	cert, err := tls.LoadX509KeyPair(*inTLSCert, *inTLSKey)
	if err != nil {
		log.Fatalf("cannot load TLS certificate from -inTLSCert=%q and -inTLSKey=%q: %s", *inTLSCert, *inTLSKey, err)
	}
	tlsConfig := &tls.Config{
		Certificates:             []tls.Certificate{cert},
		PreferServerCipherSuites: true,
	}
	if len(*inTLSSessionTicketKey) > 0 {
		tlsConfig.SessionTicketKey = sha256.Sum256([]byte(*inTLSSessionTicketKey))
	}

	lnTLS := tls.NewListener(ln, tlsConfig)
	s := newHTTPServer()

	log.Printf("listening for HTTPS requests on %q", *in)
	if err := s.Serve(lnTLS); err != nil {
		log.Fatalf("error in fasthttp server: %s", err)
	}
}

func serveUnix() {
	addr := *in
	if _, err := os.Stat(addr); err == nil {
		verifyUnixAddr(addr)
		if err := os.Remove(addr); err != nil {
			log.Fatalf("cannot remove %q: %s", addr, err)
		}
	}

	ln, err := net.Listen("unix", addr)
	if err != nil {
		log.Fatalf("cannot listen to -in=%q: %s", addr, err)
	}
	s := newHTTPServer()

	log.Printf("listening for HTTP requests on unix:%q", addr)
	if err := s.Serve(ln); err != nil {
		log.Fatalf("error in fasthttp server: %s", err)
	}
}

func serveHTTPTP() {
	ln := newTCPListener()
	inCompressType := compressType(*inCompress, "inCompress")
	s := httpteleport.Server{
		Handler:           httpteleportRequestHandler,
		Concurrency:       *concurrency,
		MaxBatchDelay:     *inDelay,
		ReduceMemoryUsage: true,
		ReadTimeout:       120 * time.Second,
		WriteTimeout:      5 * time.Second,
		CompressType:      inCompressType,
	}

	log.Printf("listening for httpteleport connections on %q", *in)
	if err := s.Serve(ln); err != nil {
		log.Fatalf("error in fasthttp server: %s", err)
	}
}

func newTCPListener() net.Listener {
	cfg := tcplisten.Config{
		ReusePort: *reusePort,
	}
	ln, err := cfg.NewListener("tcp4", *in)
	if err != nil {
		log.Fatalf("cannot listen to -in=%q: %s", *in, err)
	}
	return &expvarListener{
		Listener: ln,
	}
}

func newHTTPServer() *fasthttp.Server {
	return &fasthttp.Server{
		Handler:           httpRequestHandler,
		Concurrency:       *concurrency,
		ReduceMemoryUsage: true,
		ReadTimeout:       120 * time.Second,
		WriteTimeout:      5 * time.Second,
	}
}

var (
	inRequestStart        = expvar.NewInt("inRequestStart")
	inRequestSuccess      = expvar.NewInt("inRequestSuccess")
	inRequestNon200       = expvar.NewInt("inRequestNon200")
	inRequestTimeoutError = expvar.NewInt("inRequestTimeoutError")
	inRequestOtherError   = expvar.NewInt("inRequestOtherError")
)

func httpRequestHandler(ctx *fasthttp.RequestCtx) {
	inRequestStart.Add(1)
	if *xForwardedFor {
		var buf [16]byte
		ip := fasthttp.AppendIPv4(buf[:0], ctx.RemoteIP())
		ctx.Request.Header.SetBytesV("X-Forwarded-For", ip)
	}

	c := leastLoadedClient()
	err := c.DoTimeout(&ctx.Request, &ctx.Response, *timeout)
	if err == nil {
		inRequestSuccess.Add(1)
		if ctx.Response.StatusCode() != fasthttp.StatusOK {
			inRequestNon200.Add(1)
		}
		return
	}

	ctx.ResetBody()
	fmt.Fprintf(ctx, "HTTP proxying error: %s", err)
	if err == fasthttp.ErrTimeout {
		inRequestTimeoutError.Add(1)
		ctx.SetStatusCode(fasthttp.StatusGatewayTimeout)
	} else {
		inRequestOtherError.Add(1)
		ctx.SetStatusCode(fasthttp.StatusBadGateway)
	}
}

func httpteleportRequestHandler(ctx *fasthttp.RequestCtx) {
	inRequestStart.Add(1)
	// Reset 'Connection: close' request header in order to prevent
	// from closing keep-alive connections to -out servers.
	ctx.Request.Header.ResetConnectionClose()

	c := leastLoadedClient()
	err := c.DoTimeout(&ctx.Request, &ctx.Response, *timeout)
	if err == nil {
		inRequestSuccess.Add(1)
		if ctx.Response.StatusCode() != fasthttp.StatusOK {
			inRequestNon200.Add(1)
		}
		return
	}

	ctx.ResetBody()
	fmt.Fprintf(ctx, "httpteleport proxying error: %s", err)
	if err == httpteleport.ErrTimeout {
		inRequestTimeoutError.Add(1)
		ctx.SetStatusCode(fasthttp.StatusGatewayTimeout)
	} else {
		inRequestOtherError.Add(1)
		ctx.SetStatusCode(fasthttp.StatusBadGateway)
	}
}

type client interface {
	DoTimeout(req *fasthttp.Request, resp *fasthttp.Response, timeout time.Duration) error
	PendingRequests() int
}

var upstreamClients []client

func leastLoadedClient() client {
	minC := upstreamClients[0]
	minN := minC.PendingRequests()
	if minN == 0 {
		return minC
	}
	for _, c := range upstreamClients[1:] {
		n := c.PendingRequests()
		if n == 0 {
			return c
		}
		if n < minN {
			minC = c
			minN = n
		}
	}
	return minC
}
<|endoftext|>


<commit_before>package main

import (
	"net/url"

	"github.com/hellofresh/kandalf/pkg/amqp"
	"github.com/hellofresh/kandalf/pkg/config"
	"github.com/hellofresh/kandalf/pkg/producer"
	"github.com/hellofresh/kandalf/pkg/storage"
	"github.com/hellofresh/kandalf/pkg/workers"
	"github.com/hellofresh/stats-go"
	"github.com/hellofresh/stats-go/hooks"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

// RunApp is main application bootstrap and runner
func RunApp(cmd *cobra.Command, args []string) {
	log.WithField("version", version).Info("Kandalf starting...")

	globalConfig, err := config.Load(configPath)
	failOnError(err, "Failed to load application configuration")

	err = globalConfig.Log.Apply()
	failOnError(err, "Failed to configure logger")
	defer globalConfig.Log.Flush()

	statsClient, err := stats.NewClient(globalConfig.Stats.DSN, globalConfig.Stats.Prefix)
	failOnError(err, "Failed to connect to stats service")
	defer func() {
		if err := statsClient.Close(); err != nil {
			log.WithError(err).Error("Got error on closing stats client")
		}
	}()

	log.AddHook(hooks.NewLogrusHook(statsClient, globalConfig.Stats.ErrorsSection))

	pipesList, err := config.LoadPipesFromFile(globalConfig.Kafka.PipesConfig)
	failOnError(err, "Failed to load pipes config")

	storageURL, err := url.Parse(globalConfig.StorageDSN)
	failOnError(err, "Failed to parse Storage DSN")

	persistentStorage, err := storage.NewPersistentStorage(storageURL)
	failOnError(err, "Failed to establish Redis connection")
	// Do not close storage here as it is required in Worker close to store unhandled messages

	kafkaProducer, err := producer.NewKafkaProducer(globalConfig.Kafka, statsClient)
	failOnError(err, "Failed to establish Kafka connection")
	defer func() {
		if err := kafkaProducer.Close(); err != nil {
			log.WithError(err).Error("Got error on closing kafka producer")
		}
	}()

	worker, err := workers.NewBridgeWorker(globalConfig.Worker, persistentStorage, kafkaProducer, statsClient)
	defer func() {
		if err := worker.Close(); err != nil {
			log.WithError(err).Error("Got error on closing persistent storage")
		}
	}()

	queuesHandler := amqp.NewQueuesHandler(pipesList, worker.MessageHandler, statsClient)
	amqpConnection, err := amqp.NewConnection(globalConfig.RabbitDSN, queuesHandler)
	failOnError(err, "Failed to establish initial connection to AMQP")
	defer func() {
		if err := amqpConnection.Close(); err != nil {
			log.WithError(err).Error("Got error on closing AMQP connection")
		}
	}()

	forever := make(chan bool)

	worker.Go(forever)

	log.Infof("[*] Waiting for users. To exit press CTRL+C")
	<-forever
}
<commit_msg>Track application start/restart<commit_after>package main

import (
	"net/url"
	"os"
	"path/filepath"

	"github.com/hellofresh/kandalf/pkg/amqp"
	"github.com/hellofresh/kandalf/pkg/config"
	"github.com/hellofresh/kandalf/pkg/producer"
	"github.com/hellofresh/kandalf/pkg/storage"
	"github.com/hellofresh/kandalf/pkg/workers"
	"github.com/hellofresh/stats-go"
	"github.com/hellofresh/stats-go/bucket"
	"github.com/hellofresh/stats-go/hooks"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

// RunApp is main application bootstrap and runner
func RunApp(cmd *cobra.Command, args []string) {
	log.WithField("version", version).Info("Kandalf starting...")

	globalConfig, err := config.Load(configPath)
	failOnError(err, "Failed to load application configuration")

	err = globalConfig.Log.Apply()
	failOnError(err, "Failed to configure logger")
	defer globalConfig.Log.Flush()

	statsClient := initStatsClient(globalConfig.Stats)
	defer func() {
		if err := statsClient.Close(); err != nil {
			log.WithError(err).Error("Got error on closing stats client")
		}
	}()

	pipesList, err := config.LoadPipesFromFile(globalConfig.Kafka.PipesConfig)
	failOnError(err, "Failed to load pipes config")

	storageURL, err := url.Parse(globalConfig.StorageDSN)
	failOnError(err, "Failed to parse Storage DSN")

	persistentStorage, err := storage.NewPersistentStorage(storageURL)
	failOnError(err, "Failed to establish Redis connection")
	// Do not close storage here as it is required in Worker close to store unhandled messages

	kafkaProducer, err := producer.NewKafkaProducer(globalConfig.Kafka, statsClient)
	failOnError(err, "Failed to establish Kafka connection")
	defer func() {
		if err := kafkaProducer.Close(); err != nil {
			log.WithError(err).Error("Got error on closing kafka producer")
		}
	}()

	worker, err := workers.NewBridgeWorker(globalConfig.Worker, persistentStorage, kafkaProducer, statsClient)
	defer func() {
		if err := worker.Close(); err != nil {
			log.WithError(err).Error("Got error on closing persistent storage")
		}
	}()

	queuesHandler := amqp.NewQueuesHandler(pipesList, worker.MessageHandler, statsClient)
	amqpConnection, err := amqp.NewConnection(globalConfig.RabbitDSN, queuesHandler)
	failOnError(err, "Failed to establish initial connection to AMQP")
	defer func() {
		if err := amqpConnection.Close(); err != nil {
			log.WithError(err).Error("Got error on closing AMQP connection")
		}
	}()

	forever := make(chan bool)

	worker.Go(forever)

	log.Infof("[*] Waiting for users. To exit press CTRL+C")
	<-forever
}

func initStatsClient(config config.StatsConfig) stats.Client {
	statsClient, err := stats.NewClient(config.DSN, config.Prefix)
	failOnError(err, "Failed to init stats client!")

	log.AddHook(hooks.NewLogrusHook(statsClient, config.ErrorsSection))

	host, err := os.Hostname()
	if nil != err {
		host = "-unknown-"
	}

	_, appFile := filepath.Split(os.Args[0])
	statsClient.TrackMetric("app", bucket.MetricOperation{"init", host, appFile})

	return statsClient
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3"
	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/secrets"
	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/sshagent"
)

const (
	envBucket     = "BUILDKITE_PLUGIN_S3_SECRETS_BUCKET"
	envPrefix     = "BUILDKITE_PLUGIN_S3_SECRETS_BUCKET_PREFIX"
	envPipeline   = "BUILDKITE_PIPELINE_SLUG"
	envRepo       = "BUILDKITE_REPO"
	envEnvSink    = "BUILDKITE_PLUGIN_S3_SECRETS_ENV_SINK"
	envCredHelper = "GIT_CREDENTIAL_S3_SECRETS"

	envDefaultRegion = "AWS_DEFAULT_REGION"
	defaultRegion    = "us-east-1"
)

func main() {
	log := log.New(os.Stderr, "[secrets] ", log.Lmsgprefix)
	if err := mainWithError(log); err != nil {
		log.Fatalf("fatal error: %v", err)
	}
}

func mainWithError(log *log.Logger) error {
	bucket := os.Getenv(envBucket)
	if bucket == "" {
		return nil
	}

	prefix := os.Getenv(envPrefix)
	if prefix == "" {
		prefix = os.Getenv(envPipeline)
	}
	if prefix == "" {
		return fmt.Errorf("%s or %s required", envPrefix, envPipeline)
	}

	region := os.Getenv(envDefaultRegion)
	if region == "" {
		region = defaultRegion
	}
	client, err := s3.New(region)
	if err != nil {
		return err
	}

	agent := &sshagent.Agent{}

	envSinkPath := os.Getenv(envEnvSink)
	if envSinkPath == "" {
		return fmt.Errorf("%s required", envEnvSink)
	}
	envSink, err := os.OpenFile(envSinkPath, os.O_WRONLY|os.O_APPEND, 0)
	if err != nil {
		return err
	}

	credHelper := os.Getenv(envCredHelper)
	if credHelper == "" {
		return fmt.Errorf("%s required", envCredHelper)
	}

	return secrets.Run(secrets.Config{
		Repo:                os.Getenv(envRepo),
		Bucket:              bucket,
		Prefix:              prefix,
		Client:              client,
		Logger:              log,
		SSHAgent:            agent,
		EnvSink:             envSink,
		GitCredentialHelper: credHelper,
	})
}
<commit_msg>go: fix envCredHelper name<commit_after>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3"
	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/secrets"
	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/sshagent"
)

const (
	envBucket     = "BUILDKITE_PLUGIN_S3_SECRETS_BUCKET"
	envPrefix     = "BUILDKITE_PLUGIN_S3_SECRETS_BUCKET_PREFIX"
	envPipeline   = "BUILDKITE_PIPELINE_SLUG"
	envRepo       = "BUILDKITE_REPO"
	envEnvSink    = "BUILDKITE_PLUGIN_S3_SECRETS_ENV_SINK"
	envCredHelper = "BUILDKITE_PLUGIN_S3_SECRETS_GIT_CREDENTIAL_HELPER"

	envDefaultRegion = "AWS_DEFAULT_REGION"
	defaultRegion    = "us-east-1"
)

func main() {
	log := log.New(os.Stderr, "[secrets] ", log.Lmsgprefix)
	if err := mainWithError(log); err != nil {
		log.Fatalf("fatal error: %v", err)
	}
}

func mainWithError(log *log.Logger) error {
	bucket := os.Getenv(envBucket)
	if bucket == "" {
		return nil
	}

	prefix := os.Getenv(envPrefix)
	if prefix == "" {
		prefix = os.Getenv(envPipeline)
	}
	if prefix == "" {
		return fmt.Errorf("%s or %s required", envPrefix, envPipeline)
	}

	region := os.Getenv(envDefaultRegion)
	if region == "" {
		region = defaultRegion
	}
	client, err := s3.New(region)
	if err != nil {
		return err
	}

	agent := &sshagent.Agent{}

	envSinkPath := os.Getenv(envEnvSink)
	if envSinkPath == "" {
		return fmt.Errorf("%s required", envEnvSink)
	}
	envSink, err := os.OpenFile(envSinkPath, os.O_WRONLY|os.O_APPEND, 0)
	if err != nil {
		return err
	}

	credHelper := os.Getenv(envCredHelper)
	if credHelper == "" {
		return fmt.Errorf("%s required", envCredHelper)
	}

	return secrets.Run(secrets.Config{
		Repo:                os.Getenv(envRepo),
		Bucket:              bucket,
		Prefix:              prefix,
		Client:              client,
		Logger:              log,
		SSHAgent:            agent,
		EnvSink:             envSink,
		GitCredentialHelper: credHelper,
	})
}
<|endoftext|>


<commit_before>package common

import (
	"container/list"
	"encoding/json"
	"github.com/Sirupsen/logrus"
	"github.com/portworx/kvdb"
	"sync"
	"time"
)

var (
	path = "/var/cores/"
)

// ToBytes converts to value to a byte slice.
func ToBytes(val interface{}) ([]byte, error) {
	switch val.(type) {
	case string:
		return []byte(val.(string)), nil
	case []byte:
		b := make([]byte, len(val.([]byte)))
		copy(b, val.([]byte))
		return b, nil
	default:
		return json.Marshal(val)
	}
}

// BaseKvdb provides common functionality across kvdb types
type BaseKvdb struct {
	// LockTimeout is the maximum time any lock can be held
	LockTimeout time.Duration
	// FatalCb invoked for fatal errors
	FatalCb kvdb.FatalErrorCB
}

func (b *BaseKvdb) SetFatalCb(f kvdb.FatalErrorCB) {
	b.FatalCb = f
}

func (b *BaseKvdb) SetLockTimeout(timeout time.Duration) {
	logrus.Infof("Setting lock timeout to: %v", timeout)
	b.LockTimeout = timeout
}

func (b *BaseKvdb) CheckLockTimeout(key string, startTime time.Time) {
	if b.LockTimeout > 0 && time.Since(startTime) > b.LockTimeout {
		b.LockTimedout(key)
	}
}

func (b *BaseKvdb) LockTimedout(key string) {
	b.FatalCb("Lock %s hold timeout triggered", key)
}

// watchUpdate refers to an update to this kvdb
type watchUpdate struct {
	// key is the key that was updated
	key string
	// kvp is the key-value that was updated
	kvp *kvdb.KVPair
	// err is any error on update
	err error
}

// WatchUpdateQueue is a producer consumer queue.
type WatchUpdateQueue interface {
	// Enqueue will enqueue an update. It is non-blocking.
	Enqueue(key string, kvp *kvdb.KVPair, err error)
	// Dequeue will either return an element from front of the queue or
	// will block until element becomes available
	Dequeue() (string, *kvdb.KVPair, error)
}

// watchQueue implements WatchUpdateQueue interface for watchUpdates
type watchQueue struct {
	// updates is the list of updates
	updates *list.List
	// m is the mutex to protect updates
	m *sync.Mutex
	// cv is used to coordinate the producer-consumer threads
	cv *sync.Cond
}

// NewWatchUpdateQueue returns WatchUpdateQueue
func NewWatchUpdateQueue() WatchUpdateQueue {
	mtx := &sync.Mutex{}
	return &watchQueue{
		m:       mtx,
		cv:      sync.NewCond(mtx),
		updates: list.New()}
}

func (w *watchQueue) Dequeue() (string, *kvdb.KVPair, error) {
	w.m.Lock()
	for {
		if w.updates.Len() > 0 {
			el := w.updates.Front()
			w.updates.Remove(el)
			w.m.Unlock()
			update := el.Value.(*watchUpdate)
			return update.key, update.kvp, update.err
		}
		w.cv.Wait()
	}
}

// Enqueue enqueues and never blocks
func (w *watchQueue) Enqueue(key string, kvp *kvdb.KVPair, err error) {
	w.m.Lock()
	w.updates.PushBack(&watchUpdate{key: key, kvp: kvp, err: err})
	w.cv.Signal()
	w.m.Unlock()
}
<commit_msg>Import order<commit_after>package common

import (
	"container/list"
	"encoding/json"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/portworx/kvdb"
)

var (
	path = "/var/cores/"
)

// ToBytes converts to value to a byte slice.
func ToBytes(val interface{}) ([]byte, error) {
	switch val.(type) {
	case string:
		return []byte(val.(string)), nil
	case []byte:
		b := make([]byte, len(val.([]byte)))
		copy(b, val.([]byte))
		return b, nil
	default:
		return json.Marshal(val)
	}
}

// BaseKvdb provides common functionality across kvdb types
type BaseKvdb struct {
	// LockTimeout is the maximum time any lock can be held
	LockTimeout time.Duration
	// FatalCb invoked for fatal errors
	FatalCb kvdb.FatalErrorCB
}

func (b *BaseKvdb) SetFatalCb(f kvdb.FatalErrorCB) {
	b.FatalCb = f
}

func (b *BaseKvdb) SetLockTimeout(timeout time.Duration) {
	logrus.Infof("Setting lock timeout to: %v", timeout)
	b.LockTimeout = timeout
}

func (b *BaseKvdb) CheckLockTimeout(key string, startTime time.Time) {
	if b.LockTimeout > 0 && time.Since(startTime) > b.LockTimeout {
		b.LockTimedout(key)
	}
}

func (b *BaseKvdb) LockTimedout(key string) {
	b.FatalCb("Lock %s hold timeout triggered", key)
}

// watchUpdate refers to an update to this kvdb
type watchUpdate struct {
	// key is the key that was updated
	key string
	// kvp is the key-value that was updated
	kvp *kvdb.KVPair
	// err is any error on update
	err error
}

// WatchUpdateQueue is a producer consumer queue.
type WatchUpdateQueue interface {
	// Enqueue will enqueue an update. It is non-blocking.
	Enqueue(key string, kvp *kvdb.KVPair, err error)
	// Dequeue will either return an element from front of the queue or
	// will block until element becomes available
	Dequeue() (string, *kvdb.KVPair, error)
}

// watchQueue implements WatchUpdateQueue interface for watchUpdates
type watchQueue struct {
	// updates is the list of updates
	updates *list.List
	// m is the mutex to protect updates
	m *sync.Mutex
	// cv is used to coordinate the producer-consumer threads
	cv *sync.Cond
}

// NewWatchUpdateQueue returns WatchUpdateQueue
func NewWatchUpdateQueue() WatchUpdateQueue {
	mtx := &sync.Mutex{}
	return &watchQueue{
		m:       mtx,
		cv:      sync.NewCond(mtx),
		updates: list.New()}
}

func (w *watchQueue) Dequeue() (string, *kvdb.KVPair, error) {
	w.m.Lock()
	for {
		if w.updates.Len() > 0 {
			el := w.updates.Front()
			w.updates.Remove(el)
			w.m.Unlock()
			update := el.Value.(*watchUpdate)
			return update.key, update.kvp, update.err
		}
		w.cv.Wait()
	}
}

// Enqueue enqueues and never blocks
func (w *watchQueue) Enqueue(key string, kvp *kvdb.KVPair, err error) {
	w.m.Lock()
	w.updates.PushBack(&watchUpdate{key: key, kvp: kvp, err: err})
	w.cv.Signal()
	w.m.Unlock()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"text/tabwriter"
	"time"

	"encoding/json"

	"github.com/Microsoft/hcsshim/internal/appargs"
	"github.com/urfave/cli"
)

const formatOptions = `table or json`

// containerState represents the platform agnostic pieces relating to a
// running container's status and state
type containerState struct {
	// Version is the OCI version for the container
	Version string `json:"ociVersion"`
	// ID is the container ID
	ID string `json:"id"`
	// InitProcessPid is the init process id in the parent namespace
	InitProcessPid int `json:"pid"`
	// Status is the current status of the container, running, paused, ...
	Status string `json:"status"`
	// Bundle is the path on the filesystem to the bundle
	Bundle string `json:"bundle"`
	// Rootfs is a path to a directory containing the container's root filesystem.
	Rootfs string `json:"rootfs"`
	// Created is the unix timestamp for the creation time of the container in UTC
	Created time.Time `json:"created"`
	// Annotations is the user defined annotations added to the config.
	Annotations map[string]string `json:"annotations,omitempty"`
	// The owner of the state directory (the owner of the container).
	Owner string `json:"owner"`
}

var listCommand = cli.Command{
	Name:  "list",
	Usage: "lists containers started by runc with the given root",
	ArgsUsage: `

Where the given root is specified via the global option "--root"
(default: "/run/runc").

EXAMPLE 1:
To list containers created via the default "--root":
       # runc list

EXAMPLE 2:
To list containers created using a non-default value for "--root":
       # runc --root value list`,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "format, f",
			Value: "table",
			Usage: `select one of: ` + formatOptions,
		},
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "display only container IDs",
		},
	},
	Before: appargs.Validate(),
	Action: func(context *cli.Context) error {
		s, err := getContainers(context)
		if err != nil {
			return err
		}

		if context.Bool("quiet") {
			for _, item := range s {
				fmt.Println(item.ID)
			}
			return nil
		}

		switch context.String("format") {
		case "table":
			w := tabwriter.NewWriter(os.Stdout, 12, 1, 3, ' ', 0)
			fmt.Fprint(w, "ID\tPID\tSTATUS\tBUNDLE\tCREATED\tOWNER\n")
			for _, item := range s {
				fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\t%s\n",
					item.ID,
					item.InitProcessPid,
					item.Status,
					item.Bundle,
					item.Created.Format(time.RFC3339Nano),
					item.Owner)
			}
			if err := w.Flush(); err != nil {
				return err
			}
		case "json":
			if err := json.NewEncoder(os.Stdout).Encode(s); err != nil {
				return err
			}
		default:
			return fmt.Errorf("invalid format option")
		}
		return nil
	},
}

func getContainers(context *cli.Context) ([]containerState, error) {
	ids, err := stateKey.Enumerate()
	if err != nil {
		return nil, err
	}

	var s []containerState
	for _, id := range ids {
		c, err := getContainer(id, false)
		if err != nil {
			fmt.Fprintf(os.Stderr, "reading state for %s: %v\n", id, err)
			continue
		}
		status, err := c.Status()
		if err != nil {
			fmt.Fprintf(os.Stderr, "reading status for %s: %v\n", id, err)
		}

		s = append(s, containerState{
			ID:             id,
			Version:        c.Spec.Version,
			InitProcessPid: c.ShimPid,
			Status:         string(status),
			Bundle:         c.Bundle,
			Rootfs:         c.Rootfs,
			Created:        c.Created,
			Annotations:    c.Spec.Annotations,
		})
	}
	return s, nil
}
<commit_msg>removed runc reference from list<commit_after>package main

import (
	"fmt"
	"os"
	"text/tabwriter"
	"time"

	"encoding/json"

	"github.com/Microsoft/hcsshim/internal/appargs"
	"github.com/urfave/cli"
)

const formatOptions = `table or json`

// containerState represents the platform agnostic pieces relating to a
// running container's status and state
type containerState struct {
	// Version is the OCI version for the container
	Version string `json:"ociVersion"`
	// ID is the container ID
	ID string `json:"id"`
	// InitProcessPid is the init process id in the parent namespace
	InitProcessPid int `json:"pid"`
	// Status is the current status of the container, running, paused, ...
	Status string `json:"status"`
	// Bundle is the path on the filesystem to the bundle
	Bundle string `json:"bundle"`
	// Rootfs is a path to a directory containing the container's root filesystem.
	Rootfs string `json:"rootfs"`
	// Created is the unix timestamp for the creation time of the container in UTC
	Created time.Time `json:"created"`
	// Annotations is the user defined annotations added to the config.
	Annotations map[string]string `json:"annotations,omitempty"`
	// The owner of the state directory (the owner of the container).
	Owner string `json:"owner"`
}

var listCommand = cli.Command{
	Name:  "list",
	Usage: "lists containers started by runhcs with the given root",
	ArgsUsage: `

Where the given root is specified via the global option "--root"
(default: "/run/runhcs").

EXAMPLE 1:
To list containers created via the default "--root":
       # runhcs list

EXAMPLE 2:
To list containers created using a non-default value for "--root":
       # runhcs --root value list`,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "format, f",
			Value: "table",
			Usage: `select one of: ` + formatOptions,
		},
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "display only container IDs",
		},
	},
	Before: appargs.Validate(),
	Action: func(context *cli.Context) error {
		s, err := getContainers(context)
		if err != nil {
			return err
		}

		if context.Bool("quiet") {
			for _, item := range s {
				fmt.Println(item.ID)
			}
			return nil
		}

		switch context.String("format") {
		case "table":
			w := tabwriter.NewWriter(os.Stdout, 12, 1, 3, ' ', 0)
			fmt.Fprint(w, "ID\tPID\tSTATUS\tBUNDLE\tCREATED\tOWNER\n")
			for _, item := range s {
				fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\t%s\n",
					item.ID,
					item.InitProcessPid,
					item.Status,
					item.Bundle,
					item.Created.Format(time.RFC3339Nano),
					item.Owner)
			}
			if err := w.Flush(); err != nil {
				return err
			}
		case "json":
			if err := json.NewEncoder(os.Stdout).Encode(s); err != nil {
				return err
			}
		default:
			return fmt.Errorf("invalid format option")
		}
		return nil
	},
}

func getContainers(context *cli.Context) ([]containerState, error) {
	ids, err := stateKey.Enumerate()
	if err != nil {
		return nil, err
	}

	var s []containerState
	for _, id := range ids {
		c, err := getContainer(id, false)
		if err != nil {
			fmt.Fprintf(os.Stderr, "reading state for %s: %v\n", id, err)
			continue
		}
		status, err := c.Status()
		if err != nil {
			fmt.Fprintf(os.Stderr, "reading status for %s: %v\n", id, err)
		}

		s = append(s, containerState{
			ID:             id,
			Version:        c.Spec.Version,
			InitProcessPid: c.ShimPid,
			Status:         string(status),
			Bundle:         c.Bundle,
			Rootfs:         c.Rootfs,
			Created:        c.Created,
			Annotations:    c.Spec.Annotations,
		})
	}
	return s, nil
}
<|endoftext|>


<commit_before>// shade presents a fuse filesystem interface.
package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"path"
	"time"

	_ "expvar"

	"bazil.org/fuse"

	"github.com/asjoyner/shade"
	"github.com/asjoyner/shade/config"
	"github.com/asjoyner/shade/drive"
	"github.com/asjoyner/shade/fusefs"

	_ "github.com/asjoyner/shade/drive/amazon"
	_ "github.com/asjoyner/shade/drive/cache"
	_ "github.com/asjoyner/shade/drive/encrypt"
	_ "github.com/asjoyner/shade/drive/google"
	_ "github.com/asjoyner/shade/drive/local"
	_ "github.com/asjoyner/shade/drive/memory"
)

var (
	defaultConfig = path.Join(shade.ConfigDir(), "config.json")

	readOnly   = flag.Bool("readonly", false, "Mount the filesystem read only.")
	allowOther = flag.Bool("allow_other", false, "If other users are allowed to view the mounted filesystem.")
	configFile = flag.String("config", defaultConfig, fmt.Sprintf("The shade config file. Defaults to %q", defaultConfig))
	treeDebug  = flag.Bool("treeDebug", false, "Print Node tree debugging traces")
	port       = flag.Int("port", 33247, "HTTP port to listen on (exposes debug and monitoring handlers).")
)

func main() {
	flag.Usage = usage
	flag.Parse()

	if flag.NArg() != 1 {
		usage()
		os.Exit(2)
	}

	// initialize the webserver
	go func() { log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) }()

	// read in the config
	config, err := config.Read(*configFile)
	if err != nil {
		log.Fatalf("could not read configuration: %s\n", err)
	}

	// initialize client
	client, err := drive.NewClient(config)
	if err != nil {
		log.Fatalf("could not initialize client: %s\n", err)
	}

	// Setup fuse FS
	conn, err := mountFuse(flag.Arg(0))
	if err != nil {
		log.Fatalf("failed to mount: %s", err)
	}
	fmt.Printf("Mounting Shade FuseFS at %s...\n", flag.Arg(0))

	if err := serviceFuse(conn, client); err != nil {
		log.Fatalf("failed to service mount: %s", err)
	}

	return
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "  %s <mountpoint>\n", os.Args[0])
	flag.PrintDefaults()
}

func mountFuse(mountPoint string) (*fuse.Conn, error) {
	if err := sanityCheck(mountPoint); err != nil {
		return nil, fmt.Errorf("sanityCheck failed: %s", err)
	}

	options := []fuse.MountOption{
		fuse.FSName("Shade"),
		//fuse.Subtype(""),
		//fuse.VolumeName(<iterate clients?>),
	}

	if *allowOther {
		options = append(options, fuse.AllowOther())
	}
	if *readOnly {
		options = append(options, fuse.ReadOnly())
	}
	options = append(options, fuse.NoAppleDouble())
	c, err := fuse.Mount(mountPoint, options...)
	if err != nil {
		fmt.Println("Is the mount point busy?")
		return nil, err
	}

	// Trap control-c (sig INT) and unmount
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)
	go func() {
		for range sig {
			if err := fuse.Unmount(mountPoint); err != nil {
				log.Printf("fuse.Unmount failed: %v", err)
			}
		}
	}()

	return c, nil
}

// serviceFuse initializes fusefs, the shade implementation of a fuse file
// server, and services requests from the fuse kernel filesystem until it is
// unmounted.
func serviceFuse(conn *fuse.Conn, client drive.Client) error {
	refresh := time.NewTicker(5 * time.Minute)
	ffs, err := fusefs.New(client, conn, refresh)
	if err != nil {
		return fmt.Errorf("fuse server initialization failed: %s", err)
	}
	if *treeDebug {
		ffs.TreeDebug()
	}

	go func() {
		<-conn.Ready // block until the fuse FS is mounted and ready
		if err := conn.MountError; err != nil {
			fmt.Printf("mounting fuse fs failed: %s", err)
		}
		fmt.Println("Shade FuseFS mounted and ready to serve.")
	}()

	err = ffs.Serve()
	if err != nil {
		return fmt.Errorf("serving fuse connection failed: %s", err)
	}
	return nil

}

func sanityCheck(mountPoint string) error {
	fileInfo, err := os.Stat(mountPoint)
	if os.IsNotExist(err) {
		if err := os.MkdirAll(mountPoint, 0777); err != nil {
			return fmt.Errorf("mountpoint does not exist, could not create it")
		}
		return nil
	}
	if err != nil {
		return fmt.Errorf("error stat()ing mountpoint: %s", err)
	}
	if !fileInfo.IsDir() {
		return fmt.Errorf("the mountpoint is not a directory")
	}
	return nil
}
<commit_msg>Add /refresh HTTP handler to refresh Fuse FS<commit_after>// shade presents a fuse filesystem interface.
package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"path"
	"time"

	_ "expvar"

	"bazil.org/fuse"

	"github.com/asjoyner/shade"
	"github.com/asjoyner/shade/config"
	"github.com/asjoyner/shade/drive"
	"github.com/asjoyner/shade/fusefs"

	_ "github.com/asjoyner/shade/drive/amazon"
	_ "github.com/asjoyner/shade/drive/cache"
	_ "github.com/asjoyner/shade/drive/encrypt"
	_ "github.com/asjoyner/shade/drive/google"
	_ "github.com/asjoyner/shade/drive/local"
	_ "github.com/asjoyner/shade/drive/memory"
)

var (
	defaultConfig = path.Join(shade.ConfigDir(), "config.json")

	readOnly   = flag.Bool("readonly", false, "Mount the filesystem read only.")
	allowOther = flag.Bool("allow_other", false, "If other users are allowed to view the mounted filesystem.")
	configFile = flag.String("config", defaultConfig, fmt.Sprintf("The shade config file. Defaults to %q", defaultConfig))
	treeDebug  = flag.Bool("treeDebug", false, "Print Node tree debugging traces")
	port       = flag.Int("port", 33247, "HTTP port to listen on (exposes debug and monitoring handlers).")
)

func main() {
	flag.Usage = usage
	flag.Parse()

	if flag.NArg() != 1 {
		usage()
		os.Exit(2)
	}

	// initialize the webserver
	go func() { log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)) }()

	// read in the config
	config, err := config.Read(*configFile)
	if err != nil {
		log.Fatalf("could not read configuration: %s\n", err)
	}

	// initialize client
	client, err := drive.NewClient(config)
	if err != nil {
		log.Fatalf("could not initialize client: %s\n", err)
	}

	// Setup fuse FS
	conn, err := mountFuse(flag.Arg(0))
	if err != nil {
		log.Fatalf("failed to mount: %s", err)
	}
	fmt.Printf("Mounting Shade FuseFS at %s...\n", flag.Arg(0))

	if err := serviceFuse(conn, client); err != nil {
		log.Fatalf("failed to service mount: %s", err)
	}

	return
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "  %s <mountpoint>\n", os.Args[0])
	flag.PrintDefaults()
}

func mountFuse(mountPoint string) (*fuse.Conn, error) {
	if err := sanityCheck(mountPoint); err != nil {
		return nil, fmt.Errorf("sanityCheck failed: %s", err)
	}

	options := []fuse.MountOption{
		fuse.FSName("Shade"),
		//fuse.Subtype(""),
		//fuse.VolumeName(<iterate clients?>),
	}

	if *allowOther {
		options = append(options, fuse.AllowOther())
	}
	if *readOnly {
		options = append(options, fuse.ReadOnly())
	}
	options = append(options, fuse.NoAppleDouble())
	c, err := fuse.Mount(mountPoint, options...)
	if err != nil {
		fmt.Println("Is the mount point busy?")
		return nil, err
	}

	// Trap control-c (sig INT) and unmount
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)
	go func() {
		for range sig {
			if err := fuse.Unmount(mountPoint); err != nil {
				log.Printf("fuse.Unmount failed: %v", err)
			}
		}
	}()

	return c, nil
}

// serviceFuse initializes fusefs, the shade implementation of a fuse file
// server, and services requests from the fuse kernel filesystem until it is
// unmounted.
func serviceFuse(conn *fuse.Conn, client drive.Client) error {
	refresh := time.NewTicker(5 * time.Minute)
	ffs, err := fusefs.New(client, conn, refresh)
	if err != nil {
		return fmt.Errorf("fuse server initialization failed: %s", err)
	}
	if *treeDebug {
		ffs.TreeDebug()
	}

	http.HandleFunc("/refresh", func(w http.ResponseWriter, r *http.Request) {
		ffs.Refresh()
		fmt.Fprintf(w, "Ok")
	})

	go func() {
		<-conn.Ready // block until the fuse FS is mounted and ready
		if err := conn.MountError; err != nil {
			fmt.Printf("mounting fuse fs failed: %s", err)
		}
		fmt.Println("Shade FuseFS mounted and ready to serve.")
	}()

	err = ffs.Serve()
	if err != nil {
		return fmt.Errorf("serving fuse connection failed: %s", err)
	}
	return nil

}

func sanityCheck(mountPoint string) error {
	fileInfo, err := os.Stat(mountPoint)
	if os.IsNotExist(err) {
		if err := os.MkdirAll(mountPoint, 0777); err != nil {
			return fmt.Errorf("mountpoint does not exist, could not create it")
		}
		return nil
	}
	if err != nil {
		return fmt.Errorf("error stat()ing mountpoint: %s", err)
	}
	if !fileInfo.IsDir() {
		return fmt.Errorf("the mountpoint is not a directory")
	}
	return nil
}
<|endoftext|>