<commit_before>package cli

import (
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	kubecmd "k8s.io/kubernetes/pkg/kubectl/cmd"

	"github.com/openshift/origin/pkg/cmd/cli/cmd"
	"github.com/openshift/origin/pkg/cmd/cli/policy"
	"github.com/openshift/origin/pkg/cmd/cli/secrets"
	"github.com/openshift/origin/pkg/cmd/flagtypes"
	"github.com/openshift/origin/pkg/cmd/templates"
	cmdutil "github.com/openshift/origin/pkg/cmd/util"
	"github.com/openshift/origin/pkg/cmd/util/clientcmd"
	"github.com/openshift/origin/pkg/version"
)

const cliLong = `
Developer and Administrator Client

This client exposes commands for managing your applications, as well as lower level
tools to interact with each component of your system.

To create a new application, you can use the example app source. Login to your server and then
run new-app:

  $ %[1]s login
  $ %[1]s new-app openshift/ruby-20-centos7~https://github.com/openshift/ruby-hello-world.git

This will create an application based on the Docker image 'openshift/ruby-20-centos7' that builds
the source code at 'github.com/openshift/ruby-hello-world.git'. A build will start automatically and
a deployment will start as soon as the build finishes.

Once your application is deployed, use the status, get, and describe commands to see more about
the created components:

  $ %[1]s status
  $ %[1]s describe deploymentconfig ruby-hello-world
  $ %[1]s get pods

You'll be able to view the deployed application on the IP and port of the service that new-app
created for you.

You can easily switch between multiple projects using '%[1]s project <projectname>'.`

func NewCommandCLI(name, fullName string, in io.Reader, out, errout io.Writer) *cobra.Command {
	// Main command
	cmds := &cobra.Command{
		Use:   name,
		Short: "Command line tools for managing applications",
		Long:  fmt.Sprintf(cliLong, fullName),
		Run:   cmdutil.DefaultSubCommandRun(out),
		BashCompletionFunction: bashCompletionFunc,
	}

	f := clientcmd.New(cmds.PersistentFlags())

	loginCmd := cmd.NewCmdLogin(fullName, f, in, out)
	groups := templates.CommandGroups{
		{
			Message: "Basic Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdTypes(fullName, f, out),
				loginCmd,
				cmd.NewCmdRequestProject("new-project", fullName+" new-project", fullName+" login", fullName+" project", f, out),
				cmd.NewCmdNewApplication(fullName, f, out),
				cmd.NewCmdStatus(cmd.StatusRecommendedName, fullName+" "+cmd.StatusRecommendedName, f, out),
				cmd.NewCmdProject(fullName+" project", f, out),
			},
		},
		{
			Message: "Build and Deploy Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdStartBuild(fullName, f, out),
				cmd.NewCmdBuildLogs(fullName, f, out),
				cmd.NewCmdDeploy(fullName, f, out),
				cmd.NewCmdRollback(fullName, f, out),
				cmd.NewCmdNewBuild(fullName, f, in, out),
				cmd.NewCmdCancelBuild(fullName, f, out),
				cmd.NewCmdImportImage(fullName, f, out),
				cmd.NewCmdScale(fullName, f, out),
				cmd.NewCmdTag(fullName, f, out),
			},
		},
		{
			Message: "Application Modification Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdGet(fullName, f, out),
				cmd.NewCmdDescribe(fullName, f, out),
				cmd.NewCmdEdit(fullName, f, out),
				cmd.NewCmdEnv(fullName, f, in, out),
				cmd.NewCmdVolume(fullName, f, out, errout),
				cmd.NewCmdLabel(fullName, f, out),
				cmd.NewCmdAnnotate(fullName, f, out),
				cmd.NewCmdExpose(fullName, f, out),
				cmd.NewCmdStop(fullName, f, out),
				cmd.NewCmdDelete(fullName, f, out),
			},
		},
		{
			Message: "Troubleshooting and Debugging Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdLogs(fullName, f, out),
				cmd.NewCmdRsh(fullName, f, in, out, errout),
				cmd.NewCmdExec(fullName, f, in, out, errout),
				cmd.NewCmdPortForward(fullName, f),
				cmd.NewCmdProxy(fullName, f, out),
			},
		},
		{
			Message: "Advanced Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdCreate(fullName, f, out),
				cmd.NewCmdReplace(fullName, f, out),
				cmd.NewCmdPatch(fullName, f, out),
				cmd.NewCmdProcess(fullName, f, out),
				cmd.NewCmdExport(fullName, f, in, out),
				cmd.NewCmdRun(fullName, f, in, out, errout),
				cmd.NewCmdAttach(fullName, f, in, out, errout),
				policy.NewCmdPolicy(policy.PolicyRecommendedName, fullName+" "+policy.PolicyRecommendedName, f, out),
				secrets.NewCmdSecrets(secrets.SecretsRecommendedName, fullName+" "+secrets.SecretsRecommendedName, f, in, out, fullName+" edit"),
			},
		},
		{
			Message: "Settings Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdLogout("logout", fullName+" logout", fullName+" login", f, in, out),
				cmd.NewCmdConfig(fullName, "config"),
				cmd.NewCmdWhoAmI(cmd.WhoAmIRecommendedCommandName, fullName+" "+cmd.WhoAmIRecommendedCommandName, f, out),
			},
		},
	}
	groups.Add(cmds)
	templates.ActsAsRootCommand(cmds, groups...).
		ExposeFlags(loginCmd, "certificate-authority", "insecure-skip-tls-verify")

	if name == fullName {
		cmds.AddCommand(version.NewVersionCommand(fullName))
	}
	cmds.AddCommand(cmd.NewCmdOptions(out))

	return cmds
}

// NewCmdKubectl provides exactly the functionality from Kubernetes,
// but with support for OpenShift resources
func NewCmdKubectl(name string, out io.Writer) *cobra.Command {
	flags := pflag.NewFlagSet("", pflag.ContinueOnError)
	f := clientcmd.New(flags)
	cmds := kubecmd.NewKubectlCommand(f.Factory, os.Stdin, out, os.Stderr)
	cmds.Aliases = []string{"kubectl"}
	cmds.Use = name
	cmds.Short = "Kubernetes cluster management via kubectl"
	flags.VisitAll(func(flag *pflag.Flag) {
		if f := cmds.PersistentFlags().Lookup(flag.Name); f == nil {
			cmds.PersistentFlags().AddFlag(flag)
		} else {
			glog.V(5).Infof("already registered flag %s", flag.Name)
		}
	})
	cmds.PersistentFlags().Var(flags.Lookup("config").Value, "kubeconfig", "Specify a kubeconfig file to define the configuration")
	templates.ActsAsRootCommand(cmds)
	cmds.AddCommand(cmd.NewCmdOptions(out))
	return cmds
}

// CommandFor returns the appropriate command for this base name,
// or the OpenShift CLI command.
func CommandFor(basename string) *cobra.Command {
	var cmd *cobra.Command

	in, out, errout := os.Stdin, os.Stdout, os.Stderr

	// Make case-insensitive and strip executable suffix if present
	if runtime.GOOS == "windows" {
		basename = strings.ToLower(basename)
		basename = strings.TrimSuffix(basename, ".exe")
	}

	switch basename {
	case "kubectl":
		cmd = NewCmdKubectl(basename, out)
	default:
		cmd = NewCommandCLI(basename, basename, in, out, errout)
	}

	if cmd.UsageFunc() == nil {
		templates.ActsAsRootCommand(cmd)
	}
	flagtypes.GLog(cmd.PersistentFlags())

	return cmd
}
<commit_msg>change show-all default to true<commit_after>package cli

import (
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	kubecmd "k8s.io/kubernetes/pkg/kubectl/cmd"

	"github.com/openshift/origin/pkg/cmd/cli/cmd"
	"github.com/openshift/origin/pkg/cmd/cli/policy"
	"github.com/openshift/origin/pkg/cmd/cli/secrets"
	"github.com/openshift/origin/pkg/cmd/flagtypes"
	"github.com/openshift/origin/pkg/cmd/templates"
	cmdutil "github.com/openshift/origin/pkg/cmd/util"
	"github.com/openshift/origin/pkg/cmd/util/clientcmd"
	"github.com/openshift/origin/pkg/version"
)

const cliLong = `
Developer and Administrator Client

This client exposes commands for managing your applications, as well as lower level
tools to interact with each component of your system.

To create a new application, you can use the example app source. Login to your server and then
run new-app:

  $ %[1]s login
  $ %[1]s new-app openshift/ruby-20-centos7~https://github.com/openshift/ruby-hello-world.git

This will create an application based on the Docker image 'openshift/ruby-20-centos7' that builds
the source code at 'github.com/openshift/ruby-hello-world.git'. A build will start automatically and
a deployment will start as soon as the build finishes.

Once your application is deployed, use the status, get, and describe commands to see more about
the created components:

  $ %[1]s status
  $ %[1]s describe deploymentconfig ruby-hello-world
  $ %[1]s get pods

You'll be able to view the deployed application on the IP and port of the service that new-app
created for you.

You can easily switch between multiple projects using '%[1]s project <projectname>'.`

func NewCommandCLI(name, fullName string, in io.Reader, out, errout io.Writer) *cobra.Command {
	// Main command
	cmds := &cobra.Command{
		Use:   name,
		Short: "Command line tools for managing applications",
		Long:  fmt.Sprintf(cliLong, fullName),
		Run:   cmdutil.DefaultSubCommandRun(out),
		BashCompletionFunction: bashCompletionFunc,
	}

	f := clientcmd.New(cmds.PersistentFlags())

	loginCmd := cmd.NewCmdLogin(fullName, f, in, out)
	groups := templates.CommandGroups{
		{
			Message: "Basic Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdTypes(fullName, f, out),
				loginCmd,
				cmd.NewCmdRequestProject("new-project", fullName+" new-project", fullName+" login", fullName+" project", f, out),
				cmd.NewCmdNewApplication(fullName, f, out),
				cmd.NewCmdStatus(cmd.StatusRecommendedName, fullName+" "+cmd.StatusRecommendedName, f, out),
				cmd.NewCmdProject(fullName+" project", f, out),
			},
		},
		{
			Message: "Build and Deploy Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdStartBuild(fullName, f, out),
				cmd.NewCmdBuildLogs(fullName, f, out),
				cmd.NewCmdDeploy(fullName, f, out),
				cmd.NewCmdRollback(fullName, f, out),
				cmd.NewCmdNewBuild(fullName, f, in, out),
				cmd.NewCmdCancelBuild(fullName, f, out),
				cmd.NewCmdImportImage(fullName, f, out),
				cmd.NewCmdScale(fullName, f, out),
				cmd.NewCmdTag(fullName, f, out),
			},
		},
		{
			Message: "Application Modification Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdGet(fullName, f, out),
				cmd.NewCmdDescribe(fullName, f, out),
				cmd.NewCmdEdit(fullName, f, out),
				cmd.NewCmdEnv(fullName, f, in, out),
				cmd.NewCmdVolume(fullName, f, out, errout),
				cmd.NewCmdLabel(fullName, f, out),
				cmd.NewCmdAnnotate(fullName, f, out),
				cmd.NewCmdExpose(fullName, f, out),
				cmd.NewCmdStop(fullName, f, out),
				cmd.NewCmdDelete(fullName, f, out),
			},
		},
		{
			Message: "Troubleshooting and Debugging Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdLogs(fullName, f, out),
				cmd.NewCmdRsh(fullName, f, in, out, errout),
				cmd.NewCmdExec(fullName, f, in, out, errout),
				cmd.NewCmdPortForward(fullName, f),
				cmd.NewCmdProxy(fullName, f, out),
			},
		},
		{
			Message: "Advanced Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdCreate(fullName, f, out),
				cmd.NewCmdReplace(fullName, f, out),
				cmd.NewCmdPatch(fullName, f, out),
				cmd.NewCmdProcess(fullName, f, out),
				cmd.NewCmdExport(fullName, f, in, out),
				cmd.NewCmdRun(fullName, f, in, out, errout),
				cmd.NewCmdAttach(fullName, f, in, out, errout),
				policy.NewCmdPolicy(policy.PolicyRecommendedName, fullName+" "+policy.PolicyRecommendedName, f, out),
				secrets.NewCmdSecrets(secrets.SecretsRecommendedName, fullName+" "+secrets.SecretsRecommendedName, f, in, out, fullName+" edit"),
			},
		},
		{
			Message: "Settings Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdLogout("logout", fullName+" logout", fullName+" login", f, in, out),
				cmd.NewCmdConfig(fullName, "config"),
				cmd.NewCmdWhoAmI(cmd.WhoAmIRecommendedCommandName, fullName+" "+cmd.WhoAmIRecommendedCommandName, f, out),
			},
		},
	}
	groups.Add(cmds)
	changeSharedFlagDefaults(cmds)
	templates.ActsAsRootCommand(cmds, groups...).
		ExposeFlags(loginCmd, "certificate-authority", "insecure-skip-tls-verify")

	if name == fullName {
		cmds.AddCommand(version.NewVersionCommand(fullName))
	}
	cmds.AddCommand(cmd.NewCmdOptions(out))

	return cmds
}

// changeSharedFlagDefaults changes values of shared flags that we disagree with.  This can't be done in godep code because
// that would change behavior in our `kubectl` symlink. Defend each change.
// 1. show-all - the most interesting pods are terminated/failed pods.  We don't want to exclude them from printing
func changeSharedFlagDefaults(rootCmd *cobra.Command) {
	cmds := []*cobra.Command{rootCmd}

	for i := 0; i < len(cmds); i++ {
		currCmd := cmds[i]
		cmds = append(cmds, currCmd.Commands()...)

		if showAllFlag := currCmd.Flags().Lookup("show-all"); showAllFlag != nil {
			showAllFlag.DefValue = "true"
			showAllFlag.Value.Set("true")
			showAllFlag.Changed = false
			showAllFlag.Usage = "When printing, show all resources (false means hide terminated pods.)"
		}
	}
}

// NewCmdKubectl provides exactly the functionality from Kubernetes,
// but with support for OpenShift resources
func NewCmdKubectl(name string, out io.Writer) *cobra.Command {
	flags := pflag.NewFlagSet("", pflag.ContinueOnError)
	f := clientcmd.New(flags)
	cmds := kubecmd.NewKubectlCommand(f.Factory, os.Stdin, out, os.Stderr)
	cmds.Aliases = []string{"kubectl"}
	cmds.Use = name
	cmds.Short = "Kubernetes cluster management via kubectl"
	flags.VisitAll(func(flag *pflag.Flag) {
		if f := cmds.PersistentFlags().Lookup(flag.Name); f == nil {
			cmds.PersistentFlags().AddFlag(flag)
		} else {
			glog.V(5).Infof("already registered flag %s", flag.Name)
		}
	})
	cmds.PersistentFlags().Var(flags.Lookup("config").Value, "kubeconfig", "Specify a kubeconfig file to define the configuration")
	templates.ActsAsRootCommand(cmds)
	cmds.AddCommand(cmd.NewCmdOptions(out))
	return cmds
}

// CommandFor returns the appropriate command for this base name,
// or the OpenShift CLI command.
func CommandFor(basename string) *cobra.Command {
	var cmd *cobra.Command

	in, out, errout := os.Stdin, os.Stdout, os.Stderr

	// Make case-insensitive and strip executable suffix if present
	if runtime.GOOS == "windows" {
		basename = strings.ToLower(basename)
		basename = strings.TrimSuffix(basename, ".exe")
	}

	switch basename {
	case "kubectl":
		cmd = NewCmdKubectl(basename, out)
	default:
		cmd = NewCommandCLI(basename, basename, in, out, errout)
	}

	if cmd.UsageFunc() == nil {
		templates.ActsAsRootCommand(cmd)
	}
	flagtypes.GLog(cmd.PersistentFlags())

	return cmd
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"io"
	"io/ioutil"
	"path"
	"reflect"
	"strings"

	kyaml "github.com/ghodss/yaml" // we intentionally use a different yaml package here for Kubernetes objects because gopkg.in/yaml.v2 is not meant to serialize k8s objects because of UnmarshalJSON/UnmarshalYAML and `json:""`/`yaml:""` dichotomy resulting in panic when used
	cmdutil "github.com/redhat-developer/opencompose/pkg/cmd/util"
	"github.com/redhat-developer/opencompose/pkg/encoding"
	"github.com/redhat-developer/opencompose/pkg/object"
	"github.com/redhat-developer/opencompose/pkg/transform"
	"github.com/redhat-developer/opencompose/pkg/transform/kubernetes"
	"github.com/redhat-developer/opencompose/pkg/transform/openshift"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"k8s.io/client-go/pkg/api"
	"k8s.io/client-go/pkg/api/meta"
	"k8s.io/client-go/pkg/runtime"
)

var (
	convertExample = `  # Converts file
  opencompose convert -f opencompose.yaml`
)

func NewCmdConvert(v *viper.Viper, out, outerr io.Writer) *cobra.Command {
	cmd := &cobra.Command{
		Use:     "convert",
		Short:   "Converts OpenCompose files into Kubernetes (and OpenShift) artifacts",
		Long:    "Converts OpenCompose files into Kubernetes (and OpenShift) artifacts",
		Example: convertExample,
		RunE: func(cmd *cobra.Command, args []string) error {
			return RunConvert(v, cmd, out, outerr)
		},
		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
			if cmd.Parent().PersistentPreRunE != nil {
				if err := cmd.Parent().PersistentPreRunE(cmd, args); err != nil {
					return err
				}
			}

			// We have to bind Viper in Run because there is only one instance to avoid collisions between subcommands
			cmdutil.AddIOFlagsViper(v, cmd)

			return nil
		},
	}

	cmdutil.AddIOFlags(cmd)

	return cmd
}

func GetValidatedObject(v *viper.Viper, cmd *cobra.Command, out, outerr io.Writer) (*object.OpenCompose, error) {
	files := v.GetStringSlice(cmdutil.Flag_File_Key)
	if len(files) < 1 {
		return nil, cmdutil.UsageError(cmd, "there has to be at least one file")
	}

	var ocObjects []*object.OpenCompose
	for _, file := range files {
		data, err := ioutil.ReadFile(file)
		if err != nil {
			return nil, fmt.Errorf("unable to read file '%s': %s", file, err)
		}

		decoder, err := encoding.GetDecoderFor(data)
		if err != nil {
			return nil, fmt.Errorf("could not find decoder for file '%s': %s", file, err)
		}

		o, err := decoder.Decode(data)
		if err != nil {
			return nil, fmt.Errorf("could not unmarshal data for file '%s': %s", file, err)
		}

		ocObjects = append(ocObjects, o)
	}

	// FIXME: implement merging OpenCompose obejcts
	openCompose := ocObjects[0]

	openCompose.Validate()

	return openCompose, nil
}

func RunConvert(v *viper.Viper, cmd *cobra.Command, out, outerr io.Writer) error {
	o, err := GetValidatedObject(v, cmd, out, outerr)
	if err != nil {
		return err
	}

	var transformer transform.Transformer
	distro := v.GetString("distro")
	switch d := strings.ToLower(distro); d {
	case "kubernetes":
		transformer = &kubernetes.Transformer{}
	case "openshift":
		transformer = &openshift.Transformer{}
	default:
		return fmt.Errorf("unknown distro '%s'", distro)
	}

	runtimeObjects, err := transformer.Transform(o)
	if err != nil {
		return fmt.Errorf("transformation failed: %s", err)
	}

	var writeObject func(o runtime.Object, data []byte) error
	outputDir := v.GetString(cmdutil.Flag_OutputDir_Key)
	if outputDir == "-" {
		// don't use dir but write it to out (stdout)
		writeObject = func(o runtime.Object, data []byte) error {
			_, err := fmt.Fprintln(out, "---")
			if err != nil {
				return err
			}

			_, err = out.Write(data)
			return err
		}
	} else {
		// write files
		writeObject = func(o runtime.Object, data []byte) error {
			kind := o.GetObjectKind().GroupVersionKind().Kind
			m, ok := o.(meta.Object)
			if !ok {
				return fmt.Errorf("failed to cast runtime.object to meta.object (type is %s): %s", reflect.TypeOf(o).String(), err)
			}

			filename := fmt.Sprintf("%s-%s.yaml", strings.ToLower(kind), m.GetName())
			return ioutil.WriteFile(path.Join(outputDir, filename), data, 0644)
		}
	}

	for _, runtimeObject := range runtimeObjects {
		gvk, isUnversioned, err := api.Scheme.ObjectKind(runtimeObject)
		if err != nil {
			return fmt.Errorf("ConvertToVersion failed: %s", err)
		}
		if isUnversioned {
			return fmt.Errorf("ConvertToVersion failed: can't output unversioned type: %T", runtimeObject)
		}

		runtimeObject.GetObjectKind().SetGroupVersionKind(gvk)

		data, err := kyaml.Marshal(runtimeObject)
		if err != nil {
			return fmt.Errorf("failed to marshal object: %s", err)
		}

		err = writeObject(runtimeObject, data)
		if err != nil {
			return fmt.Errorf("failed to write object: %s", err)
		}
	}

	return nil
}
<commit_msg>add support for passing in input from STDIN<commit_after>package cmd

import (
	"fmt"
	"io"
	"io/ioutil"
	"path"
	"reflect"
	"strings"

	kyaml "github.com/ghodss/yaml" // we intentionally use a different yaml package here for Kubernetes objects because gopkg.in/yaml.v2 is not meant to serialize k8s objects because of UnmarshalJSON/UnmarshalYAML and `json:""`/`yaml:""` dichotomy resulting in panic when used
	cmdutil "github.com/redhat-developer/opencompose/pkg/cmd/util"
	"github.com/redhat-developer/opencompose/pkg/encoding"
	"github.com/redhat-developer/opencompose/pkg/object"
	"github.com/redhat-developer/opencompose/pkg/transform"
	"github.com/redhat-developer/opencompose/pkg/transform/kubernetes"
	"github.com/redhat-developer/opencompose/pkg/transform/openshift"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"k8s.io/client-go/pkg/api"
	"k8s.io/client-go/pkg/api/meta"
	"k8s.io/client-go/pkg/runtime"
	"os"
)

var (
	convertExample = `  # Converts file
  opencompose convert -f opencompose.yaml`
)

func NewCmdConvert(v *viper.Viper, out, outerr io.Writer) *cobra.Command {
	cmd := &cobra.Command{
		Use:     "convert",
		Short:   "Converts OpenCompose files into Kubernetes (and OpenShift) artifacts",
		Long:    "Converts OpenCompose files into Kubernetes (and OpenShift) artifacts",
		Example: convertExample,
		RunE: func(cmd *cobra.Command, args []string) error {
			return RunConvert(v, cmd, out, outerr)
		},
		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
			if cmd.Parent().PersistentPreRunE != nil {
				if err := cmd.Parent().PersistentPreRunE(cmd, args); err != nil {
					return err
				}
			}

			// We have to bind Viper in Run because there is only one instance to avoid collisions between subcommands
			cmdutil.AddIOFlagsViper(v, cmd)

			return nil
		},
	}

	cmdutil.AddIOFlags(cmd)

	return cmd
}

func GetValidatedObject(v *viper.Viper, cmd *cobra.Command, out, outerr io.Writer) (*object.OpenCompose, error) {
	files := v.GetStringSlice(cmdutil.Flag_File_Key)
	if len(files) < 1 {
		return nil, cmdutil.UsageError(cmd, "there has to be at least one file")
	}

	var ocObjects []*object.OpenCompose

	for _, file := range files {
		var data []byte
		var err error

		// Check if the passed resource points to STDIN or not
		if file == "-" {
			data, err = ioutil.ReadAll(os.Stdin)
			if err != nil {
				return nil, fmt.Errorf("unable to read from stdin: %s", err)
			}
		} else {
			data, err = ioutil.ReadFile(file)
			if err != nil {
				return nil, fmt.Errorf("unable to read file '%s': %s", file, err)
			}
		}
		decoder, err := encoding.GetDecoderFor(data)
		if err != nil {
			return nil, fmt.Errorf("could not find decoder for file '%s': %s", file, err)
		}

		o, err := decoder.Decode(data)
		if err != nil {
			return nil, fmt.Errorf("could not unmarshal data for file '%s': %s", file, err)
		}

		ocObjects = append(ocObjects, o)
	}

	// FIXME: implement merging OpenCompose obejcts
	openCompose := ocObjects[0]

	openCompose.Validate()

	return openCompose, nil
}

func RunConvert(v *viper.Viper, cmd *cobra.Command, out, outerr io.Writer) error {
	o, err := GetValidatedObject(v, cmd, out, outerr)
	if err != nil {
		return err
	}

	var transformer transform.Transformer
	distro := v.GetString("distro")
	switch d := strings.ToLower(distro); d {
	case "kubernetes":
		transformer = &kubernetes.Transformer{}
	case "openshift":
		transformer = &openshift.Transformer{}
	default:
		return fmt.Errorf("unknown distro '%s'", distro)
	}

	runtimeObjects, err := transformer.Transform(o)
	if err != nil {
		return fmt.Errorf("transformation failed: %s", err)
	}

	var writeObject func(o runtime.Object, data []byte) error
	outputDir := v.GetString(cmdutil.Flag_OutputDir_Key)
	if outputDir == "-" {
		// don't use dir but write it to out (stdout)
		writeObject = func(o runtime.Object, data []byte) error {
			_, err := fmt.Fprintln(out, "---")
			if err != nil {
				return err
			}

			_, err = out.Write(data)
			return err
		}
	} else {
		// write files
		writeObject = func(o runtime.Object, data []byte) error {
			kind := o.GetObjectKind().GroupVersionKind().Kind
			m, ok := o.(meta.Object)
			if !ok {
				return fmt.Errorf("failed to cast runtime.object to meta.object (type is %s): %s", reflect.TypeOf(o).String(), err)
			}

			filename := fmt.Sprintf("%s-%s.yaml", strings.ToLower(kind), m.GetName())
			return ioutil.WriteFile(path.Join(outputDir, filename), data, 0644)
		}
	}

	for _, runtimeObject := range runtimeObjects {
		gvk, isUnversioned, err := api.Scheme.ObjectKind(runtimeObject)
		if err != nil {
			return fmt.Errorf("ConvertToVersion failed: %s", err)
		}
		if isUnversioned {
			return fmt.Errorf("ConvertToVersion failed: can't output unversioned type: %T", runtimeObject)
		}

		runtimeObject.GetObjectKind().SetGroupVersionKind(gvk)

		data, err := kyaml.Marshal(runtimeObject)
		if err != nil {
			return fmt.Errorf("failed to marshal object: %s", err)
		}

		err = writeObject(runtimeObject, data)
		if err != nil {
			return fmt.Errorf("failed to write object: %s", err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016-2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Parts of this file are copied from https://github.com/google/netboot/blob/8e5c0d07937f8c1dea6e5f218b64f6b95c32ada3/pixiecore/dhcp.go

*/

package dhcp

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"strings"

	cnicurrent "github.com/containernetworking/cni/pkg/types/current"
	"github.com/golang/glog"
	"go.universe.tf/netboot/dhcp4"

	"github.com/Mirantis/virtlet/pkg/network"
)

const (
	serverPort = 67
	// option 121 is for static routes as defined in rfc3442
	classlessRouteOption = 121
)

var (
	defaultDNS = []byte{8, 8, 8, 8}
)

type Server struct {
	config   *network.ContainerSideNetwork
	listener *dhcp4.Conn
}

func NewServer(config *network.ContainerSideNetwork) *Server {
	return &Server{config: config}
}

func (s *Server) SetupListener(laddr string) error {
	if listener, err := dhcp4.NewConn(fmt.Sprintf("%s:%d", laddr, serverPort)); err != nil {
		return err
	} else {
		s.listener = listener
	}
	return nil
}

func (s *Server) Close() error {
	return s.listener.Close()
}

func (s *Server) Serve() error {
	for {
		pkt, intf, err := s.listener.RecvDHCP()
		if err != nil {
			return fmt.Errorf("receiving DHCP packet: %v", err)
		}
		if intf == nil {
			return fmt.Errorf("received DHCP packet with no interface information - please fill a bug to https://github.com/google/netboot")
		}
		glog.V(2).Infof("Received dhcp packet from: %s", pkt.HardwareAddr.String())

		serverIP, err := interfaceIP(intf)
		if err != nil {
			glog.Warningf("Want to respond to %s on %s, but couldn't get a source address: %s", pkt.HardwareAddr.String(), intf.Name, err)
			continue
		}

		var resp *dhcp4.Packet
		switch pkt.Type {
		case dhcp4.MsgDiscover:
			resp, err = s.offerDHCP(pkt, serverIP)
			if err != nil {
				glog.Warningf("Failed to construct DHCP offer for %s: %s", pkt.HardwareAddr.String(), err)
				continue
			}
		case dhcp4.MsgRequest:
			resp, err = s.ackDHCP(pkt, serverIP)
			if err != nil {
				glog.Warningf("Failed to construct DHCP ACK for %s: %s", pkt.HardwareAddr.String(), err)
				continue
			}
		default:
			glog.Warningf("Ignoring packet from %s: packet is %s", pkt.HardwareAddr.String(), pkt.Type.String())
			continue
		}

		if resp != nil {
			glog.V(2).Infof("Sending %s packet to %s", resp.Type.String(), pkt.HardwareAddr.String())
			glog.V(3).Info(resp.DebugString())
			if err = s.listener.SendDHCP(resp, intf); err != nil {
				glog.Warningf("Failed to send DHCP offer for %s: %s", pkt.HardwareAddr.String(), err)
			}
		}
	}
}

func interfaceIP(intf *net.Interface) (net.IP, error) {
	addrs, err := intf.Addrs()
	if err != nil {
		return nil, err
	}

	// Try to find an IPv4 address to use, in the following order:
	// global unicast (includes rfc1918), link-local unicast,
	// loopback.
	fs := [](func(net.IP) bool){
		net.IP.IsGlobalUnicast,
		net.IP.IsLinkLocalUnicast,
		net.IP.IsLoopback,
	}
	for _, f := range fs {
		for _, a := range addrs {
			ipaddr, ok := a.(*net.IPNet)
			if !ok {
				continue
			}
			ip := ipaddr.IP.To4()
			if ip == nil {
				continue
			}
			if f(ip) {
				return ip, nil
			}
		}
	}

	return nil, errors.New("no usable unicast address configured on interface")
}

func (s *Server) getInterfaceNo(hwAddr net.HardwareAddr) int {
	addr := hwAddr.String()
	for i, permitted := range s.config.Result.Interfaces {
		if permitted.Mac == addr {
			return i
		}
	}
	return -1
}

func (s *Server) prepareResponse(pkt *dhcp4.Packet, serverIP net.IP, mt dhcp4.MessageType) (*dhcp4.Packet, error) {
	interfaceNo := s.getInterfaceNo(pkt.HardwareAddr)
	if interfaceNo < 0 {
		return nil, fmt.Errorf("unexpected packet from %v", pkt.HardwareAddr)
	}

	var cfg *cnicurrent.IPConfig
	for _, curCfg := range s.config.Result.IPs {
		if curCfg.Version == "4" && curCfg.Interface == interfaceNo {
			cfg = curCfg
		}
	}
	var mtu uint16
	for _, iface := range s.config.Interfaces {
		if bytes.Compare(pkt.HardwareAddr, iface.HardwareAddr) == 0 {
			mtu = iface.MTU
		}
	}
	if mtu == 0 {
		return nil, fmt.Errorf("packet from mac address %s not found in CSN", pkt.HardwareAddr.String())
	}

	if cfg == nil {
		return nil, fmt.Errorf("IPv4 config for interface %s is not specified in CNI config", pkt.HardwareAddr.String())
	}

	p := &dhcp4.Packet{
		Type:          mt,
		TransactionID: pkt.TransactionID,
		Broadcast:     true,
		HardwareAddr:  pkt.HardwareAddr,
		RelayAddr:     pkt.RelayAddr,
		ServerAddr:    serverIP,
		Options:       make(dhcp4.Options),
	}
	p.Options[dhcp4.OptServerIdentifier] = serverIP

	// if guid was sent, copy it
	if pkt.Options[97] != nil {
		p.Options[97] = pkt.Options[97]
	}

	p.YourAddr = cfg.Address.IP
	p.Options[dhcp4.OptSubnetMask] = cfg.Address.Mask

	// MTU option
	p.Options[26] = []byte{uint8(mtu >> 8), uint8(mtu & 0xff)}

	router, routeData, err := s.getStaticRoutes(cfg.Address)
	if err != nil {
		glog.Warningf("Can not transform static routes for mac %v: %v", pkt.HardwareAddr, err)
	}
	if router != nil {
		p.Options[dhcp4.OptRouters] = router
	}
	if routeData != nil {
		p.Options[classlessRouteOption] = routeData
	}

	// 86400 - full 24h
	p.Options[dhcp4.OptLeaseTime] = []byte{0, 1, 81, 128}

	// 43200 - 12h
	p.Options[dhcp4.OptRenewalTime] = []byte{0, 0, 168, 192}

	// 64800 - 18h
	p.Options[dhcp4.OptRebindingTime] = []byte{0, 0, 253, 32}

	// TODO: include more dns options
	if len(s.config.Result.DNS.Nameservers) == 0 {
		p.Options[dhcp4.OptDNSServers] = defaultDNS
	} else {
		var b bytes.Buffer
		for _, nsIP := range s.config.Result.DNS.Nameservers {
			ip := net.ParseIP(nsIP).To4()
			if ip == nil {
				glog.Warningf("failed to parse nameserver ip %q", nsIP)
			} else {
				b.Write(ip)
			}
		}
		if b.Len() > 0 {
			p.Options[dhcp4.OptDNSServers] = b.Bytes()
		} else {
			p.Options[dhcp4.OptDNSServers] = defaultDNS
		}
	}
	if len(s.config.Result.DNS.Search) != 0 {
		// https://tools.ietf.org/search/rfc3397
		p.Options[119], err = compressedDomainList(s.config.Result.DNS.Search)
		if err != nil {
			return nil, err
		}
	}

	return p, nil
}

func (s *Server) offerDHCP(pkt *dhcp4.Packet, serverIP net.IP) (*dhcp4.Packet, error) {
	return s.prepareResponse(pkt, serverIP, dhcp4.MsgOffer)
}

func (s *Server) ackDHCP(pkt *dhcp4.Packet, serverIP net.IP) (*dhcp4.Packet, error) {
	return s.prepareResponse(pkt, serverIP, dhcp4.MsgAck)
}

func (s *Server) getStaticRoutes(ip net.IPNet) (router, routes []byte, err error) {
	if len(s.config.Result.Routes) == 0 {
		return nil, nil, nil
	}

	var b bytes.Buffer
	for _, route := range s.config.Result.Routes {
		if route.Dst.IP == nil {
			return nil, nil, fmt.Errorf("invalid route: %#v", route)
		}
		dstIP := route.Dst.IP.To4()
		gw := route.GW
		if gw == nil {
			// FIXME: this should not be really needed for newer CNI
			var cfg *cnicurrent.IPConfig
			for _, curCfg := range s.config.Result.IPs {
				if curCfg.Version == "4" {
					cfg = curCfg
				}
			}
			gw = cfg.Gateway.To4()
			if gw == nil && cfg.Gateway != nil {
				return nil, nil, fmt.Errorf("unexpected IPv6 gateway address: %#v", gw)
			}
		} else {
			gw = gw.To4()
		}
		if !ip.Contains(gw) {
			continue
		}
		if gw != nil && dstIP.Equal(net.IPv4zero) {
			if s, _ := route.Dst.Mask.Size(); s == 0 {
				router = gw
				continue
			}
		}
		b.Write(toDestinationDescriptor(route.Dst))
		if gw != nil {
			b.Write(gw)
		} else {
			b.Write([]byte{0, 0, 0, 0})
		}
	}

	routes = b.Bytes()
	return
}

// toDestinationDescriptor returns calculated destination descriptor according to rfc3442 (page 3)
// warning: there is no check if ipnet is in required ipv4 type
func toDestinationDescriptor(network net.IPNet) []byte {
	s, _ := network.Mask.Size()
	ipAsBytes := []byte(network.IP.To4())
	return append(
		[]byte{byte(s)},
		ipAsBytes[:widthOfMaskToSignificantOctets(s)]...,
	)
}

func widthOfMaskToSignificantOctets(mask int) int {
	return (mask + 7) / 8
}

func compressedDomainList(domainList []string) ([]byte, error) {
	// https://tools.ietf.org/search/rfc1035#section-4.1.4
	// simplified version, only encoding, without real compression
	var b bytes.Buffer
	for n, domain := range domainList {
		// add '\0' between entries
		if n > 0 {
			b.WriteByte(0)
		}

		// encode domain parts as (single byte length)(string)
		parts := strings.Split(domain, ".")
		for _, part := range parts {
			if len(part) > 254 {
				return nil, fmt.Errorf("domain name element '%s' exceeds 254 length limit", part)
			}
			b.WriteByte(byte(len(part)))
			b.WriteString(part)
		}
	}

	return b.Bytes(), nil
}
<commit_msg>Fix linter issues in pkg/dhcp/<commit_after>/*
Copyright 2016-2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Parts of this file are copied from https://github.com/google/netboot/blob/8e5c0d07937f8c1dea6e5f218b64f6b95c32ada3/pixiecore/dhcp.go

*/

package dhcp

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"strings"

	cnicurrent "github.com/containernetworking/cni/pkg/types/current"
	"github.com/golang/glog"
	"go.universe.tf/netboot/dhcp4"

	"github.com/Mirantis/virtlet/pkg/network"
)

const (
	serverPort = 67
	// option 121 is for static routes as defined in rfc3442
	classlessRouteOption = 121
)

var (
	defaultDNS = []byte{8, 8, 8, 8}
)

// Server implements a DHCP server that runs in the container network namespace.
type Server struct {
	config   *network.ContainerSideNetwork
	listener *dhcp4.Conn
}

// NewServer returns an initialized instance of Server.
func NewServer(config *network.ContainerSideNetwork) *Server {
	return &Server{config: config}
}

// SetupListener sets up a DHCP4 listener that listens on the default DHCP
// port and the specified IP address.
func (s *Server) SetupListener(laddr string) error {
	var listener *dhcp4.Conn
	var err error
	if listener, err = dhcp4.NewConn(fmt.Sprintf("%s:%d", laddr, serverPort)); err != nil {
		return err
	}
	s.listener = listener

	return nil
}

// Close shuts down DHCP listener.
func (s *Server) Close() error {
	return s.listener.Close()
}

// Serve waits in an endless loop for DHCP requests and answers them.
func (s *Server) Serve() error {
	for {
		pkt, intf, err := s.listener.RecvDHCP()
		if err != nil {
			return fmt.Errorf("receiving DHCP packet: %v", err)
		}
		if intf == nil {
			return fmt.Errorf("received DHCP packet with no interface information - please fill a bug to https://github.com/google/netboot")
		}
		glog.V(2).Infof("Received dhcp packet from: %s", pkt.HardwareAddr.String())

		serverIP, err := interfaceIP(intf)
		if err != nil {
			glog.Warningf("Want to respond to %s on %s, but couldn't get a source address: %s", pkt.HardwareAddr.String(), intf.Name, err)
			continue
		}

		var resp *dhcp4.Packet
		switch pkt.Type {
		case dhcp4.MsgDiscover:
			resp, err = s.offerDHCP(pkt, serverIP)
			if err != nil {
				glog.Warningf("Failed to construct DHCP offer for %s: %s", pkt.HardwareAddr.String(), err)
				continue
			}
		case dhcp4.MsgRequest:
			resp, err = s.ackDHCP(pkt, serverIP)
			if err != nil {
				glog.Warningf("Failed to construct DHCP ACK for %s: %s", pkt.HardwareAddr.String(), err)
				continue
			}
		default:
			glog.Warningf("Ignoring packet from %s: packet is %s", pkt.HardwareAddr.String(), pkt.Type.String())
			continue
		}

		if resp != nil {
			glog.V(2).Infof("Sending %s packet to %s", resp.Type.String(), pkt.HardwareAddr.String())
			glog.V(3).Info(resp.DebugString())
			if err = s.listener.SendDHCP(resp, intf); err != nil {
				glog.Warningf("Failed to send DHCP offer for %s: %s", pkt.HardwareAddr.String(), err)
			}
		}
	}
}

func interfaceIP(intf *net.Interface) (net.IP, error) {
	addrs, err := intf.Addrs()
	if err != nil {
		return nil, err
	}

	// Try to find an IPv4 address to use, in the following order:
	// global unicast (includes rfc1918), link-local unicast,
	// loopback.
	fs := [](func(net.IP) bool){
		net.IP.IsGlobalUnicast,
		net.IP.IsLinkLocalUnicast,
		net.IP.IsLoopback,
	}
	for _, f := range fs {
		for _, a := range addrs {
			ipaddr, ok := a.(*net.IPNet)
			if !ok {
				continue
			}
			ip := ipaddr.IP.To4()
			if ip == nil {
				continue
			}
			if f(ip) {
				return ip, nil
			}
		}
	}

	return nil, errors.New("no usable unicast address configured on interface")
}

func (s *Server) getInterfaceNo(hwAddr net.HardwareAddr) int {
	addr := hwAddr.String()
	for i, permitted := range s.config.Result.Interfaces {
		if permitted.Mac == addr {
			return i
		}
	}
	return -1
}

func (s *Server) prepareResponse(pkt *dhcp4.Packet, serverIP net.IP, mt dhcp4.MessageType) (*dhcp4.Packet, error) {
	interfaceNo := s.getInterfaceNo(pkt.HardwareAddr)
	if interfaceNo < 0 {
		return nil, fmt.Errorf("unexpected packet from %v", pkt.HardwareAddr)
	}

	var cfg *cnicurrent.IPConfig
	for _, curCfg := range s.config.Result.IPs {
		if curCfg.Version == "4" && curCfg.Interface == interfaceNo {
			cfg = curCfg
		}
	}
	var mtu uint16
	for _, iface := range s.config.Interfaces {
		if bytes.Compare(pkt.HardwareAddr, iface.HardwareAddr) == 0 {
			mtu = iface.MTU
		}
	}
	if mtu == 0 {
		return nil, fmt.Errorf("packet from mac address %s not found in CSN", pkt.HardwareAddr.String())
	}

	if cfg == nil {
		return nil, fmt.Errorf("IPv4 config for interface %s is not specified in CNI config", pkt.HardwareAddr.String())
	}

	p := &dhcp4.Packet{
		Type:          mt,
		TransactionID: pkt.TransactionID,
		Broadcast:     true,
		HardwareAddr:  pkt.HardwareAddr,
		RelayAddr:     pkt.RelayAddr,
		ServerAddr:    serverIP,
		Options:       make(dhcp4.Options),
	}
	p.Options[dhcp4.OptServerIdentifier] = serverIP

	// if guid was sent, copy it
	if pkt.Options[97] != nil {
		p.Options[97] = pkt.Options[97]
	}

	p.YourAddr = cfg.Address.IP
	p.Options[dhcp4.OptSubnetMask] = cfg.Address.Mask

	// MTU option
	p.Options[26] = []byte{uint8(mtu >> 8), uint8(mtu & 0xff)}

	router, routeData, err := s.getStaticRoutes(cfg.Address)
	if err != nil {
		glog.Warningf("Can not transform static routes for mac %v: %v", pkt.HardwareAddr, err)
	}
	if router != nil {
		p.Options[dhcp4.OptRouters] = router
	}
	if routeData != nil {
		p.Options[classlessRouteOption] = routeData
	}

	// 86400 - full 24h
	p.Options[dhcp4.OptLeaseTime] = []byte{0, 1, 81, 128}

	// 43200 - 12h
	p.Options[dhcp4.OptRenewalTime] = []byte{0, 0, 168, 192}

	// 64800 - 18h
	p.Options[dhcp4.OptRebindingTime] = []byte{0, 0, 253, 32}

	// TODO: include more dns options
	if len(s.config.Result.DNS.Nameservers) == 0 {
		p.Options[dhcp4.OptDNSServers] = defaultDNS
	} else {
		var b bytes.Buffer
		for _, nsIP := range s.config.Result.DNS.Nameservers {
			ip := net.ParseIP(nsIP).To4()
			if ip == nil {
				glog.Warningf("failed to parse nameserver ip %q", nsIP)
			} else {
				b.Write(ip)
			}
		}
		if b.Len() > 0 {
			p.Options[dhcp4.OptDNSServers] = b.Bytes()
		} else {
			p.Options[dhcp4.OptDNSServers] = defaultDNS
		}
	}
	if len(s.config.Result.DNS.Search) != 0 {
		// https://tools.ietf.org/search/rfc3397
		p.Options[119], err = compressedDomainList(s.config.Result.DNS.Search)
		if err != nil {
			return nil, err
		}
	}

	return p, nil
}

func (s *Server) offerDHCP(pkt *dhcp4.Packet, serverIP net.IP) (*dhcp4.Packet, error) {
	return s.prepareResponse(pkt, serverIP, dhcp4.MsgOffer)
}

func (s *Server) ackDHCP(pkt *dhcp4.Packet, serverIP net.IP) (*dhcp4.Packet, error) {
	return s.prepareResponse(pkt, serverIP, dhcp4.MsgAck)
}

func (s *Server) getStaticRoutes(ip net.IPNet) (router, routes []byte, err error) {
	if len(s.config.Result.Routes) == 0 {
		return nil, nil, nil
	}

	var b bytes.Buffer
	for _, route := range s.config.Result.Routes {
		if route.Dst.IP == nil {
			return nil, nil, fmt.Errorf("invalid route: %#v", route)
		}
		dstIP := route.Dst.IP.To4()
		gw := route.GW
		if gw == nil {
			// FIXME: this should not be really needed for newer CNI
			var cfg *cnicurrent.IPConfig
			for _, curCfg := range s.config.Result.IPs {
				if curCfg.Version == "4" {
					cfg = curCfg
				}
			}
			gw = cfg.Gateway.To4()
			if gw == nil && cfg.Gateway != nil {
				return nil, nil, fmt.Errorf("unexpected IPv6 gateway address: %#v", gw)
			}
		} else {
			gw = gw.To4()
		}
		if !ip.Contains(gw) {
			continue
		}
		if gw != nil && dstIP.Equal(net.IPv4zero) {
			if s, _ := route.Dst.Mask.Size(); s == 0 {
				router = gw
				continue
			}
		}
		b.Write(toDestinationDescriptor(route.Dst))
		if gw != nil {
			b.Write(gw)
		} else {
			b.Write([]byte{0, 0, 0, 0})
		}
	}

	routes = b.Bytes()
	return
}

// toDestinationDescriptor returns calculated destination descriptor according to rfc3442 (page 3)
// warning: there is no check if ipnet is in required ipv4 type
func toDestinationDescriptor(network net.IPNet) []byte {
	s, _ := network.Mask.Size()
	ipAsBytes := []byte(network.IP.To4())
	return append(
		[]byte{byte(s)},
		ipAsBytes[:widthOfMaskToSignificantOctets(s)]...,
	)
}

func widthOfMaskToSignificantOctets(mask int) int {
	return (mask + 7) / 8
}

func compressedDomainList(domainList []string) ([]byte, error) {
	// https://tools.ietf.org/search/rfc1035#section-4.1.4
	// simplified version, only encoding, without real compression
	var b bytes.Buffer
	for n, domain := range domainList {
		// add '\0' between entries
		if n > 0 {
			b.WriteByte(0)
		}

		// encode domain parts as (single byte length)(string)
		parts := strings.Split(domain, ".")
		for _, part := range parts {
			if len(part) > 254 {
				return nil, fmt.Errorf("domain name element '%s' exceeds 254 length limit", part)
			}
			b.WriteByte(byte(len(part)))
			b.WriteString(part)
		}
	}

	return b.Bytes(), nil
}
<|endoftext|>


<commit_before>package action

import (
	"context"
	"errors"
	"fmt"

	"github.com/fatih/color"
	"github.com/gopasspw/gopass/internal/backend"
	"github.com/gopasspw/gopass/internal/diff"
	"github.com/gopasspw/gopass/internal/notify"
	"github.com/gopasspw/gopass/internal/out"
	"github.com/gopasspw/gopass/internal/store"
	"github.com/gopasspw/gopass/internal/store/leaf"
	"github.com/gopasspw/gopass/internal/tree"
	"github.com/gopasspw/gopass/pkg/ctxutil"
	"github.com/gopasspw/gopass/pkg/debug"
	"github.com/urfave/cli/v2"
)

// Sync all stores with their remotes.
func (s *Action) Sync(c *cli.Context) error {
	return s.sync(ctxutil.WithGlobalFlags(c), c.String("store"))
}

func (s *Action) sync(ctx context.Context, store string) error {
	out.Printf(ctx, "🚥 Syncing with all remotes ...")

	numEntries := 0
	if l, err := s.Store.Tree(ctx); err == nil {
		numEntries = len(l.List(tree.INF))
	}
	numMPs := 0

	mps := s.Store.MountPoints()
	mps = append([]string{""}, mps...)

	// sync all stores (root and all mounted sub stores).
	for _, mp := range mps {
		if store != "" {
			if store != "root" && mp != store {
				continue
			}
			if store == "root" && mp != "" {
				continue
			}
		}

		numMPs++
		_ = s.syncMount(ctx, mp)
	}
	out.OKf(ctx, "All done")

	// Calculate number of changed entries.
	// This is a rough estimate as additions and deletions.
	// might cancel each other out.
	if l, err := s.Store.Tree(ctx); err == nil {
		numEntries = len(l.List(tree.INF)) - numEntries
	}
	diff := ""
	if numEntries > 0 {
		diff = fmt.Sprintf(" Added %d entries", numEntries)
	} else if numEntries < 0 {
		diff = fmt.Sprintf(" Removed %d entries", -1*numEntries)
	}
	_ = notify.Notify(ctx, "gopass - sync", fmt.Sprintf("Finished. Synced %d remotes.%s", numMPs, diff))

	return nil
}

// syncMount syncs a single mount.
func (s *Action) syncMount(ctx context.Context, mp string) error {
	ctxno := out.WithNewline(ctx, false)
	name := mp
	if mp == "" {
		name = "<root>"
	}
	out.Printf(ctxno, color.GreenString("[%s] ", name))

	sub, err := s.Store.GetSubStore(mp)
	if err != nil {
		out.Errorf(ctx, "Failed to get sub store %q: %s", name, err)
		return fmt.Errorf("failed to get sub stores (%s)", err)
	}

	if sub == nil {
		out.Errorf(ctx, "Failed to get sub stores '%s: nil'", name)
		return fmt.Errorf("failed to get sub stores (nil)")
	}

	l, err := sub.List(ctx, "")
	if err != nil {
		out.Errorf(ctx, "Failed to list store: %s", err)
	}

	out.Printf(ctxno, "\n   "+color.GreenString("%s pull and push ... ", sub.Storage().Name()))
	err = sub.Storage().Push(ctx, "", "")
	switch {
	case err == nil:
		debug.Log("Push succeeded")
		out.Printf(ctxno, color.GreenString("OK"))
	case errors.Is(err, store.ErrGitNoRemote):
		out.Printf(ctxno, "Skipped (no remote)")
		debug.Log("Failed to push %q to its remote: %s", name, err)
		return err
	case errors.Is(err, backend.ErrNotSupported):
		out.Printf(ctxno, "Skipped (not supported)")
	case errors.Is(err, store.ErrGitNotInit):
		out.Printf(ctxno, "Skipped (no Git repo)")
	default: // any other error
		out.Errorf(ctxno, "Failed to push %q to its remote: %s", name, err)
		return err
	}

	ln, err := sub.List(ctx, "")
	if err != nil {
		out.Errorf(ctx, "Failed to list store: %s", err)
	}
	syncPrintDiff(ctxno, l, ln)

	debug.Log("Syncing Mount %s. Exportkeys: %t", mp, ctxutil.IsExportKeys(ctx))
	if err := syncImportKeys(ctxno, sub, name); err != nil {
		return err
	}
	if ctxutil.IsExportKeys(ctx) {
		if err := syncExportKeys(ctxno, sub, name); err != nil {
			return err
		}
	}
	out.Printf(ctx, "\n   "+color.GreenString("done"))
	return nil
}

func syncImportKeys(ctx context.Context, sub *leaf.Store, name string) error {
	// import keys.
	out.Printf(ctx, "\n   "+color.GreenString("importing missing keys ... "))
	if err := sub.ImportMissingPublicKeys(ctx); err != nil {
		out.Errorf(ctx, "Failed to import missing public keys for %q: %s", name, err)
		return err
	}
	out.Printf(ctx, color.GreenString("OK"))
	return nil
}

func syncExportKeys(ctx context.Context, sub *leaf.Store, name string) error {
	// export keys.
	out.Printf(ctx, "\n   "+color.GreenString("exporting missing keys ... "))
	rs, err := sub.GetRecipients(ctx, "")
	if err != nil {
		out.Errorf(ctx, "Failed to load recipients for %q: %s", name, err)
		return err
	}
	exported, err := sub.ExportMissingPublicKeys(ctx, rs)
	if err != nil {
		out.Errorf(ctx, "Failed to export missing public keys for %q: %s", name, err)
		return err
	}

	// only run second push if we did export any keys.
	if !exported {
		out.Printf(ctx, color.GreenString("nothing to do"))
		return nil
	}

	if err := sub.Storage().Push(ctx, "", ""); err != nil {
		out.Errorf(ctx, "Failed to push %q to its remote: %s", name, err)
		return err
	}
	out.Printf(ctx, color.GreenString("OK"))
	return nil
}

func syncPrintDiff(ctxno context.Context, l, r []string) {
	added, removed := diff.Stat(l, r)
	debug.Log("diff - added: %d - removed: %d", added, removed)
	if added > 0 {
		out.Printf(ctxno, color.GreenString(" (Added %d entries)", added))
	}
	if removed > 0 {
		out.Printf(ctxno, color.GreenString(" (Removed %d entries)", removed))
	}
	if added < 1 && removed < 1 {
		out.Printf(ctxno, color.GreenString(" (no changes)"))
	}
}
<commit_msg>Small formatting fix (#2165)<commit_after>package action

import (
	"context"
	"errors"
	"fmt"

	"github.com/fatih/color"
	"github.com/gopasspw/gopass/internal/backend"
	"github.com/gopasspw/gopass/internal/diff"
	"github.com/gopasspw/gopass/internal/notify"
	"github.com/gopasspw/gopass/internal/out"
	"github.com/gopasspw/gopass/internal/store"
	"github.com/gopasspw/gopass/internal/store/leaf"
	"github.com/gopasspw/gopass/internal/tree"
	"github.com/gopasspw/gopass/pkg/ctxutil"
	"github.com/gopasspw/gopass/pkg/debug"
	"github.com/urfave/cli/v2"
)

// Sync all stores with their remotes.
func (s *Action) Sync(c *cli.Context) error {
	return s.sync(ctxutil.WithGlobalFlags(c), c.String("store"))
}

func (s *Action) sync(ctx context.Context, store string) error {
	out.Printf(ctx, "🚥 Syncing with all remotes ...")

	numEntries := 0
	if l, err := s.Store.Tree(ctx); err == nil {
		numEntries = len(l.List(tree.INF))
	}
	numMPs := 0

	mps := s.Store.MountPoints()
	mps = append([]string{""}, mps...)

	// sync all stores (root and all mounted sub stores).
	for _, mp := range mps {
		if store != "" {
			if store != "root" && mp != store {
				continue
			}
			if store == "root" && mp != "" {
				continue
			}
		}

		numMPs++
		_ = s.syncMount(ctx, mp)
	}
	out.OKf(ctx, "All done")

	// Calculate number of changed entries.
	// This is a rough estimate as additions and deletions.
	// might cancel each other out.
	if l, err := s.Store.Tree(ctx); err == nil {
		numEntries = len(l.List(tree.INF)) - numEntries
	}
	diff := ""
	if numEntries > 0 {
		diff = fmt.Sprintf(" Added %d entries", numEntries)
	} else if numEntries < 0 {
		diff = fmt.Sprintf(" Removed %d entries", -1*numEntries)
	}
	_ = notify.Notify(ctx, "gopass - sync", fmt.Sprintf("Finished. Synced %d remotes.%s", numMPs, diff))

	return nil
}

// syncMount syncs a single mount.
func (s *Action) syncMount(ctx context.Context, mp string) error {
	ctxno := out.WithNewline(ctx, false)
	name := mp
	if mp == "" {
		name = "<root>"
	}
	out.Printf(ctxno, color.GreenString("[%s] ", name))

	sub, err := s.Store.GetSubStore(mp)
	if err != nil {
		out.Errorf(ctx, "Failed to get sub store %q: %s", name, err)
		return fmt.Errorf("failed to get sub stores (%s)", err)
	}

	if sub == nil {
		out.Errorf(ctx, "Failed to get sub stores '%s: nil'", name)
		return fmt.Errorf("failed to get sub stores (nil)")
	}

	l, err := sub.List(ctx, "")
	if err != nil {
		out.Errorf(ctx, "Failed to list store: %s", err)
	}

	out.Printf(ctxno, "\n   "+color.GreenString("%s pull and push ... ", sub.Storage().Name()))
	err = sub.Storage().Push(ctx, "", "")
	switch {
	case err == nil:
		debug.Log("Push succeeded")
		out.Printf(ctxno, color.GreenString("OK"))
	case errors.Is(err, store.ErrGitNoRemote):
		out.Printf(ctx, "Skipped (no remote)")
		debug.Log("Failed to push %q to its remote: %s", name, err)
		return err
	case errors.Is(err, backend.ErrNotSupported):
		out.Printf(ctxno, "Skipped (not supported)")
	case errors.Is(err, store.ErrGitNotInit):
		out.Printf(ctxno, "Skipped (no Git repo)")
	default: // any other error
		out.Errorf(ctx, "Failed to push %q to its remote: %s", name, err)
		return err
	}

	ln, err := sub.List(ctx, "")
	if err != nil {
		out.Errorf(ctx, "Failed to list store: %s", err)
	}
	syncPrintDiff(ctxno, l, ln)

	debug.Log("Syncing Mount %s. Exportkeys: %t", mp, ctxutil.IsExportKeys(ctx))
	if err := syncImportKeys(ctxno, sub, name); err != nil {
		return err
	}
	if ctxutil.IsExportKeys(ctx) {
		if err := syncExportKeys(ctxno, sub, name); err != nil {
			return err
		}
	}
	out.Printf(ctx, "\n   "+color.GreenString("done"))
	return nil
}

func syncImportKeys(ctx context.Context, sub *leaf.Store, name string) error {
	// import keys.
	out.Printf(ctx, "\n   "+color.GreenString("importing missing keys ... "))
	if err := sub.ImportMissingPublicKeys(ctx); err != nil {
		out.Errorf(ctx, "Failed to import missing public keys for %q: %s", name, err)
		return err
	}
	out.Printf(ctx, color.GreenString("OK"))
	return nil
}

func syncExportKeys(ctx context.Context, sub *leaf.Store, name string) error {
	// export keys.
	out.Printf(ctx, "\n   "+color.GreenString("exporting missing keys ... "))
	rs, err := sub.GetRecipients(ctx, "")
	if err != nil {
		out.Errorf(ctx, "Failed to load recipients for %q: %s", name, err)
		return err
	}
	exported, err := sub.ExportMissingPublicKeys(ctx, rs)
	if err != nil {
		out.Errorf(ctx, "Failed to export missing public keys for %q: %s", name, err)
		return err
	}

	// only run second push if we did export any keys.
	if !exported {
		out.Printf(ctx, color.GreenString("nothing to do"))
		return nil
	}

	if err := sub.Storage().Push(ctx, "", ""); err != nil {
		out.Errorf(ctx, "Failed to push %q to its remote: %s", name, err)
		return err
	}
	out.Printf(ctx, color.GreenString("OK"))
	return nil
}

func syncPrintDiff(ctxno context.Context, l, r []string) {
	added, removed := diff.Stat(l, r)
	debug.Log("diff - added: %d - removed: %d", added, removed)
	if added > 0 {
		out.Printf(ctxno, color.GreenString(" (Added %d entries)", added))
	}
	if removed > 0 {
		out.Printf(ctxno, color.GreenString(" (Removed %d entries)", removed))
	}
	if added < 1 && removed < 1 {
		out.Printf(ctxno, color.GreenString(" (no changes)"))
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gcs

import (
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"k8s.io/release/pkg/gcp"
	"k8s.io/utils/pointer"
)

var (
	// GcsPrefix url prefix for google cloud storage buckets
	GcsPrefix      = "gs://"
	concurrentFlag = "-m"
	recursiveFlag  = "-r"
	noClobberFlag  = "-n"
)

type Options struct {
	// gsutil options
	Concurrent *bool
	Recursive  *bool
	NoClobber  *bool

	// local options
	// AllowMissing allows a copy operation to be skipped if the source or
	// destination does not exist. This is useful for scenarios where copy
	// operations happen in a loop/channel, so a single "failure" does not block
	// the entire operation.
	AllowMissing *bool
}

// DefaultGCSCopyOptions have the default options for the GCS copy action
var DefaultGCSCopyOptions = &Options{
	Concurrent:   pointer.BoolPtr(true),
	Recursive:    pointer.BoolPtr(true),
	NoClobber:    pointer.BoolPtr(true),
	AllowMissing: pointer.BoolPtr(true),
}

// CopyToGCS copies a local directory to the specified GCS path
func CopyToGCS(src, gcsPath string, opts *Options) error {
	logrus.Infof("Copying %s to GCS (%s)", src, gcsPath)
	gcsPath = NormalizeGCSPath(gcsPath)

	_, err := os.Stat(src)
	if err != nil {
		logrus.Info("Unable to get local source directory info")

		if *opts.AllowMissing {
			logrus.Infof("Source directory (%s) does not exist. Skipping GCS upload.", src)
			return nil
		}

		return errors.New("source directory does not exist")
	}

	return bucketCopy(src, gcsPath, opts)
}

// CopyToLocal copies a GCS path to the specified local directory
func CopyToLocal(gcsPath, dst string, opts *Options) error {
	logrus.Infof("Copying GCS (%s) to %s", gcsPath, dst)
	gcsPath = NormalizeGCSPath(gcsPath)

	return bucketCopy(gcsPath, dst, opts)
}

// CopyBucketToBucket copies between two GCS paths.
func CopyBucketToBucket(src, dst string, opts *Options) error {
	logrus.Infof("Copying %s to %s", src, dst)
	return bucketCopy(NormalizeGCSPath(src), NormalizeGCSPath(dst), opts)
}

func bucketCopy(src, dst string, opts *Options) error {
	args := []string{}

	if *opts.Concurrent {
		logrus.Debug("Setting GCS copy to run concurrently")
		args = append(args, concurrentFlag)
	}

	args = append(args, "cp")
	if *opts.Recursive {
		logrus.Debug("Setting GCS copy to run recursively")
		args = append(args, recursiveFlag)
	}
	if *opts.NoClobber {
		logrus.Debug("Setting GCS copy to not clobber existing files")
		args = append(args, noClobberFlag)
	}

	args = append(args, src, dst)

	if err := gcp.GSUtil(args...); err != nil {
		return errors.Wrap(err, "gcs copy")
	}

	return nil
}

// GetReleasePath returns a GCS path to retrieve builds from or push builds to
//
// Expected destination format:
//   gs://<bucket>/<buildType>[-<gcsSuffix>][/fast][/<version>]
func GetReleasePath(
	bucket, buildType, gcsSuffix, version string,
	fast bool) string {
	return getPath(
		bucket,
		buildType,
		gcsSuffix,
		version,
		"release",
		fast,
	)
}

// GetMarkerPath returns a GCS path where version markers should be stored
//
// Expected destination format:
//   gs://<bucket>/<buildType>[-<gcsSuffix>]
func GetMarkerPath(
	bucket, buildType, gcsSuffix string) string {
	return getPath(
		bucket,
		buildType,
		gcsSuffix,
		"",
		"marker",
		false,
	)
}

// GetReleasePath returns a GCS path to retrieve builds from or push builds to
//
// Expected destination format:
//   gs://<bucket>/<buildType>[-<gcsSuffix>][/fast][/<version>]
// TODO: Support "release" buildType
func getPath(
	bucket, buildType, gcsSuffix, version, pathType string,
	fast bool) string {
	gcsPath := bucket
	gcsPath = filepath.Join(gcsPath, buildType)

	if gcsSuffix != "" {
		gcsPath += "-" + gcsSuffix
	}

	if pathType == "release" {
		if fast {
			gcsPath = filepath.Join(gcsPath, "fast")
		}

		if version != "" {
			gcsPath = filepath.Join(gcsPath, version)
		}
	}

	logrus.Infof("GCS path is %s", gcsPath)

	return gcsPath
}

// NormalizeGCSPath takes a gcs path and ensures that the `GcsPrefix` is
// prepended to it.
func NormalizeGCSPath(gcsPath string) string {
	gcsPath = strings.TrimPrefix(gcsPath, GcsPrefix)
	gcsPath = GcsPrefix + gcsPath

	return gcsPath
}

// RsyncRecursive runs `gsutil rsync` in recursive mode. The caller of this
// function has to ensure that the provided paths are prefixed with gs:// if
// necessary (see `NormalizeGCSPath()`).
func RsyncRecursive(src, dst string) error {
	return errors.Wrap(
		gcp.GSUtil("-m", "rsync", "-r", src, dst),
		"running gsutil rsync",
	)
}

// PathExists returns true if the specified GCS path exists.
func PathExists(gcsPath string) (bool, error) {
	err := gcp.GSUtil(
		"ls",
		gcsPath,
	)
	if err != nil {
		return false, err
	}

	logrus.Infof("Found %s", gcsPath)
	return true, nil
}
<commit_msg>pkg/gcp/gcs: Improve path normalization logic<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gcs

import (
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"k8s.io/release/pkg/gcp"
	"k8s.io/utils/pointer"
)

var (
	// GcsPrefix url prefix for google cloud storage buckets
	GcsPrefix      = "gs://"
	concurrentFlag = "-m"
	recursiveFlag  = "-r"
	noClobberFlag  = "-n"
)

type Options struct {
	// gsutil options
	Concurrent *bool
	Recursive  *bool
	NoClobber  *bool

	// local options
	// AllowMissing allows a copy operation to be skipped if the source or
	// destination does not exist. This is useful for scenarios where copy
	// operations happen in a loop/channel, so a single "failure" does not block
	// the entire operation.
	AllowMissing *bool
}

// DefaultGCSCopyOptions have the default options for the GCS copy action
var DefaultGCSCopyOptions = &Options{
	Concurrent:   pointer.BoolPtr(true),
	Recursive:    pointer.BoolPtr(true),
	NoClobber:    pointer.BoolPtr(true),
	AllowMissing: pointer.BoolPtr(true),
}

// CopyToGCS copies a local directory to the specified GCS path
// TODO: Consider using isPathNormalized here
func CopyToGCS(src, gcsPath string, opts *Options) error {
	logrus.Infof("Copying %s to GCS (%s)", src, gcsPath)
	gcsPath = NormalizeGCSPath(gcsPath)

	_, err := os.Stat(src)
	if err != nil {
		logrus.Info("Unable to get local source directory info")

		if *opts.AllowMissing {
			logrus.Infof("Source directory (%s) does not exist. Skipping GCS upload.", src)
			return nil
		}

		return errors.New("source directory does not exist")
	}

	return bucketCopy(src, gcsPath, opts)
}

// CopyToLocal copies a GCS path to the specified local directory
// TODO: Consider using isPathNormalized here
func CopyToLocal(gcsPath, dst string, opts *Options) error {
	logrus.Infof("Copying GCS (%s) to %s", gcsPath, dst)
	gcsPath = NormalizeGCSPath(gcsPath)

	return bucketCopy(gcsPath, dst, opts)
}

// CopyBucketToBucket copies between two GCS paths.
// TODO: Consider using isPathNormalized here
func CopyBucketToBucket(src, dst string, opts *Options) error {
	logrus.Infof("Copying %s to %s", src, dst)
	return bucketCopy(NormalizeGCSPath(src), NormalizeGCSPath(dst), opts)
}

func bucketCopy(src, dst string, opts *Options) error {
	args := []string{}

	if *opts.Concurrent {
		logrus.Debug("Setting GCS copy to run concurrently")
		args = append(args, concurrentFlag)
	}

	args = append(args, "cp")
	if *opts.Recursive {
		logrus.Debug("Setting GCS copy to run recursively")
		args = append(args, recursiveFlag)
	}
	if *opts.NoClobber {
		logrus.Debug("Setting GCS copy to not clobber existing files")
		args = append(args, noClobberFlag)
	}

	args = append(args, src, dst)

	if err := gcp.GSUtil(args...); err != nil {
		return errors.Wrap(err, "gcs copy")
	}

	return nil
}

// GetReleasePath returns a GCS path to retrieve builds from or push builds to
//
// Expected destination format:
//   gs://<bucket>/<buildType>[-<gcsSuffix>][/fast][/<version>]
func GetReleasePath(
	bucket, buildType, gcsSuffix, version string,
	fast bool) string {
	gcsPath := getPath(
		bucket,
		buildType,
		gcsSuffix,
		version,
		"release",
		fast,
	)

	logrus.Infof("Release path is %s", gcsPath)
	return gcsPath
}

// GetMarkerPath returns a GCS path where version markers should be stored
//
// Expected destination format:
//   gs://<bucket>/<buildType>[-<gcsSuffix>]
func GetMarkerPath(
	bucket, buildType, gcsSuffix string) string {
	gcsPath := getPath(
		bucket,
		buildType,
		gcsSuffix,
		"",
		"marker",
		false,
	)

	logrus.Infof("Version marker path is %s", gcsPath)
	return gcsPath
}

// GetReleasePath returns a GCS path to retrieve builds from or push builds to
//
// Expected destination format:
//   gs://<bucket>/<buildType>[-<gcsSuffix>][/fast][/<version>]
// TODO: Support "release" buildType
func getPath(
	bucket, buildType, gcsSuffix, version, pathType string,
	fast bool) string {
	gcsPath := bucket
	gcsPath = filepath.Join(gcsPath, buildType)

	if gcsSuffix != "" {
		gcsPath += "-" + gcsSuffix
	}

	if pathType == "release" {
		if fast {
			gcsPath = filepath.Join(gcsPath, "fast")
		}

		if version != "" {
			gcsPath = filepath.Join(gcsPath, version)
		}
	}

	// Ensure any constructed GCS path is prefixed with `gs://`
	gcsPath = NormalizeGCSPath(gcsPath)

	return gcsPath
}

// NormalizeGCSPath takes a GCS path and ensures that the `GcsPrefix` is
// prepended to it.
func NormalizeGCSPath(gcsPath string) string {
	gcsPath = strings.TrimPrefix(gcsPath, GcsPrefix)
	gcsPath = GcsPrefix + gcsPath

	return gcsPath
}

// isPathNormalized determines if a GCS path is prefixed with `gs://`.
// Use this function as pre-check for any gsutil/GCS functions that manipulate
// GCS bucket contents.
func isPathNormalized(gcsPath string) bool {
	if !strings.HasPrefix(gcsPath, GcsPrefix) {
		logrus.Errorf("GCS path (%s) should be prefixed with `gs://`", GcsPrefix)

		return false
	}

	return true
}

// RsyncRecursive runs `gsutil rsync` in recursive mode. The caller of this
// function has to ensure that the provided paths are prefixed with gs:// if
// necessary (see `NormalizeGCSPath()`).
// TODO: Implementation of `gsutil rsync` should support local directory copies
//       with `-d`
func RsyncRecursive(src, dst string) error {
	if !isPathNormalized(src) || !isPathNormalized(dst) {
		return errors.New("cannot run `gsutil rsync` as one or more paths does not begin with `gs://`")
	}

	return errors.Wrap(
		gcp.GSUtil(concurrentFlag, "rsync", recursiveFlag, src, dst),
		"running gsutil rsync",
	)
}

// PathExists returns true if the specified GCS path exists.
func PathExists(gcsPath string) (bool, error) {
	if !isPathNormalized(gcsPath) {
		return false, errors.New("cannot run `gsutil ls` GCS path does not begin with `gs://`")
	}

	err := gcp.GSUtil(
		"ls",
		gcsPath,
	)
	if err != nil {
		return false, err
	}

	logrus.Infof("Found %s", gcsPath)
	return true, nil
}
<|endoftext|>


<commit_before>// local build script file, similar to a makefile or collection of bash scripts in other projects

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/urfave/cli/v2"
)

var packages = []string{"cli", "altsrc"}

func main() {
	app := cli.NewApp()

	app.Name = "builder"
	app.Usage = "Generates a new urfave/cli build!"

	app.Commands = cli.Commands{
		{
			Name:   "vet",
			Action: VetActionFunc,
		},
		{
			Name:   "test",
			Action: TestActionFunc,
		},
		{
			Name:   "gfmrun",
			Action: GfmrunActionFunc,
		},
		{
			Name:   "toc",
			Action: TocActionFunc,
		},
		{
			Name:   "check-binary-size",
			Action: checkBinarySizeActionFunc,
		},
	}

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}

func runCmd(arg string, args ...string) error {
	cmd := exec.Command(arg, args...)

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}

func VetActionFunc(_ *cli.Context) error {
	return runCmd("go", "vet")
}

func TestActionFunc(c *cli.Context) error {
	for _, pkg := range packages {
		var packageName string

		if pkg == "cli" {
			packageName = "github.com/urfave/cli/v2"
		} else {
			packageName = fmt.Sprintf("github.com/urfave/cli/v2/%s", pkg)
		}

		coverProfile := fmt.Sprintf("--coverprofile=%s.coverprofile", pkg)

		err := runCmd("go", "test", "-v", coverProfile, packageName)
		if err != nil {
			return err
		}
	}

	return testCleanup()
}

func testCleanup() error {
	var out bytes.Buffer

	for _, pkg := range packages {
		file, err := os.Open(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}

		b, err := ioutil.ReadAll(file)
		if err != nil {
			return err
		}

		out.Write(b)
		err = file.Close()
		if err != nil {
			return err
		}

		err = os.Remove(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}
	}

	outFile, err := os.Create("coverage.txt")
	if err != nil {
		return err
	}

	_, err = out.WriteTo(outFile)
	if err != nil {
		return err
	}

	err = outFile.Close()
	if err != nil {
		return err
	}

	return nil
}

func GfmrunActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()

	var counter int
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "package main") {
			counter++
		}
	}

	err = file.Close()
	if err != nil {
		return err
	}

	err = scanner.Err()
	if err != nil {
		return err
	}

	return runCmd("gfmrun", "-c", fmt.Sprint(counter), "-s", filename)
}

func TocActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	err := runCmd("markdown-toc", "-i", filename)
	if err != nil {
		return err
	}

	err = runCmd("git", "diff", "--exit-code")
	if err != nil {
		return err
	}

	return nil
}

func checkBinarySizeActionFunc(c *cli.Context) (err error) {
	err = runCmd("git", "diff", "--exit-code")
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>show file size<commit_after>// local build script file, similar to a makefile or collection of bash scripts in other projects

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/urfave/cli/v2"
)

var packages = []string{"cli", "altsrc"}

func main() {
	app := cli.NewApp()

	app.Name = "builder"
	app.Usage = "Generates a new urfave/cli build!"

	app.Commands = cli.Commands{
		{
			Name:   "vet",
			Action: VetActionFunc,
		},
		{
			Name:   "test",
			Action: TestActionFunc,
		},
		{
			Name:   "gfmrun",
			Action: GfmrunActionFunc,
		},
		{
			Name:   "toc",
			Action: TocActionFunc,
		},
		{
			Name:   "check-binary-size",
			Action: checkBinarySizeActionFunc,
		},
	}

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}

func runCmd(arg string, args ...string) error {
	cmd := exec.Command(arg, args...)

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}

func VetActionFunc(_ *cli.Context) error {
	return runCmd("go", "vet")
}

func TestActionFunc(c *cli.Context) error {
	for _, pkg := range packages {
		var packageName string

		if pkg == "cli" {
			packageName = "github.com/urfave/cli/v2"
		} else {
			packageName = fmt.Sprintf("github.com/urfave/cli/v2/%s", pkg)
		}

		coverProfile := fmt.Sprintf("--coverprofile=%s.coverprofile", pkg)

		err := runCmd("go", "test", "-v", coverProfile, packageName)
		if err != nil {
			return err
		}
	}

	return testCleanup()
}

func testCleanup() error {
	var out bytes.Buffer

	for _, pkg := range packages {
		file, err := os.Open(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}

		b, err := ioutil.ReadAll(file)
		if err != nil {
			return err
		}

		out.Write(b)
		err = file.Close()
		if err != nil {
			return err
		}

		err = os.Remove(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}
	}

	outFile, err := os.Create("coverage.txt")
	if err != nil {
		return err
	}

	_, err = out.WriteTo(outFile)
	if err != nil {
		return err
	}

	err = outFile.Close()
	if err != nil {
		return err
	}

	return nil
}

func GfmrunActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()

	var counter int
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "package main") {
			counter++
		}
	}

	err = file.Close()
	if err != nil {
		return err
	}

	err = scanner.Err()
	if err != nil {
		return err
	}

	return runCmd("gfmrun", "-c", fmt.Sprint(counter), "-s", filename)
}

func TocActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	err := runCmd("markdown-toc", "-i", filename)
	if err != nil {
		return err
	}

	err = runCmd("git", "diff", "--exit-code")
	if err != nil {
		return err
	}

	return nil
}

func checkBinarySizeActionFunc(c *cli.Context) (err error) {
	const (
		sourceFilePath = "./internal/example/example.go"
		builtFilePath  = "./internal/example/built-example"
	)

	err = runCmd("go", "build", "-o", builtFilePath, sourceFilePath)
	if err != nil {
		return err
	}

	fileInfo, err := os.Stat(builtFilePath)
	if err != nil {
		return err
	}
	fileSize := fileInfo.Size()

	// show the file size
	fmt.Println(fileSize)

	return nil
}
<|endoftext|>


<commit_before>// Copyright Istio Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
	"encoding/json"
	"path"
)

type BuildOutput int
type AutoDependency string

const (
	Docker BuildOutput = iota
	Helm
	Debian
	Archive

	// Deps will resolve by looking at the istio.deps file in istio/istio
	Deps string = "deps"
	// Modules will resolve by looking at the go.mod file in istio/istio
	Modules string = "modules"
)

// Dependency defines a git dependency for the build
type Dependency struct {
	// Git repository to pull from. Required if branch or sha is set
	Git string `json:"git,omitempty"`
	// Checkout the git branch
	Branch string `json:"branch,omitempty"`
	// Checkout the git SHA
	Sha string `json:"sha,omitempty"`
	// Copy the local path. Note this still needs to be a git repo.
	LocalPath string `json:"localpath,omitempty"`
	// Auto will fetch the SHA to use based on other repos. Currently this supports reading
	// istio.deps from istio/istio only.
	Auto string `json:"auto,omitempty"`
	// If true, go version semantic will be used for tagging the git repo, e.g. v1.2.3.
	GoVersionEnabled bool `json:"goversionenabled,omitempty"`
}

// Ref returns the git reference of a dependency.
func (d Dependency) Ref() string {
	ref := d.Branch
	if d.Sha != "" {
		ref = d.Sha
	}
	return ref
}

// Dependencies for the build
type IstioDependencies struct {
	Istio        *Dependency `json:"istio"`
	Cni          *Dependency `json:"cni"`
	Operator     *Dependency `json:"operator"`
	Api          *Dependency `json:"api"` //nolint: golint, stylecheck
	Proxy        *Dependency `json:"proxy"`
	Pkg          *Dependency `json:"pkg"`
	ClientGo     *Dependency `json:"client-go"`
	GogoGenproto *Dependency `json:"gogo-genproto"`
	TestInfra    *Dependency `json:"test-infra"`
	Tools        *Dependency `json:"tools"`
}

func (i *IstioDependencies) Get() map[string]*Dependency {
	return map[string]*Dependency{
		"istio":         i.Istio,
		"cni":           i.Cni,
		"operator":      i.Operator,
		"api":           i.Api,
		"proxy":         i.Proxy,
		"pkg":           i.Pkg,
		"client-go":     i.ClientGo,
		"gogo-genproto": i.GogoGenproto,
		"test-infra":    i.TestInfra,
		"tools":         i.Tools,
	}
}

// MarshalJSON writes the dependencies, exposing just the SHA
func (i IstioDependencies) MarshalJSON() ([]byte, error) {
	deps := make(map[string]Dependency)
	for repo, dep := range i.Get() {
		if dep == nil {
			continue
		}
		deps[repo] = Dependency{Sha: dep.Sha}
	}
	return json.Marshal(deps)
}

func (i *IstioDependencies) Set(repo string, dependency Dependency) {
	dp := i.Get()[repo]
	*dp = dependency
}

// Manifest defines what is in a release
type InputManifest struct {
	// Dependencies declares all git repositories used to build this release
	Dependencies IstioDependencies `json:"dependencies"`
	// Version specifies what version of Istio this release is
	Version string `json:"version"`
	// Docker specifies the docker hub to use in the helm charts.
	Docker string `json:"docker"`
	// Directory defines the base working directory for the release.
	// This is excluded from the final serialization
	Directory string `json:"directory"`
	// ProxyOverride specifies a URL to an Envoy binary to use instead of the default proxy
	// The binary will be pulled from `$proxyOverride/envoy-alpha-SHA.tar.gz`
	ProxyOverride string `json:"proxyOverride"`
	// BuildOutputs defines what components to build. This allows building only some components.
	BuildOutputs []string `json:"outputs"`
}

// Manifest defines what is in a release
type Manifest struct {
	// Dependencies declares all git repositories used to build this release
	Dependencies IstioDependencies `json:"dependencies"`
	// Version specifies what version of Istio this release is
	Version string `json:"version"`
	// Docker specifies the docker hub to use in the helm charts.
	Docker string `json:"docker"`
	// Directory defines the base working directory for the release.
	// This is excluded from the final serialization
	Directory string `json:"-"`
	// ProxyOverride specifies a URL to an Envoy binary to use instead of the default proxy
	// The binary will be pulled from `$proxyOverride/envoy-alpha-SHA.tar.gz`
	ProxyOverride string `json:"-"`
	// BuildOutputs defines what components to build. This allows building only some components.
	BuildOutputs map[BuildOutput]struct{} `json:"-"`
}

// RepoDir is a helper to return the working directory for a repo
func (m Manifest) RepoDir(repo string) string {
	return path.Join(m.Directory, "work", "src", "istio.io", repo)
}

// GoOutDir is a helper to return the directory of Istio build output
func (m Manifest) GoOutDir() string {
	return path.Join(m.Directory, "work", "out", "linux_amd64", "release")
}

// RepoOutDir is a helper to return the directory of Istio build output for repos the place outputs inside the repo
func (m Manifest) RepoOutDir(repo string) string {
	return path.Join(m.Directory, "work", "src", "istio.io", repo, "out", "linux_amd64", "release")
}

// WorkDir is a help to return the work directory
func (m Manifest) WorkDir() string {
	return path.Join(m.Directory, "work")
}

// SourceDir is a help to return the sources directory
func (m Manifest) SourceDir() string {
	return path.Join(m.Directory, "sources")
}

// OutDir is a help to return the out directory
func (m Manifest) OutDir() string {
	return path.Join(m.Directory, "out")
}

// IstioDep identifies a external dependency of Istio.
type IstioDep struct {
	Comment       string `json:"_comment,omitempty"`
	Name          string `json:"name,omitempty"`
	RepoName      string `json:"repoName,omitempty"`
	LastStableSHA string `json:"lastStableSHA,omitempty"`
}
<commit_msg>Adding Installer as a configurable dependency in release-builder. (#104)<commit_after>// Copyright Istio Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
	"encoding/json"
	"path"
)

type BuildOutput int
type AutoDependency string

const (
	Docker BuildOutput = iota
	Helm
	Debian
	Archive

	// Deps will resolve by looking at the istio.deps file in istio/istio
	Deps string = "deps"
	// Modules will resolve by looking at the go.mod file in istio/istio
	Modules string = "modules"
)

// Dependency defines a git dependency for the build
type Dependency struct {
	// Git repository to pull from. Required if branch or sha is set
	Git string `json:"git,omitempty"`
	// Checkout the git branch
	Branch string `json:"branch,omitempty"`
	// Checkout the git SHA
	Sha string `json:"sha,omitempty"`
	// Copy the local path. Note this still needs to be a git repo.
	LocalPath string `json:"localpath,omitempty"`
	// Auto will fetch the SHA to use based on other repos. Currently this supports reading
	// istio.deps from istio/istio only.
	Auto string `json:"auto,omitempty"`
	// If true, go version semantic will be used for tagging the git repo, e.g. v1.2.3.
	GoVersionEnabled bool `json:"goversionenabled,omitempty"`
}

// Ref returns the git reference of a dependency.
func (d Dependency) Ref() string {
	ref := d.Branch
	if d.Sha != "" {
		ref = d.Sha
	}
	return ref
}

// Dependencies for the build
type IstioDependencies struct {
	Istio        *Dependency `json:"istio"`
	Cni          *Dependency `json:"cni"`
	Operator     *Dependency `json:"operator"`
	Api          *Dependency `json:"api"` //nolint: golint, stylecheck
	Proxy        *Dependency `json:"proxy"`
	Pkg          *Dependency `json:"pkg"`
	ClientGo     *Dependency `json:"client-go"`
	GogoGenproto *Dependency `json:"gogo-genproto"`
	TestInfra    *Dependency `json:"test-infra"`
	Tools        *Dependency `json:"tools"`
	Installer    *Dependency `json:"installer"`
}

func (i *IstioDependencies) Get() map[string]*Dependency {
	return map[string]*Dependency{
		"istio":         i.Istio,
		"cni":           i.Cni,
		"operator":      i.Operator,
		"api":           i.Api,
		"proxy":         i.Proxy,
		"pkg":           i.Pkg,
		"client-go":     i.ClientGo,
		"gogo-genproto": i.GogoGenproto,
		"test-infra":    i.TestInfra,
		"tools":         i.Tools,
		"installer":     i.Installer,
	}
}

// MarshalJSON writes the dependencies, exposing just the SHA
func (i IstioDependencies) MarshalJSON() ([]byte, error) {
	deps := make(map[string]Dependency)
	for repo, dep := range i.Get() {
		if dep == nil {
			continue
		}
		deps[repo] = Dependency{Sha: dep.Sha}
	}
	return json.Marshal(deps)
}

func (i *IstioDependencies) Set(repo string, dependency Dependency) {
	dp := i.Get()[repo]
	*dp = dependency
}

// Manifest defines what is in a release
type InputManifest struct {
	// Dependencies declares all git repositories used to build this release
	Dependencies IstioDependencies `json:"dependencies"`
	// Version specifies what version of Istio this release is
	Version string `json:"version"`
	// Docker specifies the docker hub to use in the helm charts.
	Docker string `json:"docker"`
	// Directory defines the base working directory for the release.
	// This is excluded from the final serialization
	Directory string `json:"directory"`
	// ProxyOverride specifies a URL to an Envoy binary to use instead of the default proxy
	// The binary will be pulled from `$proxyOverride/envoy-alpha-SHA.tar.gz`
	ProxyOverride string `json:"proxyOverride"`
	// BuildOutputs defines what components to build. This allows building only some components.
	BuildOutputs []string `json:"outputs"`
}

// Manifest defines what is in a release
type Manifest struct {
	// Dependencies declares all git repositories used to build this release
	Dependencies IstioDependencies `json:"dependencies"`
	// Version specifies what version of Istio this release is
	Version string `json:"version"`
	// Docker specifies the docker hub to use in the helm charts.
	Docker string `json:"docker"`
	// Directory defines the base working directory for the release.
	// This is excluded from the final serialization
	Directory string `json:"-"`
	// ProxyOverride specifies a URL to an Envoy binary to use instead of the default proxy
	// The binary will be pulled from `$proxyOverride/envoy-alpha-SHA.tar.gz`
	ProxyOverride string `json:"-"`
	// BuildOutputs defines what components to build. This allows building only some components.
	BuildOutputs map[BuildOutput]struct{} `json:"-"`
}

// RepoDir is a helper to return the working directory for a repo
func (m Manifest) RepoDir(repo string) string {
	return path.Join(m.Directory, "work", "src", "istio.io", repo)
}

// GoOutDir is a helper to return the directory of Istio build output
func (m Manifest) GoOutDir() string {
	return path.Join(m.Directory, "work", "out", "linux_amd64", "release")
}

// RepoOutDir is a helper to return the directory of Istio build output for repos the place outputs inside the repo
func (m Manifest) RepoOutDir(repo string) string {
	return path.Join(m.Directory, "work", "src", "istio.io", repo, "out", "linux_amd64", "release")
}

// WorkDir is a help to return the work directory
func (m Manifest) WorkDir() string {
	return path.Join(m.Directory, "work")
}

// SourceDir is a help to return the sources directory
func (m Manifest) SourceDir() string {
	return path.Join(m.Directory, "sources")
}

// OutDir is a help to return the out directory
func (m Manifest) OutDir() string {
	return path.Join(m.Directory, "out")
}

// IstioDep identifies a external dependency of Istio.
type IstioDep struct {
	Comment       string `json:"_comment,omitempty"`
	Name          string `json:"name,omitempty"`
	RepoName      string `json:"repoName,omitempty"`
	LastStableSHA string `json:"lastStableSHA,omitempty"`
}
<|endoftext|>