<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubelet

import (
	"fmt"
	"time"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/kubelet/container"
	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
)

const (
	runOnceManifestDelay     = 1 * time.Second
	runOnceMaxRetries        = 10
	runOnceRetryDelay        = 1 * time.Second
	runOnceRetryDelayBackoff = 2
)

type RunPodResult struct {
	Pod *api.Pod
	Err error
}

// RunOnce polls from one configuration update and run the associated pods.
func (kl *Kubelet) RunOnce(updates <-chan kubetypes.PodUpdate) ([]RunPodResult, error) {
	select {
	case u := <-updates:
		glog.Infof("processing manifest with %d pods", len(u.Pods))
		result, err := kl.runOnce(u.Pods, runOnceRetryDelay)
		glog.Infof("finished processing %d pods", len(u.Pods))
		return result, err
	case <-time.After(runOnceManifestDelay):
		return nil, fmt.Errorf("no pod manifest update after %v", runOnceManifestDelay)
	}
}

// runOnce runs a given set of pods and returns their status.
func (kl *Kubelet) runOnce(pods []*api.Pod, retryDelay time.Duration) (results []RunPodResult, err error) {
	ch := make(chan RunPodResult)
	admitted := []*api.Pod{}
	for _, pod := range pods {
		// Check if we can admit the pod.
		if ok, reason, message := kl.canAdmitPod(append(admitted, pod), pod); !ok {
			kl.rejectPod(pod, reason, message)
		} else {
			admitted = append(admitted, pod)
		}
		go func(pod *api.Pod) {
			err := kl.runPod(pod, retryDelay)
			ch <- RunPodResult{pod, err}
		}(pod)
	}

	glog.Infof("waiting for %d pods", len(pods))
	failedPods := []string{}
	for i := 0; i < len(pods); i++ {
		res := <-ch
		results = append(results, res)
		if res.Err != nil {
			// TODO(proppy): report which containers failed the pod.
			glog.Infof("failed to start pod %q: %v", res.Pod.Name, res.Err)
			failedPods = append(failedPods, res.Pod.Name)
		} else {
			glog.Infof("started pod %q", res.Pod.Name)
		}
	}
	if len(failedPods) > 0 {
		return results, fmt.Errorf("error running pods: %v", failedPods)
	}
	glog.Infof("%d pods started", len(pods))
	return results, err
}

// runPod runs a single pod and wait until all containers are running.
func (kl *Kubelet) runPod(pod *api.Pod, retryDelay time.Duration) error {
	delay := retryDelay
	retry := 0
	for {
		pods, err := kl.containerRuntime.GetPods(false)
		if err != nil {
			return fmt.Errorf("failed to get kubelet pods: %v", err)
		}
		p := container.Pods(pods).FindPodByID(pod.UID)
		running, err := kl.isPodRunning(pod, p)
		if err != nil {
			return fmt.Errorf("failed to check pod status: %v", err)
		}
		if running {
			glog.Infof("pod %q containers running", pod.Name)
			return nil
		}
		glog.Infof("pod %q containers not running: syncing", pod.Name)

		podFullName := kubecontainer.GetPodFullName(pod)
		glog.Infof("Creating a mirror pod for static pod %q", podFullName)
		if err := kl.podManager.CreateMirrorPod(pod); err != nil {
			glog.Errorf("Failed creating a mirror pod %q: %v", podFullName, err)
		}
		mirrorPod, _ := kl.podManager.GetMirrorPodByPod(pod)

		if err = kl.syncPod(pod, mirrorPod, p, kubetypes.SyncPodUpdate); err != nil {
			return fmt.Errorf("error syncing pod: %v", err)
		}
		if retry >= runOnceMaxRetries {
			return fmt.Errorf("timeout error: pod %q containers not running after %d retries", pod.Name, runOnceMaxRetries)
		}
		// TODO(proppy): health checking would be better than waiting + checking the state at the next iteration.
		glog.Infof("pod %q containers synced, waiting for %v", pod.Name, delay)
		time.Sleep(delay)
		retry++
		delay *= runOnceRetryDelayBackoff
	}
}

// isPodRunning returns true if all containers of a manifest are running.
func (kl *Kubelet) isPodRunning(pod *api.Pod, runningPod container.Pod) (bool, error) {
	status, err := kl.containerRuntime.GetPodStatus(pod)
	if err != nil {
		glog.Infof("Failed to get the status of pod %q: %v", kubecontainer.GetPodFullName(pod), err)
		return false, err
	}
	for _, st := range status.ContainerStatuses {
		if st.State.Running == nil {
			glog.Infof("Container %q not running: %#v", st.Name, st.State)
			return false, nil
		}
	}
	return true, nil
}
<commit_msg>kubelet runonce: create data dirs<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubelet

import (
	"fmt"
	"os"
	"time"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/kubelet/container"
	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
)

const (
	runOnceManifestDelay     = 1 * time.Second
	runOnceMaxRetries        = 10
	runOnceRetryDelay        = 1 * time.Second
	runOnceRetryDelayBackoff = 2
)

type RunPodResult struct {
	Pod *api.Pod
	Err error
}

// RunOnce polls from one configuration update and run the associated pods.
func (kl *Kubelet) RunOnce(updates <-chan kubetypes.PodUpdate) ([]RunPodResult, error) {
	// Setup filesystem directories.
	if err := kl.setupDataDirs(); err != nil {
		return nil, err
	}

	// If the container logs directory does not exist, create it.
	if _, err := os.Stat(containerLogsDir); err != nil {
		if err := kl.os.Mkdir(containerLogsDir, 0755); err != nil {
			glog.Errorf("Failed to create directory %q: %v", containerLogsDir, err)
		}
	}

	select {
	case u := <-updates:
		glog.Infof("processing manifest with %d pods", len(u.Pods))
		result, err := kl.runOnce(u.Pods, runOnceRetryDelay)
		glog.Infof("finished processing %d pods", len(u.Pods))
		return result, err
	case <-time.After(runOnceManifestDelay):
		return nil, fmt.Errorf("no pod manifest update after %v", runOnceManifestDelay)
	}
}

// runOnce runs a given set of pods and returns their status.
func (kl *Kubelet) runOnce(pods []*api.Pod, retryDelay time.Duration) (results []RunPodResult, err error) {
	ch := make(chan RunPodResult)
	admitted := []*api.Pod{}
	for _, pod := range pods {
		// Check if we can admit the pod.
		if ok, reason, message := kl.canAdmitPod(append(admitted, pod), pod); !ok {
			kl.rejectPod(pod, reason, message)
		} else {
			admitted = append(admitted, pod)
		}
		go func(pod *api.Pod) {
			err := kl.runPod(pod, retryDelay)
			ch <- RunPodResult{pod, err}
		}(pod)
	}

	glog.Infof("waiting for %d pods", len(pods))
	failedPods := []string{}
	for i := 0; i < len(pods); i++ {
		res := <-ch
		results = append(results, res)
		if res.Err != nil {
			// TODO(proppy): report which containers failed the pod.
			glog.Infof("failed to start pod %q: %v", res.Pod.Name, res.Err)
			failedPods = append(failedPods, res.Pod.Name)
		} else {
			glog.Infof("started pod %q", res.Pod.Name)
		}
	}
	if len(failedPods) > 0 {
		return results, fmt.Errorf("error running pods: %v", failedPods)
	}
	glog.Infof("%d pods started", len(pods))
	return results, err
}

// runPod runs a single pod and wait until all containers are running.
func (kl *Kubelet) runPod(pod *api.Pod, retryDelay time.Duration) error {
	delay := retryDelay
	retry := 0
	for {
		pods, err := kl.containerRuntime.GetPods(false)
		if err != nil {
			return fmt.Errorf("failed to get kubelet pods: %v", err)
		}
		p := container.Pods(pods).FindPodByID(pod.UID)
		running, err := kl.isPodRunning(pod, p)
		if err != nil {
			return fmt.Errorf("failed to check pod status: %v", err)
		}
		if running {
			glog.Infof("pod %q containers running", pod.Name)
			return nil
		}
		glog.Infof("pod %q containers not running: syncing", pod.Name)

		podFullName := kubecontainer.GetPodFullName(pod)
		glog.Infof("Creating a mirror pod for static pod %q", podFullName)
		if err := kl.podManager.CreateMirrorPod(pod); err != nil {
			glog.Errorf("Failed creating a mirror pod %q: %v", podFullName, err)
		}
		mirrorPod, _ := kl.podManager.GetMirrorPodByPod(pod)

		if err = kl.syncPod(pod, mirrorPod, p, kubetypes.SyncPodUpdate); err != nil {
			return fmt.Errorf("error syncing pod: %v", err)
		}
		if retry >= runOnceMaxRetries {
			return fmt.Errorf("timeout error: pod %q containers not running after %d retries", pod.Name, runOnceMaxRetries)
		}
		// TODO(proppy): health checking would be better than waiting + checking the state at the next iteration.
		glog.Infof("pod %q containers synced, waiting for %v", pod.Name, delay)
		time.Sleep(delay)
		retry++
		delay *= runOnceRetryDelayBackoff
	}
}

// isPodRunning returns true if all containers of a manifest are running.
func (kl *Kubelet) isPodRunning(pod *api.Pod, runningPod container.Pod) (bool, error) {
	status, err := kl.containerRuntime.GetPodStatus(pod)
	if err != nil {
		glog.Infof("Failed to get the status of pod %q: %v", kubecontainer.GetPodFullName(pod), err)
		return false, err
	}
	for _, st := range status.ContainerStatuses {
		if st.State.Running == nil {
			glog.Infof("Container %q not running: %#v", st.Name, st.State)
			return false, nil
		}
	}
	return true, nil
}
<|endoftext|>


<commit_before>package pkg

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/davecgh/go-spew/spew"
	"github.com/ghodss/yaml"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"k8s.io/client-go/pkg/api"
	"k8s.io/client-go/pkg/api/resource"
	"k8s.io/client-go/pkg/runtime"
	"k8s.io/client-go/pkg/util/intstr"

	log "github.com/Sirupsen/logrus"
	api_v1 "k8s.io/client-go/pkg/api/v1"
	ext_v1beta1 "k8s.io/client-go/pkg/apis/extensions/v1beta1"

	// install api

	_ "k8s.io/client-go/pkg/api/install"
	_ "k8s.io/client-go/pkg/apis/extensions/install"
)

type Volume struct {
	api_v1.Volume `yaml:",inline"`
	Size          string   `yaml:"size"`
	AccessModes   []string `yaml:"accessModes"`
}

type Service struct {
	Name                    string `yaml:"name,omitempty"`
	api_v1.ServiceSpec      `yaml:",inline"`
	ext_v1beta1.IngressSpec `yaml:",inline"`
}

type App struct {
	Name              string            `yaml:"name"`
	Replicas          *int32            `yaml:"replicas,omitempty"`
	Expose            bool              `yaml:"expose,omitempty"`
	Labels            map[string]string `yaml:"labels,omitempty"`
	PersistentVolumes []Volume          `yaml:"persistentVolumes,omitempty"`
	ConfigData        map[string]string `yaml:"configData,omitempty"`
	Services          []Service         `yaml:"services,omitempty"`
	api_v1.PodSpec    `yaml:",inline"`
}

func ReadFile(f string) ([]byte, error) {
	data, err := ioutil.ReadFile(f)
	if err != nil {
		return nil, errors.Wrap(err, "file reading failed")
	}
	return data, nil
}

func Convert(v *viper.Viper, cmd *cobra.Command) error {

	for _, file := range strings.Split(v.GetStringSlice("files")[0], ",") {
		d, err := ReadFile(file)
		if err != nil {
			return errors.New(err.Error())
		}

		var app App
		err = yaml.Unmarshal(d, &app)
		if err != nil {
			return errors.Wrap(err, "could not unmarshal into internal struct")
		}
		log.Debugf("file: %s, object unmrashalled: %#v\n", file, app)

		runtimeObjects, err := CreateK8sObjects(&app)

		for _, runtimeObject := range runtimeObjects {
			gvk, isUnversioned, err := api.Scheme.ObjectKind(runtimeObject)
			if err != nil {
				return errors.Wrap(err, "ConvertToVersion failed")
			}
			if isUnversioned {
				return errors.New(fmt.Sprintf("ConvertToVersion failed: can't output unversioned type: %T", runtimeObject))
			}

			runtimeObject.GetObjectKind().SetGroupVersionKind(gvk)

			data, err := yaml.Marshal(runtimeObject)
			if err != nil {
				return errors.Wrap(err, "failed to marshal object")
			}

			writeObject := func(o runtime.Object, data []byte) error {
				_, err := fmt.Fprintln(os.Stdout, "---")
				if err != nil {
					return errors.Wrap(err, "could not print to STDOUT")
				}

				_, err = os.Stdout.Write(data)
				return errors.Wrap(err, "could not write to STDOUT")
			}

			err = writeObject(runtimeObject, data)
			if err != nil {
				return errors.Wrap(err, "failed to write object")
			}
		}
	}

	return nil
}

func getLabels(app *App) map[string]string {
	labels := map[string]string{"app": app.Name}
	return labels
}

func createServices(app *App) []runtime.Object {
	var svcs []runtime.Object
	for _, s := range app.Services {
		svc := &api_v1.Service{
			ObjectMeta: api_v1.ObjectMeta{
				Name:   s.Name,
				Labels: app.Labels,
			},
			Spec: s.ServiceSpec,
		}
		if len(svc.Spec.Selector) == 0 {
			svc.Spec.Selector = app.Labels
		}
		svcs = append(svcs, svc)

		if s.Type == api_v1.ServiceTypeLoadBalancer {
			// if more than one port given then we enforce user to specify in the http

			// autogenerate
			if len(s.Rules) == 1 && len(s.Ports) == 1 {
				http := s.Rules[0].HTTP
				if http == nil {
					http = &ext_v1beta1.HTTPIngressRuleValue{
						Paths: []ext_v1beta1.HTTPIngressPath{
							{
								Path: "/",
								Backend: ext_v1beta1.IngressBackend{
									ServiceName: s.Name,
									ServicePort: intstr.FromInt(int(s.Ports[0].Port)),
								},
							},
						},
					}
				}
				ing := &ext_v1beta1.Ingress{
					ObjectMeta: api_v1.ObjectMeta{
						Name:   s.Name,
						Labels: app.Labels,
					},
					Spec: ext_v1beta1.IngressSpec{
						Rules: []ext_v1beta1.IngressRule{
							{
								IngressRuleValue: ext_v1beta1.IngressRuleValue{
									HTTP: http,
								},
								Host: s.Rules[0].Host,
							},
						},
					},
				}
				svcs = append(svcs, ing)
			} else if len(s.Rules) == 1 && len(s.Ports) > 1 {
				if s.Rules[0].HTTP == nil {
					log.Warnf("No HTTP given for multiple ports")
				}
			} else if len(s.Rules) > 1 {
				ing := &ext_v1beta1.Ingress{
					ObjectMeta: api_v1.ObjectMeta{
						Name:   s.Name,
						Labels: app.Labels,
					},
					Spec: s.IngressSpec,
				}
				svcs = append(svcs, ing)
			}
		}
	}
	return svcs
}

func createDeployment(app *App) *ext_v1beta1.Deployment {
	// bare minimum deployment
	return &ext_v1beta1.Deployment{
		ObjectMeta: api_v1.ObjectMeta{
			Name:   app.Name,
			Labels: app.Labels,
		},
		Spec: ext_v1beta1.DeploymentSpec{
			Replicas: app.Replicas,
			Template: api_v1.PodTemplateSpec{
				ObjectMeta: api_v1.ObjectMeta{
					Name:   app.Name,
					Labels: app.Labels,
				},
				// get pod spec out of the original info
				Spec: api_v1.PodSpec(app.PodSpec),
			},
		},
	}
}

func isVolumeDefined(app *App, name string) bool {
	if i := searchVolumeIndex(app, name); i != -1 {
		return true
	}
	return false
}

func searchVolumeIndex(app *App, name string) int {
	for i, v := range app.PersistentVolumes {
		if name == v.Name {
			return i
		}
	}
	return -1
}

func createPVC(v *Volume) (*api_v1.PersistentVolumeClaim, error) {
	if v.Size == "" {
		v.Size = "100Mi"
	}
	size, err := resource.ParseQuantity(v.Size)
	if err != nil {
		return nil, errors.Wrap(err, "could not read volume size")
	}

	pvc := &api_v1.PersistentVolumeClaim{
		ObjectMeta: api_v1.ObjectMeta{
			Name: v.Name,
		},
		Spec: api_v1.PersistentVolumeClaimSpec{
			Resources: api_v1.ResourceRequirements{
				Requests: api_v1.ResourceList{
					api_v1.ResourceStorage: size,
				},
			},
		},
	}
	for _, mode := range v.AccessModes {
		switch mode {
		case "ReadWriteOnce":
			pvc.Spec.AccessModes = append(pvc.Spec.AccessModes, api_v1.ReadWriteOnce)
		case "ReadOnlyMany":
			pvc.Spec.AccessModes = append(pvc.Spec.AccessModes, api_v1.ReadOnlyMany)
		case "ReadWriteMany":
			pvc.Spec.AccessModes = append(pvc.Spec.AccessModes, api_v1.ReadWriteMany)
		}
	}
	if len(v.AccessModes) == 0 {
		pvc.Spec.AccessModes = []api_v1.PersistentVolumeAccessMode{api_v1.ReadWriteOnce}
	}

	return pvc, nil
}

func isAnyConfigMapRef(app *App) bool {
	for _, c := range app.Containers {
		for _, env := range c.Env {
			if env.ValueFrom != nil && env.ValueFrom.ConfigMapKeyRef != nil && env.ValueFrom.ConfigMapKeyRef.Name == app.Name {
				return true
			}
		}
	}
	for _, v := range app.Volumes {
		if v.ConfigMap != nil && v.ConfigMap.Name == app.Name {
			return true
		}
	}

	return false
}

func CreateK8sObjects(app *App) ([]runtime.Object, error) {

	var objects []runtime.Object

	if app.Labels == nil {
		app.Labels = getLabels(app)
	}

	svcs := createServices(app)

	var pvcs []runtime.Object
	for _, c := range app.Containers {
		for _, vm := range c.VolumeMounts {

			// User won't be giving this so we have to create it
			// so that the pod spec is complete
			podVolume := api_v1.Volume{
				Name: vm.Name,
				VolumeSource: api_v1.VolumeSource{
					PersistentVolumeClaim: &api_v1.PersistentVolumeClaimVolumeSource{
						ClaimName: vm.Name,
					},
				},
			}
			app.Volumes = append(app.Volumes, podVolume)

			if isVolumeDefined(app, vm.Name) {
				i := searchVolumeIndex(app, vm.Name)
				pvc, err := createPVC(&app.PersistentVolumes[i])
				if err != nil {
					return nil, errors.Wrap(err, "cannot create pvc")
				}
				pvcs = append(pvcs, pvc)
				continue
			}

			v := Volume{podVolume, "100Mi", []string{"ReadWriteOnce"}}
			app.PersistentVolumes = append(app.PersistentVolumes, v)
			pvc, err := createPVC(&v)
			if err != nil {
				return nil, errors.Wrap(err, "cannot create pvc")
			}
			pvcs = append(pvcs, pvc)
		}
	}

	// if only one container set name of it as app name
	if len(app.Containers) == 1 && app.Containers[0].Name == "" {
		app.Containers[0].Name = app.Name
	}

	var configMap *api_v1.ConfigMap
	if len(app.ConfigData) > 0 {
		configMap = &api_v1.ConfigMap{
			ObjectMeta: api_v1.ObjectMeta{
				Name: app.Name,
			},
			Data: app.ConfigData,
		}

		// add it to the envs if there is no configMapRef
		// we cannot re-create the entries for configMap
		// because there is no way we will know which container wants to use it
		if len(app.Containers) == 1 && !isAnyConfigMapRef(app) {
			// iterate over the data in the configMap
			for k, _ := range app.ConfigData {
				app.Containers[0].Env = append(app.Containers[0].Env,
					api_v1.EnvVar{
						Name: k,
						ValueFrom: &api_v1.EnvVarSource{
							ConfigMapKeyRef: &api_v1.ConfigMapKeySelector{
								api_v1.LocalObjectReference{
									Name: app.Name,
								},
								k,
							},
						},
					})
			}
		} else if len(app.Containers) > 1 && !isAnyConfigMapRef(app) {
			log.Warnf("You have defined a configMap but you have not mentioned where you gonna consume it!")
		}

	}

	deployment := createDeployment(app)
	objects = append(objects, deployment)
	log.Debugf("app: %s, deployment: %s\n", app.Name, spew.Sprint(deployment))

	if configMap != nil {
		objects = append(objects, configMap)
	}
	log.Debugf("app: %s, configMap: %s\n", app.Name, spew.Sprint(configMap))

	objects = append(objects, svcs...)
	log.Debugf("app: %s, service: %s\n", app.Name, spew.Sprint(svcs))

	objects = append(objects, pvcs...)
	log.Debugf("app: %s, pvc: %s\n", app.Name, spew.Sprint(pvcs))

	return objects, nil
}
<commit_msg>Add constants for volume sizes<commit_after>package pkg

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/davecgh/go-spew/spew"
	"github.com/ghodss/yaml"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"k8s.io/client-go/pkg/api"
	"k8s.io/client-go/pkg/api/resource"
	"k8s.io/client-go/pkg/runtime"
	"k8s.io/client-go/pkg/util/intstr"

	log "github.com/Sirupsen/logrus"
	api_v1 "k8s.io/client-go/pkg/api/v1"
	ext_v1beta1 "k8s.io/client-go/pkg/apis/extensions/v1beta1"

	// install api

	_ "k8s.io/client-go/pkg/api/install"
	_ "k8s.io/client-go/pkg/apis/extensions/install"
)

var (
	DefaultVolumeSize string = "100Mi"
	DefaultVolumeType string = "ReadWriteOnce"
)

type Volume struct {
	api_v1.Volume `yaml:",inline"`
	Size          string   `yaml:"size"`
	AccessModes   []string `yaml:"accessModes"`
}

type Service struct {
	Name                    string `yaml:"name,omitempty"`
	api_v1.ServiceSpec      `yaml:",inline"`
	ext_v1beta1.IngressSpec `yaml:",inline"`
}

type App struct {
	Name              string            `yaml:"name"`
	Replicas          *int32            `yaml:"replicas,omitempty"`
	Expose            bool              `yaml:"expose,omitempty"`
	Labels            map[string]string `yaml:"labels,omitempty"`
	PersistentVolumes []Volume          `yaml:"persistentVolumes,omitempty"`
	ConfigData        map[string]string `yaml:"configData,omitempty"`
	Services          []Service         `yaml:"services,omitempty"`
	api_v1.PodSpec    `yaml:",inline"`
}

func ReadFile(f string) ([]byte, error) {
	data, err := ioutil.ReadFile(f)
	if err != nil {
		return nil, errors.Wrap(err, "file reading failed")
	}
	return data, nil
}

func Convert(v *viper.Viper, cmd *cobra.Command) error {

	for _, file := range strings.Split(v.GetStringSlice("files")[0], ",") {
		d, err := ReadFile(file)
		if err != nil {
			return errors.New(err.Error())
		}

		var app App
		err = yaml.Unmarshal(d, &app)
		if err != nil {
			return errors.Wrap(err, "could not unmarshal into internal struct")
		}
		log.Debugf("file: %s, object unmrashalled: %#v\n", file, app)

		runtimeObjects, err := CreateK8sObjects(&app)

		for _, runtimeObject := range runtimeObjects {
			gvk, isUnversioned, err := api.Scheme.ObjectKind(runtimeObject)
			if err != nil {
				return errors.Wrap(err, "ConvertToVersion failed")
			}
			if isUnversioned {
				return errors.New(fmt.Sprintf("ConvertToVersion failed: can't output unversioned type: %T", runtimeObject))
			}

			runtimeObject.GetObjectKind().SetGroupVersionKind(gvk)

			data, err := yaml.Marshal(runtimeObject)
			if err != nil {
				return errors.Wrap(err, "failed to marshal object")
			}

			writeObject := func(o runtime.Object, data []byte) error {
				_, err := fmt.Fprintln(os.Stdout, "---")
				if err != nil {
					return errors.Wrap(err, "could not print to STDOUT")
				}

				_, err = os.Stdout.Write(data)
				return errors.Wrap(err, "could not write to STDOUT")
			}

			err = writeObject(runtimeObject, data)
			if err != nil {
				return errors.Wrap(err, "failed to write object")
			}
		}
	}

	return nil
}

func getLabels(app *App) map[string]string {
	labels := map[string]string{"app": app.Name}
	return labels
}

func createServices(app *App) []runtime.Object {
	var svcs []runtime.Object
	for _, s := range app.Services {
		svc := &api_v1.Service{
			ObjectMeta: api_v1.ObjectMeta{
				Name:   s.Name,
				Labels: app.Labels,
			},
			Spec: s.ServiceSpec,
		}
		if len(svc.Spec.Selector) == 0 {
			svc.Spec.Selector = app.Labels
		}
		svcs = append(svcs, svc)

		if s.Type == api_v1.ServiceTypeLoadBalancer {
			// if more than one port given then we enforce user to specify in the http

			// autogenerate
			if len(s.Rules) == 1 && len(s.Ports) == 1 {
				http := s.Rules[0].HTTP
				if http == nil {
					http = &ext_v1beta1.HTTPIngressRuleValue{
						Paths: []ext_v1beta1.HTTPIngressPath{
							{
								Path: "/",
								Backend: ext_v1beta1.IngressBackend{
									ServiceName: s.Name,
									ServicePort: intstr.FromInt(int(s.Ports[0].Port)),
								},
							},
						},
					}
				}
				ing := &ext_v1beta1.Ingress{
					ObjectMeta: api_v1.ObjectMeta{
						Name:   s.Name,
						Labels: app.Labels,
					},
					Spec: ext_v1beta1.IngressSpec{
						Rules: []ext_v1beta1.IngressRule{
							{
								IngressRuleValue: ext_v1beta1.IngressRuleValue{
									HTTP: http,
								},
								Host: s.Rules[0].Host,
							},
						},
					},
				}
				svcs = append(svcs, ing)
			} else if len(s.Rules) == 1 && len(s.Ports) > 1 {
				if s.Rules[0].HTTP == nil {
					log.Warnf("No HTTP given for multiple ports")
				}
			} else if len(s.Rules) > 1 {
				ing := &ext_v1beta1.Ingress{
					ObjectMeta: api_v1.ObjectMeta{
						Name:   s.Name,
						Labels: app.Labels,
					},
					Spec: s.IngressSpec,
				}
				svcs = append(svcs, ing)
			}
		}
	}
	return svcs
}

func createDeployment(app *App) *ext_v1beta1.Deployment {
	// bare minimum deployment
	return &ext_v1beta1.Deployment{
		ObjectMeta: api_v1.ObjectMeta{
			Name:   app.Name,
			Labels: app.Labels,
		},
		Spec: ext_v1beta1.DeploymentSpec{
			Replicas: app.Replicas,
			Template: api_v1.PodTemplateSpec{
				ObjectMeta: api_v1.ObjectMeta{
					Name:   app.Name,
					Labels: app.Labels,
				},
				// get pod spec out of the original info
				Spec: api_v1.PodSpec(app.PodSpec),
			},
		},
	}
}

func isVolumeDefined(app *App, name string) bool {
	if i := searchVolumeIndex(app, name); i != -1 {
		return true
	}
	return false
}

func searchVolumeIndex(app *App, name string) int {
	for i, v := range app.PersistentVolumes {
		if name == v.Name {
			return i
		}
	}
	return -1
}

func createPVC(v *Volume) (*api_v1.PersistentVolumeClaim, error) {
	if v.Size == "" {
		v.Size = DefaultVolumeSize
	}
	size, err := resource.ParseQuantity(v.Size)
	if err != nil {
		return nil, errors.Wrap(err, "could not read volume size")
	}

	pvc := &api_v1.PersistentVolumeClaim{
		ObjectMeta: api_v1.ObjectMeta{
			Name: v.Name,
		},
		Spec: api_v1.PersistentVolumeClaimSpec{
			Resources: api_v1.ResourceRequirements{
				Requests: api_v1.ResourceList{
					api_v1.ResourceStorage: size,
				},
			},
		},
	}
	for _, mode := range v.AccessModes {
		switch mode {
		case "ReadWriteOnce":
			pvc.Spec.AccessModes = append(pvc.Spec.AccessModes, api_v1.ReadWriteOnce)
		case "ReadOnlyMany":
			pvc.Spec.AccessModes = append(pvc.Spec.AccessModes, api_v1.ReadOnlyMany)
		case "ReadWriteMany":
			pvc.Spec.AccessModes = append(pvc.Spec.AccessModes, api_v1.ReadWriteMany)
		}
	}
	if len(v.AccessModes) == 0 {
		pvc.Spec.AccessModes = []api_v1.PersistentVolumeAccessMode{api_v1.ReadWriteOnce}
	}

	return pvc, nil
}

func isAnyConfigMapRef(app *App) bool {
	for _, c := range app.Containers {
		for _, env := range c.Env {
			if env.ValueFrom != nil && env.ValueFrom.ConfigMapKeyRef != nil && env.ValueFrom.ConfigMapKeyRef.Name == app.Name {
				return true
			}
		}
	}
	for _, v := range app.Volumes {
		if v.ConfigMap != nil && v.ConfigMap.Name == app.Name {
			return true
		}
	}

	return false
}

func CreateK8sObjects(app *App) ([]runtime.Object, error) {

	var objects []runtime.Object

	if app.Labels == nil {
		app.Labels = getLabels(app)
	}

	svcs := createServices(app)

	var pvcs []runtime.Object
	for _, c := range app.Containers {
		for _, vm := range c.VolumeMounts {

			// User won't be giving this so we have to create it
			// so that the pod spec is complete
			podVolume := api_v1.Volume{
				Name: vm.Name,
				VolumeSource: api_v1.VolumeSource{
					PersistentVolumeClaim: &api_v1.PersistentVolumeClaimVolumeSource{
						ClaimName: vm.Name,
					},
				},
			}
			app.Volumes = append(app.Volumes, podVolume)

			if isVolumeDefined(app, vm.Name) {
				i := searchVolumeIndex(app, vm.Name)
				pvc, err := createPVC(&app.PersistentVolumes[i])
				if err != nil {
					return nil, errors.Wrap(err, "cannot create pvc")
				}
				pvcs = append(pvcs, pvc)
				continue
			}

			// Retrieve a default configuration
			v := Volume{podVolume, DefaultVolumeSize, []string{DefaultVolumeType}}

			app.PersistentVolumes = append(app.PersistentVolumes, v)
			pvc, err := createPVC(&v)
			if err != nil {
				return nil, errors.Wrap(err, "cannot create pvc")
			}
			pvcs = append(pvcs, pvc)
		}
	}

	// if only one container set name of it as app name
	if len(app.Containers) == 1 && app.Containers[0].Name == "" {
		app.Containers[0].Name = app.Name
	}

	var configMap *api_v1.ConfigMap
	if len(app.ConfigData) > 0 {
		configMap = &api_v1.ConfigMap{
			ObjectMeta: api_v1.ObjectMeta{
				Name: app.Name,
			},
			Data: app.ConfigData,
		}

		// add it to the envs if there is no configMapRef
		// we cannot re-create the entries for configMap
		// because there is no way we will know which container wants to use it
		if len(app.Containers) == 1 && !isAnyConfigMapRef(app) {
			// iterate over the data in the configMap
			for k, _ := range app.ConfigData {
				app.Containers[0].Env = append(app.Containers[0].Env,
					api_v1.EnvVar{
						Name: k,
						ValueFrom: &api_v1.EnvVarSource{
							ConfigMapKeyRef: &api_v1.ConfigMapKeySelector{
								api_v1.LocalObjectReference{
									Name: app.Name,
								},
								k,
							},
						},
					})
			}
		} else if len(app.Containers) > 1 && !isAnyConfigMapRef(app) {
			log.Warnf("You have defined a configMap but you have not mentioned where you gonna consume it!")
		}

	}

	deployment := createDeployment(app)
	objects = append(objects, deployment)
	log.Debugf("app: %s, deployment: %s\n", app.Name, spew.Sprint(deployment))

	if configMap != nil {
		objects = append(objects, configMap)
	}
	log.Debugf("app: %s, configMap: %s\n", app.Name, spew.Sprint(configMap))

	objects = append(objects, svcs...)
	log.Debugf("app: %s, service: %s\n", app.Name, spew.Sprint(svcs))

	objects = append(objects, pvcs...)
	log.Debugf("app: %s, pvc: %s\n", app.Name, spew.Sprint(pvcs))

	return objects, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
	"flag"
	"fmt"
	"os"

	"k8s.io/klog/v2"

	"github.com/spf13/pflag"
)

// Options are the configurable parameters for kube-state-metrics.
type Options struct {
	Apiserver            string
	Kubeconfig           string
	Help                 bool
	Port                 int
	Host                 string
	TelemetryPort        int
	TelemetryHost        string
	TLSConfig            string
	Resources            ResourceSet
	Namespaces           NamespaceList
	NamespacesDenylist   NamespaceList
	Shard                int32
	TotalShards          int
	Pod                  string
	Namespace            string
	MetricDenylist       MetricSet
	MetricAllowlist      MetricSet
	MetricOptInList      MetricSet
	Version              bool
	AnnotationsAllowList LabelsAllowList
	LabelsAllowList      LabelsAllowList

	EnableGZIPEncoding bool

	UseAPIServerCache bool

	CustomResourceConfig     string
	CustomResourceConfigFile string

	flags *pflag.FlagSet
}

// NewOptions returns a new instance of `Options`.
func NewOptions() *Options {
	return &Options{
		Resources:            ResourceSet{},
		MetricAllowlist:      MetricSet{},
		MetricDenylist:       MetricSet{},
		MetricOptInList:      MetricSet{},
		AnnotationsAllowList: LabelsAllowList{},
		LabelsAllowList:      LabelsAllowList{},
	}
}

// AddFlags populated the Options struct from the command line arguments passed.
func (o *Options) AddFlags() {
	o.flags = pflag.NewFlagSet("", pflag.ExitOnError)
	// add klog flags
	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
	klog.InitFlags(klogFlags)
	o.flags.AddGoFlagSet(klogFlags)
	o.flags.Lookup("logtostderr").Value.Set("true")
	o.flags.Lookup("logtostderr").DefValue = "true"
	o.flags.Lookup("logtostderr").NoOptDefVal = "true"

	o.flags.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		o.flags.PrintDefaults()
	}

	o.flags.BoolVarP(&o.UseAPIServerCache, "use-apiserver-cache", "", false, "Sets resourceVersion=0 for ListWatch requests, using cached resources from the apiserver instead of an etcd quorum read.")
	o.flags.StringVar(&o.Apiserver, "apiserver", "", `The URL of the apiserver to use as a master`)
	o.flags.StringVar(&o.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig file")
	o.flags.StringVar(&o.TLSConfig, "tls-config", "", "Path to the TLS configuration file")
	o.flags.BoolVarP(&o.Help, "help", "h", false, "Print Help text")
	o.flags.IntVar(&o.Port, "port", 8080, `Port to expose metrics on.`)
	o.flags.StringVar(&o.Host, "host", "::", `Host to expose metrics on.`)
	o.flags.IntVar(&o.TelemetryPort, "telemetry-port", 8081, `Port to expose kube-state-metrics self metrics on.`)
	o.flags.StringVar(&o.TelemetryHost, "telemetry-host", "::", `Host to expose kube-state-metrics self metrics on.`)
	o.flags.Var(&o.Resources, "resources", fmt.Sprintf("Comma-separated list of Resources to be enabled. Defaults to %q", &DefaultResources))
	o.flags.Var(&o.Namespaces, "namespaces", fmt.Sprintf("Comma-separated list of namespaces to be enabled. Defaults to %q", &DefaultNamespaces))
	o.flags.Var(&o.NamespacesDenylist, "namespaces-denylist", "Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, only namespaces that are excluded in namespaces-denylist will be used.")
	o.flags.Var(&o.MetricAllowlist, "metric-allowlist", "Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.")
	o.flags.Var(&o.MetricDenylist, "metric-denylist", "Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.")
	o.flags.Var(&o.MetricOptInList, "metric-opt-in-list", "Comma-separated list of metrics which are opt-in and not enabled by default. This is in addition to the metric allow- and denylists")
	o.flags.Var(&o.AnnotationsAllowList, "metric-annotations-allowlist", "Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional annotations provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').")
	o.flags.Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional labels provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]').")
	o.flags.Int32Var(&o.Shard, "shard", int32(0), "The instances shard nominal (zero indexed) within the total number of shards. (default 0)")
	o.flags.IntVar(&o.TotalShards, "total-shards", 1, "The total number of shards. Sharding is disabled when total shards is set to 1.")

	autoshardingNotice := "When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice."

	o.flags.StringVar(&o.Pod, "pod", "", "Name of the pod that contains the kube-state-metrics container. "+autoshardingNotice)
	o.flags.StringVar(&o.Namespace, "pod-namespace", "", "Name of the namespace of the pod specified by --pod. "+autoshardingNotice)
	o.flags.BoolVarP(&o.Version, "version", "", false, "kube-state-metrics build version information")
	o.flags.BoolVar(&o.EnableGZIPEncoding, "enable-gzip-encoding", false, "Gzip responses when requested by clients via 'Accept-Encoding: gzip' header.")

	o.flags.StringVar(&o.CustomResourceConfig, "custom-resource-state-config", "", "Inline Custom Resource State Metrics config YAML")
	o.flags.StringVar(&o.CustomResourceConfigFile, "custom-resource-state-config-file", "", "Path to a Custom Resource State Metrics config file")
}

// Parse parses the flag definitions from the argument list.
func (o *Options) Parse() error {
	err := o.flags.Parse(os.Args)
	return err
}

// Usage is the function called when an error occurs while parsing flags.
func (o *Options) Usage() {
	o.flags.Usage()
}
<commit_msg>note experimental status<commit_after>/*
Copyright 2018 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
	"flag"
	"fmt"
	"os"

	"k8s.io/klog/v2"

	"github.com/spf13/pflag"
)

// Options are the configurable parameters for kube-state-metrics.
type Options struct {
	Apiserver            string
	Kubeconfig           string
	Help                 bool
	Port                 int
	Host                 string
	TelemetryPort        int
	TelemetryHost        string
	TLSConfig            string
	Resources            ResourceSet
	Namespaces           NamespaceList
	NamespacesDenylist   NamespaceList
	Shard                int32
	TotalShards          int
	Pod                  string
	Namespace            string
	MetricDenylist       MetricSet
	MetricAllowlist      MetricSet
	MetricOptInList      MetricSet
	Version              bool
	AnnotationsAllowList LabelsAllowList
	LabelsAllowList      LabelsAllowList

	EnableGZIPEncoding bool

	UseAPIServerCache bool

	CustomResourceConfig     string
	CustomResourceConfigFile string

	flags *pflag.FlagSet
}

// NewOptions returns a new instance of `Options`.
func NewOptions() *Options {
	return &Options{
		Resources:            ResourceSet{},
		MetricAllowlist:      MetricSet{},
		MetricDenylist:       MetricSet{},
		MetricOptInList:      MetricSet{},
		AnnotationsAllowList: LabelsAllowList{},
		LabelsAllowList:      LabelsAllowList{},
	}
}

// AddFlags populated the Options struct from the command line arguments passed.
func (o *Options) AddFlags() {
	o.flags = pflag.NewFlagSet("", pflag.ExitOnError)
	// add klog flags
	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
	klog.InitFlags(klogFlags)
	o.flags.AddGoFlagSet(klogFlags)
	o.flags.Lookup("logtostderr").Value.Set("true")
	o.flags.Lookup("logtostderr").DefValue = "true"
	o.flags.Lookup("logtostderr").NoOptDefVal = "true"

	o.flags.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		o.flags.PrintDefaults()
	}

	o.flags.BoolVarP(&o.UseAPIServerCache, "use-apiserver-cache", "", false, "Sets resourceVersion=0 for ListWatch requests, using cached resources from the apiserver instead of an etcd quorum read.")
	o.flags.StringVar(&o.Apiserver, "apiserver", "", `The URL of the apiserver to use as a master`)
	o.flags.StringVar(&o.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig file")
	o.flags.StringVar(&o.TLSConfig, "tls-config", "", "Path to the TLS configuration file")
	o.flags.BoolVarP(&o.Help, "help", "h", false, "Print Help text")
	o.flags.IntVar(&o.Port, "port", 8080, `Port to expose metrics on.`)
	o.flags.StringVar(&o.Host, "host", "::", `Host to expose metrics on.`)
	o.flags.IntVar(&o.TelemetryPort, "telemetry-port", 8081, `Port to expose kube-state-metrics self metrics on.`)
	o.flags.StringVar(&o.TelemetryHost, "telemetry-host", "::", `Host to expose kube-state-metrics self metrics on.`)
	o.flags.Var(&o.Resources, "resources", fmt.Sprintf("Comma-separated list of Resources to be enabled. Defaults to %q", &DefaultResources))
	o.flags.Var(&o.Namespaces, "namespaces", fmt.Sprintf("Comma-separated list of namespaces to be enabled. Defaults to %q", &DefaultNamespaces))
	o.flags.Var(&o.NamespacesDenylist, "namespaces-denylist", "Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, only namespaces that are excluded in namespaces-denylist will be used.")
	o.flags.Var(&o.MetricAllowlist, "metric-allowlist", "Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.")
	o.flags.Var(&o.MetricDenylist, "metric-denylist", "Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.")
	o.flags.Var(&o.MetricOptInList, "metric-opt-in-list", "Comma-separated list of metrics which are opt-in and not enabled by default. This is in addition to the metric allow- and denylists")
	o.flags.Var(&o.AnnotationsAllowList, "metric-annotations-allowlist", "Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional annotations provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').")
	o.flags.Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional labels provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]').")
	o.flags.Int32Var(&o.Shard, "shard", int32(0), "The instances shard nominal (zero indexed) within the total number of shards. (default 0)")
	o.flags.IntVar(&o.TotalShards, "total-shards", 1, "The total number of shards. Sharding is disabled when total shards is set to 1.")

	autoshardingNotice := "When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice."

	o.flags.StringVar(&o.Pod, "pod", "", "Name of the pod that contains the kube-state-metrics container. "+autoshardingNotice)
	o.flags.StringVar(&o.Namespace, "pod-namespace", "", "Name of the namespace of the pod specified by --pod. "+autoshardingNotice)
	o.flags.BoolVarP(&o.Version, "version", "", false, "kube-state-metrics build version information")
	o.flags.BoolVar(&o.EnableGZIPEncoding, "enable-gzip-encoding", false, "Gzip responses when requested by clients via 'Accept-Encoding: gzip' header.")

	o.flags.StringVar(&o.CustomResourceConfig, "custom-resource-state-config", "", "Inline Custom Resource State Metrics config YAML")
	o.flags.StringVar(&o.CustomResourceConfigFile, "custom-resource-state-config-file", "", "Path to a Custom Resource State Metrics config file (experimental)")
}

// Parse parses the flag definitions from the argument list.
func (o *Options) Parse() error {
	err := o.flags.Parse(os.Args)
	return err
}

// Usage is the function called when an error occurs while parsing flags.
func (o *Options) Usage() {
	o.flags.Usage()
}
<|endoftext|>


<commit_before>// Package pidfile provides structure and helper functions to create and remove
// PID file. A PID file is usually a file used to store the process ID of a
// running process.
package pidfile

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
)

// PIDFile is a file used to store the process ID of a running process.
type PIDFile struct {
	path string
}

func checkPIDFileAlreadyExists(path string) error {
	if pidString, err := ioutil.ReadFile(path); err == nil {
		if pid, err := strconv.Atoi(string(pidString)); err == nil {
			if _, err := os.Stat(filepath.Join("/proc", string(pid))); err == nil {
				return fmt.Errorf("pid file found, ensure docker is not running or delete %s", path)
			}
		}
	}
	return nil
}

// New creates a PIDfile using the specified path.
func New(path string) (*PIDFile, error) {
	if err := checkPIDFileAlreadyExists(path); err != nil {
		return nil, err
	}
	if err := ioutil.WriteFile(path, []byte(fmt.Sprintf("%d", os.Getpid())), 0644); err != nil {
		return nil, err
	}

	return &PIDFile{path: path}, nil
}

// Remove removes the PIDFile.
func (file PIDFile) Remove() error {
	if err := os.Remove(file.path); err != nil {
		return err
	}
	return nil
}
<commit_msg>fix pidfile, pid is num use '/proc + string(pid)' can't found it<commit_after>// Package pidfile provides structure and helper functions to create and remove
// PID file. A PID file is usually a file used to store the process ID of a
// running process.
package pidfile

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

// PIDFile is a file used to store the process ID of a running process.
type PIDFile struct {
	path string
}

func checkPIDFileAlreadyExists(path string) error {
	if pidByte, err := ioutil.ReadFile(path); err == nil {
		pidString := strings.TrimSpace(string(pidByte))
		if pid, err := strconv.Atoi(pidString); err == nil {
			if _, err := os.Stat(filepath.Join("/proc", strconv.Itoa(pid))); err == nil {
				return fmt.Errorf("pid file found, ensure docker is not running or delete %s", path)
			}
		}
	}
	return nil
}

// New creates a PIDfile using the specified path.
func New(path string) (*PIDFile, error) {
	if err := checkPIDFileAlreadyExists(path); err != nil {
		return nil, err
	}
	if err := ioutil.WriteFile(path, []byte(fmt.Sprintf("%d", os.Getpid())), 0644); err != nil {
		return nil, err
	}

	return &PIDFile{path: path}, nil
}

// Remove removes the PIDFile.
func (file PIDFile) Remove() error {
	if err := os.Remove(file.path); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package project

var (
	description        = "The aws-operator manages Kubernetes clusters running on AWS."
	gitSHA             = "n/a"
	name        string = "aws-operator"
	source      string = "https://github.com/giantswarm/aws-operator"
	version            = "10.17.0"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<commit_msg>Bump version to 10.17.1-dev (#3263)<commit_after>package project

var (
	description        = "The aws-operator manages Kubernetes clusters running on AWS."
	gitSHA             = "n/a"
	name        string = "aws-operator"
	source      string = "https://github.com/giantswarm/aws-operator"
	version            = "10.17.1-dev"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<|endoftext|>


<commit_before>package project

var (
	description        = "The aws-operator manages Kubernetes clusters running on AWS."
	gitSHA             = "n/a"
	name        string = "aws-operator"
	source      string = "https://github.com/giantswarm/aws-operator"
	version            = "10.0.0"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<commit_msg>Bump version to 10.0.1-dev (#2941)<commit_after>package project

var (
	description        = "The aws-operator manages Kubernetes clusters running on AWS."
	gitSHA             = "n/a"
	name        string = "aws-operator"
	source      string = "https://github.com/giantswarm/aws-operator"
	version            = "10.0.1-dev"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<|endoftext|>


<commit_before>package project

var (
	description string = "The azure-operator manages Kubernetes clusters on Azure."
	gitSHA             = "n/a"
	name        string = "azure-operator"
	source      string = "https://github.com/giantswarm/azure-operator"
	version            = "5.0.1-rollwithk8s"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<commit_msg>reset project.go<commit_after>package project

var (
	description string = "The azure-operator manages Kubernetes clusters on Azure."
	gitSHA             = "n/a"
	name        string = "azure-operator"
	source      string = "https://github.com/giantswarm/azure-operator"
	version            = "5.0.1-dev"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package rollsum implements rolling checksums similar to apenwarr's bup, which
// is similar to librsync.
//
// The bup project is at https://github.com/apenwarr/bup and its splitting in
// particular is at https://github.com/apenwarr/bup/blob/master/lib/bup/bupsplit.c
package rollsum // import "perkeep.org/pkg/rollsum"

import ()

const windowSize = 64
const charOffset = 31

const blobBits = 13
const blobSize = 1 << blobBits // 8k

type RollSum struct {
	s1, s2 uint32
	window [windowSize]uint8
	wofs   int
}

func New() *RollSum {
	return &RollSum{
		s1: windowSize * charOffset,
		s2: windowSize * (windowSize - 1) * charOffset,
	}
}

func (rs *RollSum) add(drop, add uint8) {
	rs.s1 += uint32(add) - uint32(drop)
	rs.s2 += rs.s1 - uint32(windowSize)*(uint32(drop)+charOffset)
}

func (rs *RollSum) Roll(ch byte) {
	rs.add(rs.window[rs.wofs], ch)
	rs.window[rs.wofs] = ch
	rs.wofs = (rs.wofs + 1) % windowSize
}

// OnSplit returns whether at least 13 consecutive trailing bits of
// the current checksum are set the same way.
func (rs *RollSum) OnSplit() bool {
	return (rs.s2 & (blobSize - 1)) == ((^0) & (blobSize - 1))
}

// OnSplit returns whether at least n consecutive trailing bits
// of the current checksum are set the same way.
func (rs *RollSum) OnSplitWithBits(n uint32) bool {
	mask := (uint32(1) << n) - 1
	return rs.s2&mask == (^uint32(0))&mask
}

func (rs *RollSum) Bits() int {
	bits := blobBits
	rsum := rs.Digest()
	rsum >>= blobBits
	for ; (rsum>>1)&1 != 0; bits++ {
		rsum >>= 1
	}
	return bits
}

func (rs *RollSum) Digest() uint32 {
	return (rs.s1 << 16) | (rs.s2 & 0xffff)
}
<commit_msg>rollsum: help the compiler<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package rollsum implements rolling checksums similar to apenwarr's bup, which
// is similar to librsync.
//
// The bup project is at https://github.com/apenwarr/bup and its splitting in
// particular is at https://github.com/apenwarr/bup/blob/master/lib/bup/bupsplit.c
package rollsum // import "perkeep.org/pkg/rollsum"

const windowSize = 64 // Roll assumes windowSize is a power of 2
const charOffset = 31

const blobBits = 13
const blobSize = 1 << blobBits // 8k

type RollSum struct {
	s1, s2 uint32
	window [windowSize]uint8
	wofs   int
}

func New() *RollSum {
	return &RollSum{
		s1: windowSize * charOffset,
		s2: windowSize * (windowSize - 1) * charOffset,
	}
}

func (rs *RollSum) add(drop, add uint32) {
	s1 := rs.s1 + add - drop
	rs.s1 = s1
	rs.s2 += s1 - uint32(windowSize)*(drop+charOffset)
}

func (rs *RollSum) Roll(ch byte) {
	wp := &rs.window[rs.wofs]
	rs.add(uint32(*wp), uint32(ch))
	*wp = ch
	rs.wofs = (rs.wofs + 1) & (windowSize - 1)
}

// OnSplit returns whether at least 13 consecutive trailing bits of
// the current checksum are set the same way.
func (rs *RollSum) OnSplit() bool {
	return (rs.s2 & (blobSize - 1)) == ((^0) & (blobSize - 1))
}

// OnSplit returns whether at least n consecutive trailing bits
// of the current checksum are set the same way.
func (rs *RollSum) OnSplitWithBits(n uint32) bool {
	mask := (uint32(1) << n) - 1
	return rs.s2&mask == (^uint32(0))&mask
}

func (rs *RollSum) Bits() int {
	bits := blobBits
	rsum := rs.Digest()
	rsum >>= blobBits
	for ; (rsum>>1)&1 != 0; bits++ {
		rsum >>= 1
	}
	return bits
}

func (rs *RollSum) Digest() uint32 {
	return (rs.s1 << 16) | (rs.s2 & 0xffff)
}
<|endoftext|>


<commit_before>package secrets

import (
	"bufio"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/containers/buildah/pkg/umask"
	"github.com/containers/storage/pkg/idtools"
	rspec "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/opencontainers/selinux/go-selinux/label"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

var (
	// DefaultMountsFile holds the default mount paths in the form
	// "host_path:container_path"
	DefaultMountsFile = "/usr/share/containers/mounts.conf"
	// OverrideMountsFile holds the default mount paths in the form
	// "host_path:container_path" overridden by the user
	OverrideMountsFile = "/etc/containers/mounts.conf"
	// UserOverrideMountsFile holds the default mount paths in the form
	// "host_path:container_path" overridden by the rootless user
	UserOverrideMountsFile = filepath.Join(os.Getenv("HOME"), ".config/containers/mounts.conf")
)

// secretData stores the name of the file and the content read from it
type secretData struct {
	name    string
	data    []byte
	mode    os.FileMode
	dirMode os.FileMode
}

// saveTo saves secret data to given directory
func (s secretData) saveTo(dir string) error {
	path := filepath.Join(dir, s.name)
	if err := os.MkdirAll(filepath.Dir(path), s.dirMode); err != nil && !os.IsExist(err) {
		return err
	}
	return ioutil.WriteFile(path, s.data, s.mode)
}

func readAll(root, prefix string, parentMode os.FileMode) ([]secretData, error) {
	path := filepath.Join(root, prefix)

	data := []secretData{}

	files, err := ioutil.ReadDir(path)
	if err != nil {
		if os.IsNotExist(err) {
			return data, nil
		}

		return nil, err
	}

	for _, f := range files {
		fileData, err := readFileOrDir(root, filepath.Join(prefix, f.Name()), parentMode)
		if err != nil {
			// If the file did not exist, might be a dangling symlink
			// Ignore the error
			if os.IsNotExist(err) {
				continue
			}
			return nil, err
		}
		data = append(data, fileData...)
	}

	return data, nil
}

func readFileOrDir(root, name string, parentMode os.FileMode) ([]secretData, error) {
	path := filepath.Join(root, name)

	s, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	if s.IsDir() {
		dirData, err := readAll(root, name, s.Mode())
		if err != nil {
			return nil, err
		}
		return dirData, nil
	}
	bytes, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	return []secretData{{
		name:    name,
		data:    bytes,
		mode:    s.Mode(),
		dirMode: parentMode,
	}}, nil
}

func getHostSecretData(hostDir string, mode os.FileMode) ([]secretData, error) {
	var allSecrets []secretData
	hostSecrets, err := readAll(hostDir, "", mode)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to read secrets from %q", hostDir)
	}
	return append(allSecrets, hostSecrets...), nil
}

func getMounts(filePath string) []string {
	file, err := os.Open(filePath)
	if err != nil {
		// This is expected on most systems
		logrus.Debugf("file %q not found, skipping...", filePath)
		return nil
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	if err = scanner.Err(); err != nil {
		logrus.Errorf("error reading file %q, %v skipping...", filePath, err)
		return nil
	}
	var mounts []string
	for scanner.Scan() {
		if strings.HasPrefix(strings.TrimSpace(scanner.Text()), "/") {
			mounts = append(mounts, scanner.Text())
		} else {
			logrus.Debugf("skipping unrecognized mount in %v: %q",
				filePath, scanner.Text())
		}
	}
	return mounts
}

// getHostAndCtrDir separates the host:container paths
func getMountsMap(path string) (string, string, error) {
	arr := strings.SplitN(path, ":", 2)
	switch len(arr) {
	case 1:
		return arr[0], arr[0], nil
	case 2:
		return arr[0], arr[1], nil
	}
	return "", "", errors.Errorf("unable to get host and container dir from path: %s", path)
}

// SecretMounts copies, adds, and mounts the secrets to the container root filesystem
// Deprecated, Please use SecretMountWithUIDGID
func SecretMounts(mountLabel, containerWorkingDir, mountFile string, rootless, disableFips bool) []rspec.Mount {
	return SecretMountsWithUIDGID(mountLabel, containerWorkingDir, mountFile, containerWorkingDir, 0, 0, rootless, disableFips)
}

// SecretMountsWithUIDGID copies, adds, and mounts the secrets to the container root filesystem
// mountLabel: MAC/SELinux label for container content
// containerWorkingDir: Private data for storing secrets on the host mounted in container.
// mountFile: Additional mount points required for the container.
// mountPoint: Container image mountpoint
// uid: to assign to content created for secrets
// gid: to assign to content created for secrets
// rootless: indicates whether container is running in rootless mode
// disableFips: indicates whether system should ignore fips mode
func SecretMountsWithUIDGID(mountLabel, containerWorkingDir, mountFile, mountPoint string, uid, gid int, rootless, disableFips bool) []rspec.Mount {
	var (
		secretMounts []rspec.Mount
		mountFiles   []string
	)
	// Add secrets from paths given in the mounts.conf files
	// mountFile will have a value if the hidden --default-mounts-file flag is set
	// Note for testing purposes only
	if mountFile == "" {
		mountFiles = append(mountFiles, []string{OverrideMountsFile, DefaultMountsFile}...)
		if rootless {
			mountFiles = append([]string{UserOverrideMountsFile}, mountFiles...)
		}
	} else {
		mountFiles = append(mountFiles, mountFile)
	}
	for _, file := range mountFiles {
		if _, err := os.Stat(file); err == nil {
			mounts, err := addSecretsFromMountsFile(file, mountLabel, containerWorkingDir, uid, gid)
			if err != nil {
				logrus.Warnf("error mounting secrets, skipping entry in %s: %v", file, err)
			}
			secretMounts = mounts
			break
		}
	}

	// Only add FIPS secret mount if disableFips=false
	if disableFips {
		return secretMounts
	}
	// Add FIPS mode secret if /etc/system-fips exists on the host
	_, err := os.Stat("/etc/system-fips")
	if err == nil {
		if err := addFIPSModeSecret(&secretMounts, containerWorkingDir, mountPoint, mountLabel, uid, gid); err != nil {
			logrus.Errorf("error adding FIPS mode secret to container: %v", err)
		}
	} else if os.IsNotExist(err) {
		logrus.Debug("/etc/system-fips does not exist on host, not mounting FIPS mode secret")
	} else {
		logrus.Errorf("stat /etc/system-fips failed for FIPS mode secret: %v", err)
	}
	return secretMounts
}

func rchown(chowndir string, uid, gid int) error {
	return filepath.Walk(chowndir, func(filePath string, f os.FileInfo, err error) error {
		return os.Lchown(filePath, uid, gid)
	})
}

// addSecretsFromMountsFile copies the contents of host directory to container directory
// and returns a list of mounts
func addSecretsFromMountsFile(filePath, mountLabel, containerWorkingDir string, uid, gid int) ([]rspec.Mount, error) {
	var mounts []rspec.Mount
	defaultMountsPaths := getMounts(filePath)
	for _, path := range defaultMountsPaths {
		hostDirOrFile, ctrDirOrFile, err := getMountsMap(path)
		if err != nil {
			return nil, err
		}
		// skip if the hostDirOrFile path doesn't exist
		fileInfo, err := os.Stat(hostDirOrFile)
		if err != nil {
			if os.IsNotExist(err) {
				logrus.Warnf("Path %q from %q doesn't exist, skipping", hostDirOrFile, filePath)
				continue
			}
			return nil, errors.Wrapf(err, "failed to stat %q", hostDirOrFile)
		}

		ctrDirOrFileOnHost := filepath.Join(containerWorkingDir, ctrDirOrFile)

		// In the event of a restart, don't want to copy secrets over again as they already would exist in ctrDirOrFileOnHost
		_, err = os.Stat(ctrDirOrFileOnHost)
		if os.IsNotExist(err) {

			hostDirOrFile, err = resolveSymbolicLink(hostDirOrFile)
			if err != nil {
				return nil, err
			}

			// Don't let the umask have any influence on the file and directory creation
			oldUmask := umask.SetUmask(0)
			defer umask.SetUmask(oldUmask)

			switch mode := fileInfo.Mode(); {
			case mode.IsDir():
				if err = os.MkdirAll(ctrDirOrFileOnHost, mode.Perm()); err != nil {
					return nil, errors.Wrapf(err, "making container directory %q failed", ctrDirOrFileOnHost)
				}
				data, err := getHostSecretData(hostDirOrFile, mode.Perm())
				if err != nil {
					return nil, errors.Wrapf(err, "getting host secret data failed")
				}
				for _, s := range data {
					if err := s.saveTo(ctrDirOrFileOnHost); err != nil {
						return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOrFileOnHost)
					}
				}
			case mode.IsRegular():
				data, err := readFileOrDir("", hostDirOrFile, mode.Perm())
				if err != nil {
					return nil, errors.Wrapf(err, "error reading file %q", hostDirOrFile)

				}
				for _, s := range data {
					if err := os.MkdirAll(filepath.Dir(ctrDirOrFileOnHost), s.dirMode); err != nil {
						return nil, err
					}
					if err := ioutil.WriteFile(ctrDirOrFileOnHost, s.data, s.mode); err != nil {
						return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOrFileOnHost)
					}
				}
			default:
				return nil, errors.Errorf("unsupported file type for: %q", hostDirOrFile)
			}

			err = label.Relabel(ctrDirOrFileOnHost, mountLabel, false)
			if err != nil {
				return nil, errors.Wrap(err, "error applying correct labels")
			}
			if uid != 0 || gid != 0 {
				if err := rchown(ctrDirOrFileOnHost, uid, gid); err != nil {
					return nil, err
				}
			}
		} else if err != nil {
			return nil, errors.Wrapf(err, "error getting status of %q", ctrDirOrFileOnHost)
		}

		m := rspec.Mount{
			Source:      ctrDirOrFileOnHost,
			Destination: ctrDirOrFile,
			Type:        "bind",
			Options:     []string{"bind", "rprivate"},
		}

		mounts = append(mounts, m)
	}
	return mounts, nil
}

// addFIPSModeSecret creates /run/secrets/system-fips in the container
// root filesystem if /etc/system-fips exists on hosts.
// This enables the container to be FIPS compliant and run openssl in
// FIPS mode as the host is also in FIPS mode.
func addFIPSModeSecret(mounts *[]rspec.Mount, containerWorkingDir, mountPoint, mountLabel string, uid, gid int) error {
	secretsDir := "/run/secrets"
	ctrDirOnHost := filepath.Join(containerWorkingDir, secretsDir)
	if _, err := os.Stat(ctrDirOnHost); os.IsNotExist(err) {
		if err = idtools.MkdirAllAs(ctrDirOnHost, 0755, uid, gid); err != nil {
			return errors.Wrapf(err, "making container directory %q on host failed", ctrDirOnHost)
		}
		if err = label.Relabel(ctrDirOnHost, mountLabel, false); err != nil {
			return errors.Wrapf(err, "error applying correct labels on %q", ctrDirOnHost)
		}
	}
	fipsFile := filepath.Join(ctrDirOnHost, "system-fips")
	// In the event of restart, it is possible for the FIPS mode file to already exist
	if _, err := os.Stat(fipsFile); os.IsNotExist(err) {
		file, err := os.Create(fipsFile)
		if err != nil {
			return errors.Wrapf(err, "error creating system-fips file in container for FIPS mode")
		}
		defer file.Close()
	}

	if !mountExists(*mounts, secretsDir) {
		m := rspec.Mount{
			Source:      ctrDirOnHost,
			Destination: secretsDir,
			Type:        "bind",
			Options:     []string{"bind", "rprivate"},
		}
		*mounts = append(*mounts, m)
	}

	srcBackendDir := "/usr/share/crypto-policies/FIPS"
	destDir := "/etc/crypto-policies/back-ends"
	srcOnHost := filepath.Join(mountPoint, srcBackendDir)
	if _, err := os.Stat(srcOnHost); err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return errors.Wrapf(err, "failed to stat FIPS Backend directory %q", ctrDirOnHost)
	}

	if !mountExists(*mounts, destDir) {
		m := rspec.Mount{
			Source:      srcOnHost,
			Destination: destDir,
			Type:        "bind",
			Options:     []string{"bind", "rprivate"},
		}
		*mounts = append(*mounts, m)
	}
	return nil
}

// mountExists checks if a mount already exists in the spec
func mountExists(mounts []rspec.Mount, dest string) bool {
	for _, mount := range mounts {
		if mount.Destination == dest {
			return true
		}
	}
	return false
}

// resolveSymbolicLink resolves a possbile symlink path. If the path is a symlink, returns resolved
// path; if not, returns the original path.
func resolveSymbolicLink(path string) (string, error) {
	info, err := os.Lstat(path)
	if err != nil {
		return "", err
	}
	if info.Mode()&os.ModeSymlink != os.ModeSymlink {
		return path, nil
	}
	return filepath.EvalSymlinks(path)
}
<commit_msg>revert #2246 FIPS mode change<commit_after>package secrets

import (
	"bufio"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/containers/buildah/pkg/umask"
	"github.com/containers/storage/pkg/idtools"
	rspec "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/opencontainers/selinux/go-selinux/label"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

var (
	// DefaultMountsFile holds the default mount paths in the form
	// "host_path:container_path"
	DefaultMountsFile = "/usr/share/containers/mounts.conf"
	// OverrideMountsFile holds the default mount paths in the form
	// "host_path:container_path" overridden by the user
	OverrideMountsFile = "/etc/containers/mounts.conf"
	// UserOverrideMountsFile holds the default mount paths in the form
	// "host_path:container_path" overridden by the rootless user
	UserOverrideMountsFile = filepath.Join(os.Getenv("HOME"), ".config/containers/mounts.conf")
)

// secretData stores the name of the file and the content read from it
type secretData struct {
	name    string
	data    []byte
	mode    os.FileMode
	dirMode os.FileMode
}

// saveTo saves secret data to given directory
func (s secretData) saveTo(dir string) error {
	path := filepath.Join(dir, s.name)
	if err := os.MkdirAll(filepath.Dir(path), s.dirMode); err != nil && !os.IsExist(err) {
		return err
	}
	return ioutil.WriteFile(path, s.data, s.mode)
}

func readAll(root, prefix string, parentMode os.FileMode) ([]secretData, error) {
	path := filepath.Join(root, prefix)

	data := []secretData{}

	files, err := ioutil.ReadDir(path)
	if err != nil {
		if os.IsNotExist(err) {
			return data, nil
		}

		return nil, err
	}

	for _, f := range files {
		fileData, err := readFileOrDir(root, filepath.Join(prefix, f.Name()), parentMode)
		if err != nil {
			// If the file did not exist, might be a dangling symlink
			// Ignore the error
			if os.IsNotExist(err) {
				continue
			}
			return nil, err
		}
		data = append(data, fileData...)
	}

	return data, nil
}

func readFileOrDir(root, name string, parentMode os.FileMode) ([]secretData, error) {
	path := filepath.Join(root, name)

	s, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	if s.IsDir() {
		dirData, err := readAll(root, name, s.Mode())
		if err != nil {
			return nil, err
		}
		return dirData, nil
	}
	bytes, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	return []secretData{{
		name:    name,
		data:    bytes,
		mode:    s.Mode(),
		dirMode: parentMode,
	}}, nil
}

func getHostSecretData(hostDir string, mode os.FileMode) ([]secretData, error) {
	var allSecrets []secretData
	hostSecrets, err := readAll(hostDir, "", mode)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to read secrets from %q", hostDir)
	}
	return append(allSecrets, hostSecrets...), nil
}

func getMounts(filePath string) []string {
	file, err := os.Open(filePath)
	if err != nil {
		// This is expected on most systems
		logrus.Debugf("file %q not found, skipping...", filePath)
		return nil
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	if err = scanner.Err(); err != nil {
		logrus.Errorf("error reading file %q, %v skipping...", filePath, err)
		return nil
	}
	var mounts []string
	for scanner.Scan() {
		if strings.HasPrefix(strings.TrimSpace(scanner.Text()), "/") {
			mounts = append(mounts, scanner.Text())
		} else {
			logrus.Debugf("skipping unrecognized mount in %v: %q",
				filePath, scanner.Text())
		}
	}
	return mounts
}

// getHostAndCtrDir separates the host:container paths
func getMountsMap(path string) (string, string, error) {
	arr := strings.SplitN(path, ":", 2)
	switch len(arr) {
	case 1:
		return arr[0], arr[0], nil
	case 2:
		return arr[0], arr[1], nil
	}
	return "", "", errors.Errorf("unable to get host and container dir from path: %s", path)
}

// SecretMounts copies, adds, and mounts the secrets to the container root filesystem
// Deprecated, Please use SecretMountWithUIDGID
func SecretMounts(mountLabel, containerWorkingDir, mountFile string, rootless, disableFips bool) []rspec.Mount {
	return SecretMountsWithUIDGID(mountLabel, containerWorkingDir, mountFile, containerWorkingDir, 0, 0, rootless, disableFips)
}

// SecretMountsWithUIDGID copies, adds, and mounts the secrets to the container root filesystem
// mountLabel: MAC/SELinux label for container content
// containerWorkingDir: Private data for storing secrets on the host mounted in container.
// mountFile: Additional mount points required for the container.
// mountPoint: Container image mountpoint
// uid: to assign to content created for secrets
// gid: to assign to content created for secrets
// rootless: indicates whether container is running in rootless mode
// disableFips: indicates whether system should ignore fips mode
func SecretMountsWithUIDGID(mountLabel, containerWorkingDir, mountFile, mountPoint string, uid, gid int, rootless, disableFips bool) []rspec.Mount {
	var (
		secretMounts []rspec.Mount
		mountFiles   []string
	)
	// Add secrets from paths given in the mounts.conf files
	// mountFile will have a value if the hidden --default-mounts-file flag is set
	// Note for testing purposes only
	if mountFile == "" {
		mountFiles = append(mountFiles, []string{OverrideMountsFile, DefaultMountsFile}...)
		if rootless {
			mountFiles = append([]string{UserOverrideMountsFile}, mountFiles...)
		}
	} else {
		mountFiles = append(mountFiles, mountFile)
	}
	for _, file := range mountFiles {
		if _, err := os.Stat(file); err == nil {
			mounts, err := addSecretsFromMountsFile(file, mountLabel, containerWorkingDir, uid, gid)
			if err != nil {
				logrus.Warnf("error mounting secrets, skipping entry in %s: %v", file, err)
			}
			secretMounts = mounts
			break
		}
	}

	// Only add FIPS secret mount if disableFips=false
	if disableFips {
		return secretMounts
	}
	// Add FIPS mode secret if /etc/system-fips exists on the host
	_, err := os.Stat("/etc/system-fips")
	if err == nil {
		if err := addFIPSModeSecret(&secretMounts, containerWorkingDir, mountPoint, mountLabel, uid, gid); err != nil {
			logrus.Errorf("error adding FIPS mode secret to container: %v", err)
		}
	} else if os.IsNotExist(err) {
		logrus.Debug("/etc/system-fips does not exist on host, not mounting FIPS mode secret")
	} else {
		logrus.Errorf("stat /etc/system-fips failed for FIPS mode secret: %v", err)
	}
	return secretMounts
}

func rchown(chowndir string, uid, gid int) error {
	return filepath.Walk(chowndir, func(filePath string, f os.FileInfo, err error) error {
		return os.Lchown(filePath, uid, gid)
	})
}

// addSecretsFromMountsFile copies the contents of host directory to container directory
// and returns a list of mounts
func addSecretsFromMountsFile(filePath, mountLabel, containerWorkingDir string, uid, gid int) ([]rspec.Mount, error) {
	var mounts []rspec.Mount
	defaultMountsPaths := getMounts(filePath)
	for _, path := range defaultMountsPaths {
		hostDirOrFile, ctrDirOrFile, err := getMountsMap(path)
		if err != nil {
			return nil, err
		}
		// skip if the hostDirOrFile path doesn't exist
		fileInfo, err := os.Stat(hostDirOrFile)
		if err != nil {
			if os.IsNotExist(err) {
				logrus.Warnf("Path %q from %q doesn't exist, skipping", hostDirOrFile, filePath)
				continue
			}
			return nil, errors.Wrapf(err, "failed to stat %q", hostDirOrFile)
		}

		ctrDirOrFileOnHost := filepath.Join(containerWorkingDir, ctrDirOrFile)

		// In the event of a restart, don't want to copy secrets over again as they already would exist in ctrDirOrFileOnHost
		_, err = os.Stat(ctrDirOrFileOnHost)
		if os.IsNotExist(err) {

			hostDirOrFile, err = resolveSymbolicLink(hostDirOrFile)
			if err != nil {
				return nil, err
			}

			// Don't let the umask have any influence on the file and directory creation
			oldUmask := umask.SetUmask(0)
			defer umask.SetUmask(oldUmask)

			switch mode := fileInfo.Mode(); {
			case mode.IsDir():
				if err = os.MkdirAll(ctrDirOrFileOnHost, mode.Perm()); err != nil {
					return nil, errors.Wrapf(err, "making container directory %q failed", ctrDirOrFileOnHost)
				}
				data, err := getHostSecretData(hostDirOrFile, mode.Perm())
				if err != nil {
					return nil, errors.Wrapf(err, "getting host secret data failed")
				}
				for _, s := range data {
					if err := s.saveTo(ctrDirOrFileOnHost); err != nil {
						return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOrFileOnHost)
					}
				}
			case mode.IsRegular():
				data, err := readFileOrDir("", hostDirOrFile, mode.Perm())
				if err != nil {
					return nil, errors.Wrapf(err, "error reading file %q", hostDirOrFile)

				}
				for _, s := range data {
					if err := os.MkdirAll(filepath.Dir(ctrDirOrFileOnHost), s.dirMode); err != nil {
						return nil, err
					}
					if err := ioutil.WriteFile(ctrDirOrFileOnHost, s.data, s.mode); err != nil {
						return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOrFileOnHost)
					}
				}
			default:
				return nil, errors.Errorf("unsupported file type for: %q", hostDirOrFile)
			}

			err = label.Relabel(ctrDirOrFileOnHost, mountLabel, false)
			if err != nil {
				return nil, errors.Wrap(err, "error applying correct labels")
			}
			if uid != 0 || gid != 0 {
				if err := rchown(ctrDirOrFileOnHost, uid, gid); err != nil {
					return nil, err
				}
			}
		} else if err != nil {
			return nil, errors.Wrapf(err, "error getting status of %q", ctrDirOrFileOnHost)
		}

		m := rspec.Mount{
			Source:      ctrDirOrFileOnHost,
			Destination: ctrDirOrFile,
			Type:        "bind",
			Options:     []string{"bind", "rprivate"},
		}

		mounts = append(mounts, m)
	}
	return mounts, nil
}

// addFIPSModeSecret creates /run/secrets/system-fips in the container
// root filesystem if /etc/system-fips exists on hosts.
// This enables the container to be FIPS compliant and run openssl in
// FIPS mode as the host is also in FIPS mode.
func addFIPSModeSecret(mounts *[]rspec.Mount, containerWorkingDir, mountPoint, mountLabel string, uid, gid int) error {
	secretsDir := "/run/secrets"
	ctrDirOnHost := filepath.Join(containerWorkingDir, secretsDir)
	if _, err := os.Stat(ctrDirOnHost); os.IsNotExist(err) {
		if err = idtools.MkdirAllAs(ctrDirOnHost, 0755, uid, gid); err != nil {
			return errors.Wrapf(err, "making container directory %q on host failed", ctrDirOnHost)
		}
		if err = label.Relabel(ctrDirOnHost, mountLabel, false); err != nil {
			return errors.Wrapf(err, "error applying correct labels on %q", ctrDirOnHost)
		}
	}
	fipsFile := filepath.Join(ctrDirOnHost, "system-fips")
	// In the event of restart, it is possible for the FIPS mode file to already exist
	if _, err := os.Stat(fipsFile); os.IsNotExist(err) {
		file, err := os.Create(fipsFile)
		if err != nil {
			return errors.Wrapf(err, "error creating system-fips file in container for FIPS mode")
		}
		defer file.Close()
	}

	if !mountExists(*mounts, secretsDir) {
		m := rspec.Mount{
			Source:      ctrDirOnHost,
			Destination: secretsDir,
			Type:        "bind",
			Options:     []string{"bind", "rprivate"},
		}
		*mounts = append(*mounts, m)
	}

	srcBackendDir := "/usr/share/crypto-policies/back-ends/FIPS"
	destDir := "/etc/crypto-policies/back-ends"
	srcOnHost := filepath.Join(mountPoint, srcBackendDir)
	if _, err := os.Stat(srcOnHost); err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return errors.Wrapf(err, "failed to stat FIPS Backend directory %q", ctrDirOnHost)
	}

	if !mountExists(*mounts, destDir) {
		m := rspec.Mount{
			Source:      srcOnHost,
			Destination: destDir,
			Type:        "bind",
			Options:     []string{"bind", "rprivate"},
		}
		*mounts = append(*mounts, m)
	}
	return nil
}

// mountExists checks if a mount already exists in the spec
func mountExists(mounts []rspec.Mount, dest string) bool {
	for _, mount := range mounts {
		if mount.Destination == dest {
			return true
		}
	}
	return false
}

// resolveSymbolicLink resolves a possbile symlink path. If the path is a symlink, returns resolved
// path; if not, returns the original path.
func resolveSymbolicLink(path string) (string, error) {
	info, err := os.Lstat(path)
	if err != nil {
		return "", err
	}
	if info.Mode()&os.ModeSymlink != os.ModeSymlink {
		return path, nil
	}
	return filepath.EvalSymlinks(path)
}
<|endoftext|>


<commit_before>package sub

import (
	"context"
	"fmt"
	"path/filepath"
	"strings"

	"github.com/justwatchcom/gopass/pkg/agent/client"
	"github.com/justwatchcom/gopass/pkg/backend"
	"github.com/justwatchcom/gopass/pkg/backend/rcs/noop"
	"github.com/justwatchcom/gopass/pkg/ctxutil"
	"github.com/justwatchcom/gopass/pkg/out"
	"github.com/justwatchcom/gopass/pkg/store"

	"github.com/muesli/goprogressbar"
	"github.com/pkg/errors"
)

// Store is password store
type Store struct {
	alias   string
	url     *backend.URL
	crypto  backend.Crypto
	rcs     backend.RCS
	storage backend.Storage
	cfgdir  string
	agent   *client.Client
}

// New creates a new store, copying settings from the given root store
func New(ctx context.Context, alias string, u *backend.URL, cfgdir string, agent *client.Client) (*Store, error) {
	out.Debug(ctx, "sub.New - URL: %s", u.String())

	s := &Store{
		alias:  alias,
		url:    u,
		rcs:    noop.New(),
		cfgdir: cfgdir,
		agent:  agent,
	}

	// init store backend
	if backend.HasStorageBackend(ctx) {
		s.url.Storage = backend.GetStorageBackend(ctx)
		out.Debug(ctx, "sub.New - Using storage backend from ctx: %s", backend.StorageBackendName(s.url.Storage))
	}
	if err := s.initStorageBackend(ctx); err != nil {
		return nil, errors.Wrapf(err, "failed to init storage backend: %s", err)
	}

	// init sync backend
	if backend.HasRCSBackend(ctx) {
		s.url.RCS = backend.GetRCSBackend(ctx)
		out.Debug(ctx, "sub.New - Using RCS backend from ctx: %s", backend.RCSBackendName(s.url.RCS))
	}
	if err := s.initRCSBackend(ctx); err != nil {
		return nil, errors.Wrapf(err, "failed to init RCS backend: %s", err)
	}

	// init crypto backend
	if backend.HasCryptoBackend(ctx) {
		s.url.Crypto = backend.GetCryptoBackend(ctx)
		out.Debug(ctx, "sub.New - Using Crypto backend from ctx: %s", backend.CryptoBackendName(s.url.Crypto))
	}
	if err := s.initCryptoBackend(ctx); err != nil {
		return nil, errors.Wrapf(err, "failed to init crypto backend: %s", err)
	}

	out.Debug(ctx, "sub.New - initialized - storage: %p - rcs: %p - crypto: %p", s.storage, s.rcs, s.crypto)
	return s, nil
}

// idFile returns the path to the recipient list for this.storage
// it walks up from the given filename until it finds a directory containing
// a gpg id file or it leaves the scope of this.storage.
func (s *Store) idFile(ctx context.Context, name string) string {
	fn := name
	var cnt uint8
	for {
		cnt++
		if cnt > 100 {
			break
		}
		if fn == "" || fn == sep {
			break
		}
		gfn := filepath.Join(fn, s.crypto.IDFile())
		if s.storage.Exists(ctx, gfn) {
			return gfn
		}
		fn = filepath.Dir(fn)
	}
	return s.crypto.IDFile()
}

// Equals returns true if this.storage has the same on-disk path as the other
func (s *Store) Equals(other store.Store) bool {
	if other == nil {
		return false
	}
	return s.URL() == other.URL()
}

// IsDir returns true if the entry is folder inside the store
func (s *Store) IsDir(ctx context.Context, name string) bool {
	return s.storage.IsDir(ctx, name)
}

// Exists checks the existence of a single entry
func (s *Store) Exists(ctx context.Context, name string) bool {
	return s.storage.Exists(ctx, s.passfile(name))
}

func (s *Store) useableKeys(ctx context.Context, name string) ([]string, error) {
	rs, err := s.GetRecipients(ctx, name)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to get recipients")
	}

	if !IsCheckRecipients(ctx) {
		return rs, nil
	}

	kl, err := s.crypto.FindPublicKeys(ctx, rs...)
	if err != nil {
		return rs, err
	}

	return kl, nil
}

// passfile returns the name of gpg file on disk, for the given key/name
func (s *Store) passfile(name string) string {
	return strings.TrimPrefix(name+"."+s.crypto.Ext(), "/")
}

// String implement fmt.Stringer
func (s *Store) String() string {
	return fmt.Sprintf("Store(Alias: %s, Path: %s)", s.alias, s.url.String())
}

// reencrypt will re-encrypt all entries for the current recipients
func (s *Store) reencrypt(ctx context.Context) error {
	entries, err := s.List(ctx, "")
	if err != nil {
		return errors.Wrapf(err, "failed to list store")
	}

	// save original value of auto push
	{
		// shadow ctx in this block only
		ctx := WithAutoSync(ctx, false)
		ctx = ctxutil.WithGitCommit(ctx, false)

		// progress bar
		bar := &goprogressbar.ProgressBar{
			Total: int64(len(entries)),
			Width: 120,
		}
		if !ctxutil.IsTerminal(ctx) || out.IsHidden(ctx) {
			bar = nil
		}
		for _, e := range entries {
			// check for context cancelation
			select {
			case <-ctx.Done():
				return errors.New("context canceled")
			default:
			}

			if bar != nil {
				bar.Current++
				bar.Text = fmt.Sprintf("%d of %d secrets reencrypted", bar.Current, bar.Total)
				bar.LazyPrint()
			}

			content, err := s.Get(ctx, e)
			if err != nil {
				out.Red(ctx, "Failed to get current value for %s: %s", e, err)
				continue
			}
			if err := s.Set(ctx, e, content); err != nil {
				out.Red(ctx, "Failed to write %s: %s", e, err)
			}
		}
	}

	if err := s.rcs.Commit(ctx, GetReason(ctx)); err != nil {
		if errors.Cause(err) != store.ErrGitNotInit {
			return errors.Wrapf(err, "failed to commit changes to git")
		}
	}

	if !IsAutoSync(ctx) {
		return nil
	}

	return s.reencryptGitPush(ctx)
}

func (s *Store) reencryptGitPush(ctx context.Context) error {
	if err := s.rcs.Push(ctx, "", ""); err != nil {
		if errors.Cause(err) == store.ErrGitNotInit {
			msg := "Warning: git is not initialized for this.storage. Ignoring auto-push option\n" +
				"Run: gopass git init"
			out.Red(ctx, msg)
			return nil
		}
		if errors.Cause(err) == store.ErrGitNoRemote {
			msg := "Warning: git has no remote. Ignoring auto-push option\n" +
				"Run: gopass git remote add origin ..."
			out.Yellow(ctx, msg)
			return nil
		}
		return errors.Wrapf(err, "failed to push change to git remote")
	}
	return nil
}

// Path returns the value of path
func (s *Store) Path() string {
	if s.url == nil {
		return ""
	}
	return s.url.Path
}

// Alias returns the value of alias
func (s *Store) Alias() string {
	return s.alias
}

// URL returns the store URL
func (s *Store) URL() string {
	return s.url.String()
}

// Storage returns the storage backend used by this.storage
func (s *Store) Storage() backend.Storage {
	return s.storage
}

// Valid returns true if this store is not nil
func (s *Store) Valid() bool {
	return s != nil
}
<commit_msg>Fix reencrypt (#796)<commit_after>package sub

import (
	"context"
	"fmt"
	"path/filepath"
	"strings"

	"github.com/justwatchcom/gopass/pkg/agent/client"
	"github.com/justwatchcom/gopass/pkg/backend"
	"github.com/justwatchcom/gopass/pkg/backend/rcs/noop"
	"github.com/justwatchcom/gopass/pkg/ctxutil"
	"github.com/justwatchcom/gopass/pkg/out"
	"github.com/justwatchcom/gopass/pkg/store"

	"github.com/muesli/goprogressbar"
	"github.com/pkg/errors"
)

// Store is password store
type Store struct {
	alias   string
	url     *backend.URL
	crypto  backend.Crypto
	rcs     backend.RCS
	storage backend.Storage
	cfgdir  string
	agent   *client.Client
}

// New creates a new store, copying settings from the given root store
func New(ctx context.Context, alias string, u *backend.URL, cfgdir string, agent *client.Client) (*Store, error) {
	out.Debug(ctx, "sub.New - URL: %s", u.String())

	s := &Store{
		alias:  alias,
		url:    u,
		rcs:    noop.New(),
		cfgdir: cfgdir,
		agent:  agent,
	}

	// init store backend
	if backend.HasStorageBackend(ctx) {
		s.url.Storage = backend.GetStorageBackend(ctx)
		out.Debug(ctx, "sub.New - Using storage backend from ctx: %s", backend.StorageBackendName(s.url.Storage))
	}
	if err := s.initStorageBackend(ctx); err != nil {
		return nil, errors.Wrapf(err, "failed to init storage backend: %s", err)
	}

	// init sync backend
	if backend.HasRCSBackend(ctx) {
		s.url.RCS = backend.GetRCSBackend(ctx)
		out.Debug(ctx, "sub.New - Using RCS backend from ctx: %s", backend.RCSBackendName(s.url.RCS))
	}
	if err := s.initRCSBackend(ctx); err != nil {
		return nil, errors.Wrapf(err, "failed to init RCS backend: %s", err)
	}

	// init crypto backend
	if backend.HasCryptoBackend(ctx) {
		s.url.Crypto = backend.GetCryptoBackend(ctx)
		out.Debug(ctx, "sub.New - Using Crypto backend from ctx: %s", backend.CryptoBackendName(s.url.Crypto))
	}
	if err := s.initCryptoBackend(ctx); err != nil {
		return nil, errors.Wrapf(err, "failed to init crypto backend: %s", err)
	}

	out.Debug(ctx, "sub.New - initialized - storage: %p - rcs: %p - crypto: %p", s.storage, s.rcs, s.crypto)
	return s, nil
}

// idFile returns the path to the recipient list for this.storage
// it walks up from the given filename until it finds a directory containing
// a gpg id file or it leaves the scope of this.storage.
func (s *Store) idFile(ctx context.Context, name string) string {
	fn := name
	var cnt uint8
	for {
		cnt++
		if cnt > 100 {
			break
		}
		if fn == "" || fn == sep {
			break
		}
		gfn := filepath.Join(fn, s.crypto.IDFile())
		if s.storage.Exists(ctx, gfn) {
			return gfn
		}
		fn = filepath.Dir(fn)
	}
	return s.crypto.IDFile()
}

// Equals returns true if this.storage has the same on-disk path as the other
func (s *Store) Equals(other store.Store) bool {
	if other == nil {
		return false
	}
	return s.URL() == other.URL()
}

// IsDir returns true if the entry is folder inside the store
func (s *Store) IsDir(ctx context.Context, name string) bool {
	return s.storage.IsDir(ctx, name)
}

// Exists checks the existence of a single entry
func (s *Store) Exists(ctx context.Context, name string) bool {
	return s.storage.Exists(ctx, s.passfile(name))
}

func (s *Store) useableKeys(ctx context.Context, name string) ([]string, error) {
	rs, err := s.GetRecipients(ctx, name)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to get recipients")
	}

	if !IsCheckRecipients(ctx) {
		return rs, nil
	}

	kl, err := s.crypto.FindPublicKeys(ctx, rs...)
	if err != nil {
		return rs, err
	}

	return kl, nil
}

// passfile returns the name of gpg file on disk, for the given key/name
func (s *Store) passfile(name string) string {
	return strings.TrimPrefix(name+"."+s.crypto.Ext(), "/")
}

// String implement fmt.Stringer
func (s *Store) String() string {
	return fmt.Sprintf("Store(Alias: %s, Path: %s)", s.alias, s.url.String())
}

// reencrypt will re-encrypt all entries for the current recipients
func (s *Store) reencrypt(ctx context.Context) error {
	entries, err := s.List(ctx, "")
	if err != nil {
		return errors.Wrapf(err, "failed to list store")
	}

	// save original value of auto push
	{
		// shadow ctx in this block only
		ctx := WithAutoSync(ctx, false)
		ctx = ctxutil.WithGitCommit(ctx, false)

		// progress bar
		bar := &goprogressbar.ProgressBar{
			Total: int64(len(entries)),
			Width: 120,
		}
		if !ctxutil.IsTerminal(ctx) || out.IsHidden(ctx) {
			bar = nil
		}
		for _, e := range entries {
			// check for context cancelation
			select {
			case <-ctx.Done():
				return errors.New("context canceled")
			default:
			}

			if bar != nil {
				bar.Current++
				bar.Text = fmt.Sprintf("%d of %d secrets reencrypted", bar.Current, bar.Total)
				bar.LazyPrint()
			}

			e = strings.TrimPrefix(e, s.alias)
			content, err := s.Get(ctx, e)
			if err != nil {
				out.Red(ctx, "\n[%s] Failed to get current value for '%s': %s", s.alias, e, err)
				continue
			}
			if err := s.Set(ctx, e, content); err != nil {
				out.Red(ctx, "Failed to write %s: %s", e, err)
			}
		}
	}

	if err := s.rcs.Commit(ctx, GetReason(ctx)); err != nil {
		if errors.Cause(err) != store.ErrGitNotInit {
			return errors.Wrapf(err, "failed to commit changes to git")
		}
	}

	if !IsAutoSync(ctx) {
		return nil
	}

	return s.reencryptGitPush(ctx)
}

func (s *Store) reencryptGitPush(ctx context.Context) error {
	if err := s.rcs.Push(ctx, "", ""); err != nil {
		if errors.Cause(err) == store.ErrGitNotInit {
			msg := "Warning: git is not initialized for this.storage. Ignoring auto-push option\n" +
				"Run: gopass git init"
			out.Red(ctx, msg)
			return nil
		}
		if errors.Cause(err) == store.ErrGitNoRemote {
			msg := "Warning: git has no remote. Ignoring auto-push option\n" +
				"Run: gopass git remote add origin ..."
			out.Yellow(ctx, msg)
			return nil
		}
		return errors.Wrapf(err, "failed to push change to git remote")
	}
	return nil
}

// Path returns the value of path
func (s *Store) Path() string {
	if s.url == nil {
		return ""
	}
	return s.url.Path
}

// Alias returns the value of alias
func (s *Store) Alias() string {
	return s.alias
}

// URL returns the store URL
func (s *Store) URL() string {
	return s.url.String()
}

// Storage returns the storage backend used by this.storage
func (s *Store) Storage() backend.Storage {
	return s.storage
}

// Valid returns true if this store is not nil
func (s *Store) Valid() bool {
	return s != nil
}
<|endoftext|>


<commit_before>// Copyright 2016 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types_test

import (
	"reflect"

	. "github.com/containernetworking/cni/pkg/types"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
)

var _ = Describe("UnmarshallableBool UnmarshalText", func() {
	DescribeTable("string to bool detection should succeed in all cases",
		func(inputs []string, expected bool) {
			for _, s := range inputs {
				var ub UnmarshallableBool
				err := ub.UnmarshalText([]byte(s))
				Expect(err).ToNot(HaveOccurred())
				Expect(ub).To(Equal(UnmarshallableBool(expected)))
			}
		},
		Entry("parse to true", []string{"True", "true", "1"}, true),
		Entry("parse to false", []string{"False", "false", "0"}, false),
	)

	Context("When passed an invalid value", func() {
		It("should result in an error", func() {
			var ub UnmarshallableBool
			err := ub.UnmarshalText([]byte("invalid"))
			Expect(err).To(HaveOccurred())
		})
	})
})

var _ = Describe("GetKeyField", func() {
	type testcontainer struct {
		Valid string `json:"valid,omitempty"`
	}
	var (
		container          = testcontainer{Valid: "valid"}
		containerInterface = func(i interface{}) interface{} { return i }(&container)
		containerValue     = reflect.ValueOf(containerInterface)
	)
	Context("When a valid field is provided", func() {
		It("should return the correct field", func() {
			field := GetKeyField("Valid", containerValue)
			Expect(field.String()).To(Equal("valid"))
		})
	})
})

var _ = Describe("LoadArgs", func() {
	Context("When no arguments are passed", func() {
		It("LoadArgs should succeed", func() {
			err := LoadArgs("", struct{}{})
			Expect(err).NotTo(HaveOccurred())
		})
	})

	Context("When unknown arguments are passed and ignored", func() {
		It("LoadArgs should succeed", func() {
			ca := CommonArgs{}
			err := LoadArgs("IgnoreUnknown=True;Unk=nown", &ca)
			Expect(err).NotTo(HaveOccurred())
		})
	})

	Context("When unknown arguments are passed and not ignored", func() {
		It("LoadArgs should fail", func() {
			ca := CommonArgs{}
			err := LoadArgs("Unk=nown", &ca)
			Expect(err).To(HaveOccurred())
		})
	})

	Context("When unknown arguments are passed and explicitly not ignored", func() {
		It("LoadArgs should fail", func() {
			ca := CommonArgs{}
			err := LoadArgs("IgnoreUnknown=0, Unk=nown", &ca)
			Expect(err).To(HaveOccurred())
		})
	})

	Context("When known arguments are passed", func() {
		It("LoadArgs should succeed", func() {
			ca := CommonArgs{}
			err := LoadArgs("IgnoreUnknown=1", &ca)
			Expect(err).NotTo(HaveOccurred())
		})
	})
})
<commit_msg>pkg/types: cover string for unmarshal tests<commit_after>// Copyright 2016 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types_test

import (
	"reflect"

	. "github.com/containernetworking/cni/pkg/types"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
)

var _ = Describe("UnmarshallableBool UnmarshalText", func() {
	DescribeTable("string to bool detection should succeed in all cases",
		func(inputs []string, expected bool) {
			for _, s := range inputs {
				var ub UnmarshallableBool
				err := ub.UnmarshalText([]byte(s))
				Expect(err).ToNot(HaveOccurred())
				Expect(ub).To(Equal(UnmarshallableBool(expected)))
			}
		},
		Entry("parse to true", []string{"True", "true", "1"}, true),
		Entry("parse to false", []string{"False", "false", "0"}, false),
	)

	Context("When passed an invalid value", func() {
		It("should result in an error", func() {
			var ub UnmarshallableBool
			err := ub.UnmarshalText([]byte("invalid"))
			Expect(err).To(HaveOccurred())
		})
	})
})

var _ = Describe("UnmarshallableString UnmarshalText", func() {
	DescribeTable("string to string detection should succeed in all cases",
		func(inputs []string, expected string) {
			for _, s := range inputs {
				var us UnmarshallableString
				err := us.UnmarshalText([]byte(s))
				Expect(err).ToNot(HaveOccurred())
				Expect(string(us)).To(Equal(expected))
			}
		},
		Entry("parse empty string", []string{""}, ""),
		Entry("parse non-empty string", []string{"notempty"}, "notempty"),
	)
})

var _ = Describe("GetKeyField", func() {
	type testcontainer struct {
		Valid string `json:"valid,omitempty"`
	}
	var (
		container          = testcontainer{Valid: "valid"}
		containerInterface = func(i interface{}) interface{} { return i }(&container)
		containerValue     = reflect.ValueOf(containerInterface)
	)
	Context("When a valid field is provided", func() {
		It("should return the correct field", func() {
			field := GetKeyField("Valid", containerValue)
			Expect(field.String()).To(Equal("valid"))
		})
	})
})

var _ = Describe("LoadArgs", func() {
	Context("When no arguments are passed", func() {
		It("LoadArgs should succeed", func() {
			err := LoadArgs("", struct{}{})
			Expect(err).NotTo(HaveOccurred())
		})
	})

	Context("When unknown arguments are passed and ignored", func() {
		It("LoadArgs should succeed", func() {
			ca := CommonArgs{}
			err := LoadArgs("IgnoreUnknown=True;Unk=nown", &ca)
			Expect(err).NotTo(HaveOccurred())
		})
	})

	Context("When unknown arguments are passed and not ignored", func() {
		It("LoadArgs should fail", func() {
			ca := CommonArgs{}
			err := LoadArgs("Unk=nown", &ca)
			Expect(err).To(HaveOccurred())
		})
	})

	Context("When unknown arguments are passed and explicitly not ignored", func() {
		It("LoadArgs should fail", func() {
			ca := CommonArgs{}
			err := LoadArgs("IgnoreUnknown=0, Unk=nown", &ca)
			Expect(err).To(HaveOccurred())
		})
	})

	Context("When known arguments are passed", func() {
		It("LoadArgs should succeed", func() {
			ca := CommonArgs{}
			err := LoadArgs("IgnoreUnknown=1", &ca)
			Expect(err).NotTo(HaveOccurred())
		})
	})
})
<|endoftext|>