<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sidecar

import(
	"testing"

	"k8s.io/dns/pkg/dnsmasq"
	"github.com/stretchr/testify/assert"
)

// initMetrics initializes dnsmasq.Metrics with values for testing.
func initMetrics(metricsList []*dnsmasq.Metrics, values []int64) {
	defineDnsmasqMetrics(&Options{PrometheusNamespace:"dnsmasq"})
	for i := range metricsList {
		metricsList[i] = &dnsmasq.Metrics{}
		for j := range dnsmasq.AllMetrics {
			metric := dnsmasq.AllMetrics[j]
			// Avoids giving each metric the same value.
			(*(metricsList[i]))[metric] = values[j] * int64(i+1)
		}
	}
}

// TestExportMetrics tests if our countersCache works as expected.
func TestExportMetrics(t *testing.T) {
	var m1, m2, m3 *dnsmasq.Metrics
        l := []*dnsmasq.Metrics{m1, m2, m3}

	testMetricValues := []int64{10, 20, 30, 40, 50}
	initMetrics(l, testMetricValues)

	for i := range l {
		exportMetrics(l[i])
		for j := range dnsmasq.AllMetrics {
			assert.Equal(t, countersCache[dnsmasq.AllMetrics[j]], float64(testMetricValues[j] * int64(i+1)))
		}
	}
}
<commit_msg>Ran gofmt<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sidecar

import (
	"testing"

	"github.com/stretchr/testify/assert"
	"k8s.io/dns/pkg/dnsmasq"
)

// initMetrics initializes dnsmasq.Metrics with values for testing.
func initMetrics(metricsList []*dnsmasq.Metrics, values []int64) {
	defineDnsmasqMetrics(&Options{PrometheusNamespace: "dnsmasq"})
	for i := range metricsList {
		metricsList[i] = &dnsmasq.Metrics{}
		for j := range dnsmasq.AllMetrics {
			metric := dnsmasq.AllMetrics[j]
			// Avoids giving each metric the same value.
			(*(metricsList[i]))[metric] = values[j] * int64(i+1)
		}
	}
}

// TestExportMetrics tests if our countersCache works as expected.
func TestExportMetrics(t *testing.T) {
	var m1, m2, m3 *dnsmasq.Metrics
	l := []*dnsmasq.Metrics{m1, m2, m3}

	testMetricValues := []int64{10, 20, 30, 40, 50}
	initMetrics(l, testMetricValues)

	for i := range l {
		exportMetrics(l[i])
		for j := range dnsmasq.AllMetrics {
			assert.Equal(t, countersCache[dnsmasq.AllMetrics[j]], float64(testMetricValues[j]*int64(i+1)))
		}
	}
}
<|endoftext|>


<commit_before>package tarheader

import (
	"archive/tar"
	"os"
	"syscall"
)

func init() {
	populateHeaderStat = append(populateHeaderStat, populateHeaderUnix)
}

func populateHeaderUnix(h *tar.Header, fi os.FileInfo, seen map[uint64]string) {
	st, ok := fi.Sys().(*syscall.Stat_t)
	if !ok {
		return
	}
	h.Uid = int(st.Uid)
	h.Gid = int(st.Gid)
	// If we have already seen this inode, generate a hardlink
	p, ok := seen[st.Ino]
	if ok {
		h.Linkname = p
		h.Typeflag = tar.TypeLink
	} else {
		seen[st.Ino] = h.Name
	}
}
<commit_msg>pkg/tarheader: cast inode up to uint64<commit_after>package tarheader

import (
	"archive/tar"
	"os"
	"syscall"
)

func init() {
	populateHeaderStat = append(populateHeaderStat, populateHeaderUnix)
}

func populateHeaderUnix(h *tar.Header, fi os.FileInfo, seen map[uint64]string) {
	st, ok := fi.Sys().(*syscall.Stat_t)
	if !ok {
		return
	}
	h.Uid = int(st.Uid)
	h.Gid = int(st.Gid)
	// If we have already seen this inode, generate a hardlink
	p, ok := seen[uint64(st.Ino)]
	if ok {
		h.Linkname = p
		h.Typeflag = tar.TypeLink
	} else {
		seen[uint64(st.Ino)] = h.Name
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sutil

import (
	"encoding/json"
	"fmt"
	"path"
	"time"

	backupenv "github.com/coreos/etcd-operator/pkg/backup/env"
	"github.com/coreos/etcd-operator/pkg/spec"
	"github.com/coreos/etcd-operator/pkg/util/constants"

	"github.com/coreos/etcd-operator/pkg/backup/s3/s3config"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	unversionedAPI "k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apis/extensions"
	"k8s.io/kubernetes/pkg/apis/storage"
	"k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/util/intstr"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/pkg/watch"
)

const (
	storageClassPrefix        = "etcd-operator-backup"
	BackupPodSelectorAppField = "etcd_backup_tool"
	backupPVVolName           = "etcd-backup-storage"
	awsCredentialDir          = "/root/.aws/"
	awsConfigDir              = "/root/.aws/config/"
	awsSecretVolName          = "secret-aws"
	awsConfigVolName          = "config-aws"
	fromDirMountDir           = "/mnt/backup/from"
)

func CreateStorageClass(kubecli *unversioned.Client, pvProvisioner string) error {
	// We need to get rid of prefix because naming doesn't support "/".
	name := storageClassPrefix + "-" + path.Base(pvProvisioner)
	class := &storage.StorageClass{
		ObjectMeta: api.ObjectMeta{
			Name: name,
		},
		Provisioner: pvProvisioner,
	}
	_, err := kubecli.StorageClasses().Create(class)
	return err
}

func CreateAndWaitPVC(kubecli *unversioned.Client, clusterName, ns, pvProvisioner string, volumeSizeInMB int) error {
	name := makePVCName(clusterName)
	storageClassName := storageClassPrefix + "-" + path.Base(pvProvisioner)
	claim := &api.PersistentVolumeClaim{
		ObjectMeta: api.ObjectMeta{
			Name: name,
			Labels: map[string]string{
				"etcd_cluster": clusterName,
			},
			Annotations: map[string]string{
				"volume.beta.kubernetes.io/storage-class": storageClassName,
			},
		},
		Spec: api.PersistentVolumeClaimSpec{
			AccessModes: []api.PersistentVolumeAccessMode{
				api.ReadWriteOnce,
			},
			Resources: api.ResourceRequirements{
				Requests: api.ResourceList{
					api.ResourceStorage: resource.MustParse(fmt.Sprintf("%dMi", volumeSizeInMB)),
				},
			},
		},
	}
	_, err := kubecli.PersistentVolumeClaims(ns).Create(claim)
	if err != nil {
		return err
	}

	err = wait.Poll(4*time.Second, 20*time.Second, func() (bool, error) {
		var err error
		claim, err = kubecli.PersistentVolumeClaims(ns).Get(name)
		if err != nil {
			return false, err
		}
		if claim.Status.Phase != api.ClaimBound {
			return false, nil
		}
		return true, nil
	})
	if err != nil {
		wErr := fmt.Errorf("fail to wait PVC (%s) '(%v)/Bound': %v", name, claim.Status.Phase, err)
		return wErr
	}

	return nil
}

var BackupImage = "quay.io/coreos/etcd-operator:latest"

func PodSpecWithPV(ps *api.PodSpec, clusterName string) *api.PodSpec {
	ps.Containers[0].VolumeMounts = []api.VolumeMount{{
		Name:      backupPVVolName,
		MountPath: constants.BackupDir,
	}}
	ps.Volumes = []api.Volume{{
		Name: backupPVVolName,
		VolumeSource: api.VolumeSource{
			PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{
				ClaimName: makePVCName(clusterName),
			},
		},
	}}
	return ps
}

func PodSpecWithS3(ps *api.PodSpec, s3Ctx s3config.S3Context) *api.PodSpec {
	ps.Containers[0].VolumeMounts = []api.VolumeMount{{
		Name:      awsSecretVolName,
		MountPath: awsCredentialDir,
	}, {
		Name:      awsConfigVolName,
		MountPath: awsConfigDir,
	}}
	ps.Volumes = []api.Volume{{
		Name: awsSecretVolName,
		VolumeSource: api.VolumeSource{
			Secret: &api.SecretVolumeSource{
				SecretName: s3Ctx.AWSSecret,
			},
		},
	}, {
		Name: awsConfigVolName,
		VolumeSource: api.VolumeSource{
			ConfigMap: &api.ConfigMapVolumeSource{
				LocalObjectReference: api.LocalObjectReference{
					Name: s3Ctx.AWSConfig,
				},
			},
		},
	}}
	ps.Containers[0].Env = append(ps.Containers[0].Env, api.EnvVar{
		Name:  backupenv.AWSConfig,
		Value: awsConfigDir,
	}, api.EnvVar{
		Name:  backupenv.AWSS3Bucket,
		Value: s3Ctx.S3Bucket,
	})
	return ps
}

func MakeBackupPodSpec(clusterName string, policy *spec.BackupPolicy) (*api.PodSpec, error) {
	bp, err := json.Marshal(policy)
	if err != nil {
		return nil, err
	}

	ps := &api.PodSpec{
		Containers: []api.Container{
			{
				Name:  "backup",
				Image: BackupImage,
				Command: []string{
					"/bin/sh",
					"-c",
					"/usr/local/bin/etcd-backup --etcd-cluster=" + clusterName,
				},
				Env: []api.EnvVar{{
					Name:      "MY_POD_NAMESPACE",
					ValueFrom: &api.EnvVarSource{FieldRef: &api.ObjectFieldSelector{FieldPath: "metadata.namespace"}},
				}, {
					Name:  backupenv.BackupPolicy,
					Value: string(bp),
				}},
			},
		},
	}
	return ps, nil
}

func CreateBackupReplicaSetAndService(kubecli *unversioned.Client, clusterName, ns string, ps api.PodSpec) error {
	labels := map[string]string{
		"app":          BackupPodSelectorAppField,
		"etcd_cluster": clusterName,
	}
	name := MakeBackupName(clusterName)
	_, err := kubecli.ReplicaSets(ns).Create(&extensions.ReplicaSet{
		ObjectMeta: api.ObjectMeta{
			Name: name,
		},
		Spec: extensions.ReplicaSetSpec{
			Replicas: 1,
			Selector: &unversionedAPI.LabelSelector{MatchLabels: labels},
			Template: api.PodTemplateSpec{
				ObjectMeta: api.ObjectMeta{
					Labels: labels,
				},
				Spec: ps,
			},
		},
	})
	if err != nil {
		if !IsKubernetesResourceAlreadyExistError(err) {
			return err
		}
	}

	svc := &api.Service{
		ObjectMeta: api.ObjectMeta{
			Name:   name,
			Labels: labels,
		},
		Spec: api.ServiceSpec{
			Ports: []api.ServicePort{
				{
					Name:       "backup-service",
					Port:       constants.DefaultBackupPodHTTPPort,
					TargetPort: intstr.FromInt(constants.DefaultBackupPodHTTPPort),
					Protocol:   api.ProtocolTCP,
				},
			},
			Selector: labels,
		},
	}
	if _, err := kubecli.Services(ns).Create(svc); err != nil {
		if !IsKubernetesResourceAlreadyExistError(err) {
			return err
		}
	}
	return nil
}

func DeleteBackupReplicaSetAndService(kubecli *unversioned.Client, clusterName, ns string, cleanup bool) error {
	name := MakeBackupName(clusterName)
	err := kubecli.Services(ns).Delete(name)
	if err != nil {
		return err
	}
	orphanOption := false
	gracePeriod := int64(0)
	err = kubecli.ReplicaSets(ns).Delete(name, &api.DeleteOptions{
		OrphanDependents:   &orphanOption,
		GracePeriodSeconds: &gracePeriod,
	})
	if err != nil {
		return err
	}
	if cleanup {
		kubecli.PersistentVolumeClaims(ns).Delete(makePVCName(clusterName))
	}
	return nil
}

func CopyVolume(kubecli *unversioned.Client, fromClusterName, toClusterName, ns string) error {
	pod := &api.Pod{
		ObjectMeta: api.ObjectMeta{
			Name: copyVolumePodName(toClusterName),
			Labels: map[string]string{
				"etcd_cluster": toClusterName,
			},
		},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:  "copy-backup",
					Image: "alpine",
					Command: []string{
						"/bin/sh",
						"-c",
						fmt.Sprintf("cp -r %s/* %s/", fromDirMountDir, constants.BackupDir),
					},
					VolumeMounts: []api.VolumeMount{{
						Name:      "from-dir",
						MountPath: fromDirMountDir,
					}, {
						Name:      "to-dir",
						MountPath: constants.BackupDir,
					}},
				},
			},
			RestartPolicy: api.RestartPolicyNever,
			Volumes: []api.Volume{{
				Name: "from-dir",
				VolumeSource: api.VolumeSource{
					PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{
						ClaimName: makePVCName(fromClusterName),
						ReadOnly:  true,
					},
				},
			}, {
				Name: "to-dir",
				VolumeSource: api.VolumeSource{
					PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{
						ClaimName: makePVCName(toClusterName),
					},
				},
			}},
		},
	}
	if _, err := kubecli.Pods(ns).Create(pod); err != nil {
		return err
	}

	w, err := kubecli.Pods(ns).Watch(api.SingleObject(api.ObjectMeta{Name: pod.Name}))
	if err != nil {
		return err
	}
	// It could take long due to delay of k8s controller detaching the volume
	_, err = watch.Until(120*time.Second, w, unversioned.PodCompleted)
	if err != nil {
		return fmt.Errorf("fail to wait data copy job completed: %v", err)
	}
	// Delete the pod to detach the volume from the node
	return kubecli.Pods(ns).Delete(pod.Name, api.NewDeleteOptions(0))
}

func copyVolumePodName(clusterName string) string {
	return clusterName + "-copyvolume"
}

func makePVCName(clusterName string) string {
	return fmt.Sprintf("%s-pvc", clusterName)
}
<commit_msg>s3: fix config env<commit_after>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sutil

import (
	"encoding/json"
	"fmt"
	"path"
	"time"

	backupenv "github.com/coreos/etcd-operator/pkg/backup/env"
	"github.com/coreos/etcd-operator/pkg/spec"
	"github.com/coreos/etcd-operator/pkg/util/constants"

	"github.com/coreos/etcd-operator/pkg/backup/s3/s3config"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	unversionedAPI "k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apis/extensions"
	"k8s.io/kubernetes/pkg/apis/storage"
	"k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/util/intstr"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/pkg/watch"
)

const (
	storageClassPrefix        = "etcd-operator-backup"
	BackupPodSelectorAppField = "etcd_backup_tool"
	backupPVVolName           = "etcd-backup-storage"
	awsCredentialDir          = "/root/.aws/"
	awsConfigDir              = "/root/.aws/config/"
	awsSecretVolName          = "secret-aws"
	awsConfigVolName          = "config-aws"
	fromDirMountDir           = "/mnt/backup/from"
)

func CreateStorageClass(kubecli *unversioned.Client, pvProvisioner string) error {
	// We need to get rid of prefix because naming doesn't support "/".
	name := storageClassPrefix + "-" + path.Base(pvProvisioner)
	class := &storage.StorageClass{
		ObjectMeta: api.ObjectMeta{
			Name: name,
		},
		Provisioner: pvProvisioner,
	}
	_, err := kubecli.StorageClasses().Create(class)
	return err
}

func CreateAndWaitPVC(kubecli *unversioned.Client, clusterName, ns, pvProvisioner string, volumeSizeInMB int) error {
	name := makePVCName(clusterName)
	storageClassName := storageClassPrefix + "-" + path.Base(pvProvisioner)
	claim := &api.PersistentVolumeClaim{
		ObjectMeta: api.ObjectMeta{
			Name: name,
			Labels: map[string]string{
				"etcd_cluster": clusterName,
			},
			Annotations: map[string]string{
				"volume.beta.kubernetes.io/storage-class": storageClassName,
			},
		},
		Spec: api.PersistentVolumeClaimSpec{
			AccessModes: []api.PersistentVolumeAccessMode{
				api.ReadWriteOnce,
			},
			Resources: api.ResourceRequirements{
				Requests: api.ResourceList{
					api.ResourceStorage: resource.MustParse(fmt.Sprintf("%dMi", volumeSizeInMB)),
				},
			},
		},
	}
	_, err := kubecli.PersistentVolumeClaims(ns).Create(claim)
	if err != nil {
		return err
	}

	err = wait.Poll(4*time.Second, 20*time.Second, func() (bool, error) {
		var err error
		claim, err = kubecli.PersistentVolumeClaims(ns).Get(name)
		if err != nil {
			return false, err
		}
		if claim.Status.Phase != api.ClaimBound {
			return false, nil
		}
		return true, nil
	})
	if err != nil {
		wErr := fmt.Errorf("fail to wait PVC (%s) '(%v)/Bound': %v", name, claim.Status.Phase, err)
		return wErr
	}

	return nil
}

var BackupImage = "quay.io/coreos/etcd-operator:latest"

func PodSpecWithPV(ps *api.PodSpec, clusterName string) *api.PodSpec {
	ps.Containers[0].VolumeMounts = []api.VolumeMount{{
		Name:      backupPVVolName,
		MountPath: constants.BackupDir,
	}}
	ps.Volumes = []api.Volume{{
		Name: backupPVVolName,
		VolumeSource: api.VolumeSource{
			PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{
				ClaimName: makePVCName(clusterName),
			},
		},
	}}
	return ps
}

func PodSpecWithS3(ps *api.PodSpec, s3Ctx s3config.S3Context) *api.PodSpec {
	ps.Containers[0].VolumeMounts = []api.VolumeMount{{
		Name:      awsSecretVolName,
		MountPath: awsCredentialDir,
	}, {
		Name:      awsConfigVolName,
		MountPath: awsConfigDir,
	}}
	ps.Volumes = []api.Volume{{
		Name: awsSecretVolName,
		VolumeSource: api.VolumeSource{
			Secret: &api.SecretVolumeSource{
				SecretName: s3Ctx.AWSSecret,
			},
		},
	}, {
		Name: awsConfigVolName,
		VolumeSource: api.VolumeSource{
			ConfigMap: &api.ConfigMapVolumeSource{
				LocalObjectReference: api.LocalObjectReference{
					Name: s3Ctx.AWSConfig,
				},
			},
		},
	}}
	ps.Containers[0].Env = append(ps.Containers[0].Env, api.EnvVar{
		Name:  backupenv.AWSConfig,
		Value: path.Join(awsConfigDir, "config"),
	}, api.EnvVar{
		Name:  backupenv.AWSS3Bucket,
		Value: s3Ctx.S3Bucket,
	})
	return ps
}

func MakeBackupPodSpec(clusterName string, policy *spec.BackupPolicy) (*api.PodSpec, error) {
	bp, err := json.Marshal(policy)
	if err != nil {
		return nil, err
	}

	ps := &api.PodSpec{
		Containers: []api.Container{
			{
				Name:  "backup",
				Image: BackupImage,
				Command: []string{
					"/bin/sh",
					"-c",
					"/usr/local/bin/etcd-backup --etcd-cluster=" + clusterName,
				},
				Env: []api.EnvVar{{
					Name:      "MY_POD_NAMESPACE",
					ValueFrom: &api.EnvVarSource{FieldRef: &api.ObjectFieldSelector{FieldPath: "metadata.namespace"}},
				}, {
					Name:  backupenv.BackupPolicy,
					Value: string(bp),
				}},
			},
		},
	}
	return ps, nil
}

func CreateBackupReplicaSetAndService(kubecli *unversioned.Client, clusterName, ns string, ps api.PodSpec) error {
	labels := map[string]string{
		"app":          BackupPodSelectorAppField,
		"etcd_cluster": clusterName,
	}
	name := MakeBackupName(clusterName)
	_, err := kubecli.ReplicaSets(ns).Create(&extensions.ReplicaSet{
		ObjectMeta: api.ObjectMeta{
			Name: name,
		},
		Spec: extensions.ReplicaSetSpec{
			Replicas: 1,
			Selector: &unversionedAPI.LabelSelector{MatchLabels: labels},
			Template: api.PodTemplateSpec{
				ObjectMeta: api.ObjectMeta{
					Labels: labels,
				},
				Spec: ps,
			},
		},
	})
	if err != nil {
		if !IsKubernetesResourceAlreadyExistError(err) {
			return err
		}
	}

	svc := &api.Service{
		ObjectMeta: api.ObjectMeta{
			Name:   name,
			Labels: labels,
		},
		Spec: api.ServiceSpec{
			Ports: []api.ServicePort{
				{
					Name:       "backup-service",
					Port:       constants.DefaultBackupPodHTTPPort,
					TargetPort: intstr.FromInt(constants.DefaultBackupPodHTTPPort),
					Protocol:   api.ProtocolTCP,
				},
			},
			Selector: labels,
		},
	}
	if _, err := kubecli.Services(ns).Create(svc); err != nil {
		if !IsKubernetesResourceAlreadyExistError(err) {
			return err
		}
	}
	return nil
}

func DeleteBackupReplicaSetAndService(kubecli *unversioned.Client, clusterName, ns string, cleanup bool) error {
	name := MakeBackupName(clusterName)
	err := kubecli.Services(ns).Delete(name)
	if err != nil {
		return err
	}
	orphanOption := false
	gracePeriod := int64(0)
	err = kubecli.ReplicaSets(ns).Delete(name, &api.DeleteOptions{
		OrphanDependents:   &orphanOption,
		GracePeriodSeconds: &gracePeriod,
	})
	if err != nil {
		return err
	}
	if cleanup {
		kubecli.PersistentVolumeClaims(ns).Delete(makePVCName(clusterName))
	}
	return nil
}

func CopyVolume(kubecli *unversioned.Client, fromClusterName, toClusterName, ns string) error {
	pod := &api.Pod{
		ObjectMeta: api.ObjectMeta{
			Name: copyVolumePodName(toClusterName),
			Labels: map[string]string{
				"etcd_cluster": toClusterName,
			},
		},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:  "copy-backup",
					Image: "alpine",
					Command: []string{
						"/bin/sh",
						"-c",
						fmt.Sprintf("cp -r %s/* %s/", fromDirMountDir, constants.BackupDir),
					},
					VolumeMounts: []api.VolumeMount{{
						Name:      "from-dir",
						MountPath: fromDirMountDir,
					}, {
						Name:      "to-dir",
						MountPath: constants.BackupDir,
					}},
				},
			},
			RestartPolicy: api.RestartPolicyNever,
			Volumes: []api.Volume{{
				Name: "from-dir",
				VolumeSource: api.VolumeSource{
					PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{
						ClaimName: makePVCName(fromClusterName),
						ReadOnly:  true,
					},
				},
			}, {
				Name: "to-dir",
				VolumeSource: api.VolumeSource{
					PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{
						ClaimName: makePVCName(toClusterName),
					},
				},
			}},
		},
	}
	if _, err := kubecli.Pods(ns).Create(pod); err != nil {
		return err
	}

	w, err := kubecli.Pods(ns).Watch(api.SingleObject(api.ObjectMeta{Name: pod.Name}))
	if err != nil {
		return err
	}
	// It could take long due to delay of k8s controller detaching the volume
	_, err = watch.Until(120*time.Second, w, unversioned.PodCompleted)
	if err != nil {
		return fmt.Errorf("fail to wait data copy job completed: %v", err)
	}
	// Delete the pod to detach the volume from the node
	return kubecli.Pods(ns).Delete(pod.Name, api.NewDeleteOptions(0))
}

func copyVolumePodName(clusterName string) string {
	return clusterName + "-copyvolume"
}

func makePVCName(clusterName string) string {
	return fmt.Sprintf("%s-pvc", clusterName)
}
<|endoftext|>


<commit_before>package utils

import (
	"fmt"

	buildv1alpha1 "github.com/knative/build/pkg/apis/build/v1alpha1"
	servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1"
	runtimev1alpha1 "github.com/kyma-incubator/runtime/pkg/apis/runtime/v1alpha1"
	corev1 "k8s.io/api/core/v1"
)

// GetServiceSpec gets ServiceSpec for a function
func GetServiceSpec(imageName string, fn runtimev1alpha1.Function, rnInfo *RuntimeInfo) servingv1alpha1.ServiceSpec {
	defaultMode := int32(420)
	buildContainer := getBuildContainer(imageName, fn, rnInfo)
	volumes := []corev1.Volume{
		{
			Name: "dockerfile-vol",
			VolumeSource: corev1.VolumeSource{
				ConfigMap: &corev1.ConfigMapVolumeSource{
					DefaultMode: &defaultMode,
					LocalObjectReference: corev1.LocalObjectReference{
						Name: rnInfo.DockerFileConfigMapName(fn.Spec.Runtime),
					},
				},
			},
		},
		{
			Name: "func-vol",
			VolumeSource: corev1.VolumeSource{
				ConfigMap: &corev1.ConfigMapVolumeSource{
					DefaultMode: &defaultMode,
					LocalObjectReference: corev1.LocalObjectReference{
						Name: fn.Name,
					},
				},
			},
		},
	}

	// TODO: Make it constant for nodejs8/nodejs6
	envVarsForRevision := []corev1.EnvVar{
		{
			Name:  "FUNC_HANDLER",
			Value: "main",
		},
		{
			Name:  "MOD_NAME",
			Value: "handler",
		},
		{
			Name:  "FUNC_TIMEOUT",
			Value: "180",
		},
		{
			Name:  "FUNC_RUNTIME",
			Value: "nodejs8",
		},
		{
			Name:  "FUNC_MEMORY_LIMIT",
			Value: "128Mi",
		},
		{
			Name:  "FUNC_PORT",
			Value: "8080",
		},
		{
			Name:  "NODE_PATH",
			Value: "$(KUBELESS_INSTALL_VOLUME)/node_modules",
		},
	}

	return servingv1alpha1.ServiceSpec{
		RunLatest: &servingv1alpha1.RunLatestType{
			Configuration: servingv1alpha1.ConfigurationSpec{
				Build: &servingv1alpha1.RawExtension{
					BuildSpec: &buildv1alpha1.BuildSpec{
						ServiceAccountName: rnInfo.ServiceAccount,
						Steps: []corev1.Container{
							*buildContainer,
						},
						Volumes: volumes,
					},
				},
				RevisionTemplate: servingv1alpha1.RevisionTemplateSpec{
					Spec: servingv1alpha1.RevisionSpec{
						Container: corev1.Container{
							Image: imageName,
							Env:   envVarsForRevision,
							Name:  fn.Name,
						},
					},
				},
			},
		},
	}
}

func getBuildContainer(imageName string, fn runtimev1alpha1.Function, riUtil *RuntimeInfo) *corev1.Container {
	destination := fmt.Sprintf("--destination=%s", imageName)
	buildContainer := corev1.Container{
		Name:  "build-and-push",
		Image: "gcr.io/kaniko-project/executor",
		Args:  []string{"--dockerfile=/workspace/Dockerfile", destination},
		VolumeMounts: []corev1.VolumeMount{
			{
				Name:      "dockerfile-vol", //TODO: make it configurable
				MountPath: "/workspace",
			},
			{
				Name:      "func-vol",
				MountPath: "/src",
			},
		},
	}

	return &buildContainer
}
<commit_msg>Revert "add functionName as container name in service"<commit_after>package utils

import (
	"fmt"

	buildv1alpha1 "github.com/knative/build/pkg/apis/build/v1alpha1"
	servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1"
	runtimev1alpha1 "github.com/kyma-incubator/runtime/pkg/apis/runtime/v1alpha1"
	corev1 "k8s.io/api/core/v1"
)

// GetServiceSpec gets ServiceSpec for a function
func GetServiceSpec(imageName string, fn runtimev1alpha1.Function, rnInfo *RuntimeInfo) servingv1alpha1.ServiceSpec {
	defaultMode := int32(420)
	buildContainer := getBuildContainer(imageName, fn, rnInfo)
	volumes := []corev1.Volume{
		{
			Name: "dockerfile-vol",
			VolumeSource: corev1.VolumeSource{
				ConfigMap: &corev1.ConfigMapVolumeSource{
					DefaultMode: &defaultMode,
					LocalObjectReference: corev1.LocalObjectReference{
						Name: rnInfo.DockerFileConfigMapName(fn.Spec.Runtime),
					},
				},
			},
		},
		{
			Name: "func-vol",
			VolumeSource: corev1.VolumeSource{
				ConfigMap: &corev1.ConfigMapVolumeSource{
					DefaultMode: &defaultMode,
					LocalObjectReference: corev1.LocalObjectReference{
						Name: fn.Name,
					},
				},
			},
		},
	}

	// TODO: Make it constant for nodejs8/nodejs6
	envVarsForRevision := []corev1.EnvVar{
		{
			Name:  "FUNC_HANDLER",
			Value: "main",
		},
		{
			Name:  "MOD_NAME",
			Value: "handler",
		},
		{
			Name:  "FUNC_TIMEOUT",
			Value: "180",
		},
		{
			Name:  "FUNC_RUNTIME",
			Value: "nodejs8",
		},
		{
			Name:  "FUNC_MEMORY_LIMIT",
			Value: "128Mi",
		},
		{
			Name:  "FUNC_PORT",
			Value: "8080",
		},
		{
			Name:  "NODE_PATH",
			Value: "$(KUBELESS_INSTALL_VOLUME)/node_modules",
		},
	}

	return servingv1alpha1.ServiceSpec{
		RunLatest: &servingv1alpha1.RunLatestType{
			Configuration: servingv1alpha1.ConfigurationSpec{
				Build: &servingv1alpha1.RawExtension{
					BuildSpec: &buildv1alpha1.BuildSpec{
						ServiceAccountName: rnInfo.ServiceAccount,
						Steps: []corev1.Container{
							*buildContainer,
						},
						Volumes: volumes,
					},
				},
				RevisionTemplate: servingv1alpha1.RevisionTemplateSpec{
					Spec: servingv1alpha1.RevisionSpec{
						Container: corev1.Container{
							Image: imageName,
							Env:   envVarsForRevision,
						},
					},
				},
			},
		},
	}
}

func getBuildContainer(imageName string, fn runtimev1alpha1.Function, riUtil *RuntimeInfo) *corev1.Container {
	destination := fmt.Sprintf("--destination=%s", imageName)
	buildContainer := corev1.Container{
		Name:  "build-and-push",
		Image: "gcr.io/kaniko-project/executor",
		Args:  []string{"--dockerfile=/workspace/Dockerfile", destination},
		VolumeMounts: []corev1.VolumeMount{
			{
				Name:      "dockerfile-vol", //TODO: make it configurable
				MountPath: "/workspace",
			},
			{
				Name:      "func-vol",
				MountPath: "/src",
			},
		},
	}

	return &buildContainer
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"player"
)

const PORT = 4711

func main() {
	http.HandleFunc("/", handleRequest)
	if err := http.ListenAndServe(fmt.Sprintf(":%d", PORT), nil); err != nil {
		log.Fatal(err)
	}
}

func handleRequest(w http.ResponseWriter, request *http.Request) {
	if err := request.ParseForm(); err != nil {
		log.Printf("Error parsing form data: %s", err)
		http.Error(w, "Internal Server Error", 500)
		return
	}
	action := request.FormValue("action")
	log.Printf("Request method=%s url=%s action=%s from client=%s\n", request.Method, request.URL, action, request.RemoteAddr)
	switch action {
	case "bet_request":
		gameState := parseGameState(request.FormValue("game_state"))
		if gameState == nil {
			http.Error(w, "Internal Server Error", 500)
			return
		}
		result := player.BetRequest(gameState)
		fmt.Fprintf(w, "%d", result)
		return
	case "showdown":
		gameState := parseGameState(request.FormValue("game_state"))
		if gameState == nil {
			http.Error(w, "Internal Server Error", 500)
			return
		}
		player.Showdown(gameState)
		fmt.Fprint(w, "")
		return
	case "version":
		fmt.Fprint(w, player.Version())
		return
	default:
		http.Error(w, "Invalid action", 400)
	}
}

func parseGameState(stateStr string) *player.GameState {
	stateBytes := []byte(stateStr)
	gameState := new(player.GameState)
	if err := json.Unmarshal(stateBytes, &gameState); err != nil {
		log.Printf("Error parsing game state: %s", err)
		return nil
	}
	return gameState
}
<commit_msg>Add check action<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"player"
)

const PORT = 4711

func main() {
	http.HandleFunc("/", handleRequest)
	if err := http.ListenAndServe(fmt.Sprintf(":%d", PORT), nil); err != nil {
		log.Fatal(err)
	}
}

func handleRequest(w http.ResponseWriter, request *http.Request) {
	if err := request.ParseForm(); err != nil {
		log.Printf("Error parsing form data: %s", err)
		http.Error(w, "Internal Server Error", 500)
		return
	}
	action := request.FormValue("action")
	log.Printf("Request method=%s url=%s action=%s from client=%s\n", request.Method, request.URL, action, request.RemoteAddr)
	switch action {
	case "check":
		fmt.Fprint(w, "")
		return
	case "bet_request":
		gameState := parseGameState(request.FormValue("game_state"))
		if gameState == nil {
			http.Error(w, "Internal Server Error", 500)
			return
		}
		result := player.BetRequest(gameState)
		fmt.Fprintf(w, "%d", result)
		return
	case "showdown":
		gameState := parseGameState(request.FormValue("game_state"))
		if gameState == nil {
			http.Error(w, "Internal Server Error", 500)
			return
		}
		player.Showdown(gameState)
		fmt.Fprint(w, "")
		return
	case "version":
		fmt.Fprint(w, player.Version())
		return
	default:
		http.Error(w, "Invalid action", 400)
	}
}

func parseGameState(stateStr string) *player.GameState {
	stateBytes := []byte(stateStr)
	gameState := new(player.GameState)
	if err := json.Unmarshal(stateBytes, &gameState); err != nil {
		log.Printf("Error parsing game state: %s", err)
		return nil
	}
	return gameState
}
<|endoftext|>


<commit_before>// +build linux

/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package volume

import (
	"path/filepath"
	"syscall"

	"os"
	"time"

	v1 "k8s.io/api/core/v1"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	"k8s.io/klog/v2"
	"k8s.io/kubernetes/pkg/features"
	"k8s.io/kubernetes/pkg/volume/util/types"
)

const (
	rwMask   = os.FileMode(0660)
	roMask   = os.FileMode(0440)
	execMask = os.FileMode(0110)
)

// SetVolumeOwnership modifies the given volume to be owned by
// fsGroup, and sets SetGid so that newly created files are owned by
// fsGroup. If fsGroup is nil nothing is done.
func SetVolumeOwnership(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(types.CompleteFuncParam)) error {
	if fsGroup == nil {
		return nil
	}

	fsGroupPolicyEnabled := utilfeature.DefaultFeatureGate.Enabled(features.ConfigurableFSGroupPolicy)

	timer := time.AfterFunc(30*time.Second, func() {
		klog.Warningf("Setting volume ownership for %s and fsGroup set. If the volume has a lot of files then setting volume ownership could be slow, see https://github.com/kubernetes/kubernetes/issues/69699", mounter.GetPath())
	})
	defer timer.Stop()

	// This code exists for legacy purposes, so as old behaviour is entirely preserved when feature gate is disabled
	// TODO: remove this when ConfigurableFSGroupPolicy turns GA.
	if !fsGroupPolicyEnabled {
		err := legacyOwnershipChange(mounter, fsGroup)
		if completeFunc != nil {
			completeFunc(types.CompleteFuncParam{
				Err: &err,
			})
		}
		return err
	}

	if skipPermissionChange(mounter, fsGroup, fsGroupChangePolicy) {
		klog.V(3).Infof("skipping permission and ownership change for volume %s", mounter.GetPath())
		return nil
	}

	err := walkDeep(mounter.GetPath(), func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		return changeFilePermission(path, fsGroup, mounter.GetAttributes().ReadOnly, info)
	})
	if completeFunc != nil {
		completeFunc(types.CompleteFuncParam{
			Err: &err,
		})
	}
	return err
}

func legacyOwnershipChange(mounter Mounter, fsGroup *int64) error {
	return filepath.Walk(mounter.GetPath(), func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		return changeFilePermission(path, fsGroup, mounter.GetAttributes().ReadOnly, info)
	})
}

func changeFilePermission(filename string, fsGroup *int64, readonly bool, info os.FileInfo) error {
	err := os.Lchown(filename, -1, int(*fsGroup))
	if err != nil {
		klog.Errorf("Lchown failed on %v: %v", filename, err)
	}

	// chmod passes through to the underlying file for symlinks.
	// Symlinks have a mode of 777 but this really doesn't mean anything.
	// The permissions of the underlying file are what matter.
	// However, if one reads the mode of a symlink then chmods the symlink
	// with that mode, it changes the mode of the underlying file, overridden
	// the defaultMode and permissions initialized by the volume plugin, which
	// is not what we want; thus, we skip chmod for symlinks.
	if info.Mode()&os.ModeSymlink != 0 {
		return nil
	}

	mask := rwMask
	if readonly {
		mask = roMask
	}

	if info.IsDir() {
		mask |= os.ModeSetgid
		mask |= execMask
	}

	err = os.Chmod(filename, info.Mode()|mask)
	if err != nil {
		klog.Errorf("Chmod failed on %v: %v", filename, err)
	}

	return nil
}

func skipPermissionChange(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy) bool {
	dir := mounter.GetPath()

	if fsGroupChangePolicy == nil || *fsGroupChangePolicy != v1.FSGroupChangeOnRootMismatch {
		klog.V(4).Infof("perform recursive ownership change for %s", dir)
		return false
	}
	return !requiresPermissionChange(mounter.GetPath(), fsGroup, mounter.GetAttributes().ReadOnly)
}

func requiresPermissionChange(rootDir string, fsGroup *int64, readonly bool) bool {
	fsInfo, err := os.Stat(rootDir)
	if err != nil {
		klog.Errorf("performing recursive ownership change on %s because reading permissions of root volume failed: %v", rootDir, err)
		return true
	}
	stat, ok := fsInfo.Sys().(*syscall.Stat_t)
	if !ok || stat == nil {
		klog.Errorf("performing recursive ownership change on %s because reading permissions of root volume failed", rootDir)
		return true
	}

	if int(stat.Gid) != int(*fsGroup) {
		klog.V(4).Infof("expected group ownership of volume %s did not match with: %d", rootDir, stat.Gid)
		return true
	}
	unixPerms := rwMask

	if readonly {
		unixPerms = roMask
	}

	// if rootDir is not a directory then we should apply permission change anyways
	if !fsInfo.IsDir() {
		return true
	}
	unixPerms |= execMask
	filePerm := fsInfo.Mode().Perm()

	// We need to check if actual permissions of root directory is a superset of permissions required by unixPerms.
	// This is done by checking if permission bits expected in unixPerms is set in actual permissions of the directory.
	// We use bitwise AND operation to check set bits. For example:
	//     unixPerms: 770, filePerms: 775 : 770&775 = 770 (perms on directory is a superset)
	//     unixPerms: 770, filePerms: 770 : 770&770 = 770 (perms on directory is a superset)
	//     unixPerms: 770, filePerms: 750 : 770&750 = 750 (perms on directory is NOT a superset)
	// We also need to check if setgid bits are set in permissions of the directory.
	if (unixPerms&filePerm != unixPerms) || (fsInfo.Mode()&os.ModeSetgid == 0) {
		klog.V(4).Infof("performing recursive ownership change on %s because of mismatching mode", rootDir)
		return true
	}
	return false
}

// readDirNames reads the directory named by dirname and returns
// a list of directory entries.
// We are not using filepath.readDirNames because we do not want to sort files found in a directory before changing
// permissions for performance reasons.
func readDirNames(dirname string) ([]string, error) {
	f, err := os.Open(dirname)
	if err != nil {
		return nil, err
	}
	names, err := f.Readdirnames(-1)
	f.Close()
	if err != nil {
		return nil, err
	}
	return names, nil
}

// walkDeep can be used to traverse directories and has two minor differences
// from filepath.Walk:
//   - List of files/dirs is not sorted for performance reasons
//   - callback walkFunc is invoked on root directory after visiting children dirs and files
func walkDeep(root string, walkFunc filepath.WalkFunc) error {
	info, err := os.Lstat(root)
	if err != nil {
		return walkFunc(root, nil, err)
	}
	return walk(root, info, walkFunc)
}

func walk(path string, info os.FileInfo, walkFunc filepath.WalkFunc) error {
	if !info.IsDir() {
		return walkFunc(path, info, nil)
	}
	names, err := readDirNames(path)
	if err != nil {
		return err
	}
	for _, name := range names {
		filename := filepath.Join(path, name)
		fileInfo, err := os.Lstat(filename)
		if err != nil {
			if err := walkFunc(filename, fileInfo, err); err != nil {
				return err
			}
		} else {
			err = walk(filename, fileInfo, walkFunc)
			if err != nil {
				return err
			}
		}
	}
	return walkFunc(path, info, nil)
}
<commit_msg>migrate log in pkg/volume/volume_linux.go<commit_after>// +build linux

/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package volume

import (
	"path/filepath"
	"syscall"

	"os"
	"time"

	v1 "k8s.io/api/core/v1"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	"k8s.io/klog/v2"
	"k8s.io/kubernetes/pkg/features"
	"k8s.io/kubernetes/pkg/volume/util/types"
)

const (
	rwMask   = os.FileMode(0660)
	roMask   = os.FileMode(0440)
	execMask = os.FileMode(0110)
)

// SetVolumeOwnership modifies the given volume to be owned by
// fsGroup, and sets SetGid so that newly created files are owned by
// fsGroup. If fsGroup is nil nothing is done.
func SetVolumeOwnership(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy, completeFunc func(types.CompleteFuncParam)) error {
	if fsGroup == nil {
		return nil
	}

	fsGroupPolicyEnabled := utilfeature.DefaultFeatureGate.Enabled(features.ConfigurableFSGroupPolicy)

	timer := time.AfterFunc(30*time.Second, func() {
		klog.Warningf("Setting volume ownership for %s and fsGroup set. If the volume has a lot of files then setting volume ownership could be slow, see https://github.com/kubernetes/kubernetes/issues/69699", mounter.GetPath())
	})
	defer timer.Stop()

	// This code exists for legacy purposes, so as old behaviour is entirely preserved when feature gate is disabled
	// TODO: remove this when ConfigurableFSGroupPolicy turns GA.
	if !fsGroupPolicyEnabled {
		err := legacyOwnershipChange(mounter, fsGroup)
		if completeFunc != nil {
			completeFunc(types.CompleteFuncParam{
				Err: &err,
			})
		}
		return err
	}

	if skipPermissionChange(mounter, fsGroup, fsGroupChangePolicy) {
		klog.V(3).InfoS("Skipping permission and ownership change for volume", "path", mounter.GetPath())
		return nil
	}

	err := walkDeep(mounter.GetPath(), func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		return changeFilePermission(path, fsGroup, mounter.GetAttributes().ReadOnly, info)
	})
	if completeFunc != nil {
		completeFunc(types.CompleteFuncParam{
			Err: &err,
		})
	}
	return err
}

func legacyOwnershipChange(mounter Mounter, fsGroup *int64) error {
	return filepath.Walk(mounter.GetPath(), func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		return changeFilePermission(path, fsGroup, mounter.GetAttributes().ReadOnly, info)
	})
}

func changeFilePermission(filename string, fsGroup *int64, readonly bool, info os.FileInfo) error {
	err := os.Lchown(filename, -1, int(*fsGroup))
	if err != nil {
		klog.ErrorS(err, "Lchown failed", "path", filename)
	}

	// chmod passes through to the underlying file for symlinks.
	// Symlinks have a mode of 777 but this really doesn't mean anything.
	// The permissions of the underlying file are what matter.
	// However, if one reads the mode of a symlink then chmods the symlink
	// with that mode, it changes the mode of the underlying file, overridden
	// the defaultMode and permissions initialized by the volume plugin, which
	// is not what we want; thus, we skip chmod for symlinks.
	if info.Mode()&os.ModeSymlink != 0 {
		return nil
	}

	mask := rwMask
	if readonly {
		mask = roMask
	}

	if info.IsDir() {
		mask |= os.ModeSetgid
		mask |= execMask
	}

	err = os.Chmod(filename, info.Mode()|mask)
	if err != nil {
		klog.ErrorS(err, "Chown failed", "path", filename)
	}

	return nil
}

func skipPermissionChange(mounter Mounter, fsGroup *int64, fsGroupChangePolicy *v1.PodFSGroupChangePolicy) bool {
	dir := mounter.GetPath()

	if fsGroupChangePolicy == nil || *fsGroupChangePolicy != v1.FSGroupChangeOnRootMismatch {
		klog.V(4).InfoS("Perform recursive ownership change for directory", "path", dir)
		return false
	}
	return !requiresPermissionChange(mounter.GetPath(), fsGroup, mounter.GetAttributes().ReadOnly)
}

func requiresPermissionChange(rootDir string, fsGroup *int64, readonly bool) bool {
	fsInfo, err := os.Stat(rootDir)
	if err != nil {
		klog.ErrorS(err, "Performing recursive ownership change on rootDir because reading permissions of root volume failed", "path", rootDir)
		return true
	}
	stat, ok := fsInfo.Sys().(*syscall.Stat_t)
	if !ok || stat == nil {
		klog.ErrorS(nil, "Performing recursive ownership change on rootDir because reading permissions of root volume failed", "path", rootDir)
		return true
	}

	if int(stat.Gid) != int(*fsGroup) {
		klog.V(4).InfoS("Expected group ownership of volume did not match with Gid", "path", rootDir, "GID", stat.Gid)
		return true
	}
	unixPerms := rwMask

	if readonly {
		unixPerms = roMask
	}

	// if rootDir is not a directory then we should apply permission change anyways
	if !fsInfo.IsDir() {
		return true
	}
	unixPerms |= execMask
	filePerm := fsInfo.Mode().Perm()

	// We need to check if actual permissions of root directory is a superset of permissions required by unixPerms.
	// This is done by checking if permission bits expected in unixPerms is set in actual permissions of the directory.
	// We use bitwise AND operation to check set bits. For example:
	//     unixPerms: 770, filePerms: 775 : 770&775 = 770 (perms on directory is a superset)
	//     unixPerms: 770, filePerms: 770 : 770&770 = 770 (perms on directory is a superset)
	//     unixPerms: 770, filePerms: 750 : 770&750 = 750 (perms on directory is NOT a superset)
	// We also need to check if setgid bits are set in permissions of the directory.
	if (unixPerms&filePerm != unixPerms) || (fsInfo.Mode()&os.ModeSetgid == 0) {
		klog.V(4).InfoS("Performing recursive ownership change on rootDir because of mismatching mode", "path", rootDir)
		return true
	}
	return false
}

// readDirNames reads the directory named by dirname and returns
// a list of directory entries.
// We are not using filepath.readDirNames because we do not want to sort files found in a directory before changing
// permissions for performance reasons.
func readDirNames(dirname string) ([]string, error) {
	f, err := os.Open(dirname)
	if err != nil {
		return nil, err
	}
	names, err := f.Readdirnames(-1)
	f.Close()
	if err != nil {
		return nil, err
	}
	return names, nil
}

// walkDeep can be used to traverse directories and has two minor differences
// from filepath.Walk:
//   - List of files/dirs is not sorted for performance reasons
//   - callback walkFunc is invoked on root directory after visiting children dirs and files
func walkDeep(root string, walkFunc filepath.WalkFunc) error {
	info, err := os.Lstat(root)
	if err != nil {
		return walkFunc(root, nil, err)
	}
	return walk(root, info, walkFunc)
}

func walk(path string, info os.FileInfo, walkFunc filepath.WalkFunc) error {
	if !info.IsDir() {
		return walkFunc(path, info, nil)
	}
	names, err := readDirNames(path)
	if err != nil {
		return err
	}
	for _, name := range names {
		filename := filepath.Join(path, name)
		fileInfo, err := os.Lstat(filename)
		if err != nil {
			if err := walkFunc(filename, fileInfo, err); err != nil {
				return err
			}
		} else {
			err = walk(filename, fileInfo, walkFunc)
			if err != nil {
				return err
			}
		}
	}
	return walkFunc(path, info, nil)
}
<|endoftext|>


<commit_before>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fileutil

import (
	"os"

	"github.com/pkg/errors"
)

type MmapFile struct {
	f *os.File
	b []byte
}

func OpenMmapFile(path string) (*MmapFile, error) {
	return OpenMmapFileWithSize(path, 0)
}

func OpenMmapFileWithSize(path string, size int) (mf *MmapFile, retErr error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, errors.Wrap(err, "try lock file")
	}
	defer func() {
		if retErr != nil {
			f.Close()
		}
	}()
	if size <= 0 {
		info, err := f.Stat()
		if err != nil {
			return nil, errors.Wrap(err, "stat")
		}
		size = int(info.Size())
	}

	b, err := mmap(f, size)
	if err != nil {
		return nil, errors.Wrap(err, "mmap")
	}

	return &MmapFile{f: f, b: b}, nil
}

func (f *MmapFile) Close() error {
	err0 := munmap(f.b)
	err1 := f.f.Close()

	if err0 != nil {
		return err0
	}
	return err1
}

func (f *MmapFile) File() *os.File {
	return f.f
}

func (f *MmapFile) Bytes() []byte {
	return f.b
}
<commit_msg>More info in mmap error message (#8058)<commit_after>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fileutil

import (
	"os"

	"github.com/pkg/errors"
)

type MmapFile struct {
	f *os.File
	b []byte
}

func OpenMmapFile(path string) (*MmapFile, error) {
	return OpenMmapFileWithSize(path, 0)
}

func OpenMmapFileWithSize(path string, size int) (mf *MmapFile, retErr error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, errors.Wrap(err, "try lock file")
	}
	defer func() {
		if retErr != nil {
			f.Close()
		}
	}()
	if size <= 0 {
		info, err := f.Stat()
		if err != nil {
			return nil, errors.Wrap(err, "stat")
		}
		size = int(info.Size())
	}

	b, err := mmap(f, size)
	if err != nil {
		return nil, errors.Wrapf(err, "mmap, size %d", size)
	}

	return &MmapFile{f: f, b: b}, nil
}

func (f *MmapFile) Close() error {
	err0 := munmap(f.b)
	err1 := f.f.Close()

	if err0 != nil {
		return err0
	}
	return err1
}

func (f *MmapFile) File() *os.File {
	return f.f
}

func (f *MmapFile) Bytes() []byte {
	return f.b
}
<|endoftext|>


<commit_before>package finalize

const (
	initScript = `
# ------------------------------------------------------------------------------------------------
# Copyright 2013 Jordon Bedwell.
# Apache License.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
# except  in compliance with the License. You may obtain a copy of the License at:
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the
# License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied. See the License for the specific language governing permissions
# and  limitations under the License.
# ------------------------------------------------------------------------------------------------

export APP_ROOT=$HOME
export LD_LIBRARY_PATH=$APP_ROOT/nginx/lib:$LD_LIBRARY_PATH

mv $APP_ROOT/nginx/conf/nginx.conf $APP_ROOT/nginx/conf/nginx.conf.erb
erb $APP_ROOT/nginx/conf/nginx.conf.erb > $APP_ROOT/nginx/conf/nginx.conf

if [[ ! -f $APP_ROOT/nginx/logs/access.log ]]; then
    mkfifo $APP_ROOT/nginx/logs/access.log
fi

if [[ ! -f $APP_ROOT/nginx/logs/error.log ]]; then
    mkfifo $APP_ROOT/nginx/logs/error.log
fi
`

	startLoggingScript = `
cat < $APP_ROOT/nginx/logs/access.log &
(>&2 cat) < $APP_ROOT/nginx/logs/error.log &
`

	startCommand = `#!/bin/sh
set -ex
$APP_ROOT/start_logging.sh
nginx -p $APP_ROOT/nginx -c $APP_ROOT/nginx/conf/nginx.conf
`

	nginxConfTemplate = `
worker_processes 1;
daemon off;

error_log <%= ENV["APP_ROOT"] %>/nginx/logs/error.log;
events { worker_connections 1024; }

http {
  charset utf-8;
  log_format cloudfoundry '$http_x_forwarded_for - $http_referer - [$time_local] "$request" $status $body_bytes_sent';
  access_log <%= ENV["APP_ROOT"] %>/nginx/logs/access.log cloudfoundry;
  default_type application/octet-stream;
  include mime.types;
  sendfile on;

  gzip on;
  gzip_disable "msie6";
  gzip_comp_level 6;
  gzip_min_length 1100;
  gzip_buffers 16 8k;
  gzip_proxied any;
  gunzip on;
  gzip_static always;
  gzip_types text/plain text/css text/js text/xml text/javascript application/javascript application/x-javascript application/json application/xml application/xml+rss;
  gzip_vary on;

  tcp_nopush on;
  keepalive_timeout 30;
  port_in_redirect off; # Ensure that redirects don't include the internal container PORT - <%= ENV["PORT"] %>
  server_tokens off;

  server {
    listen <%= ENV["PORT"] %>;
    server_name localhost;

    root <%= ENV["APP_ROOT"] %>/public;

   {{if .ForceHTTPS}}
     if ($http_x_forwarded_proto != "https") {
       return 301 https://$host$request_uri;
     }
   {{else}}
   <% if ENV["FORCE_HTTPS"] %>
     if ($http_x_forwarded_proto != "https") {
       return 301 https://$host$request_uri;
     }
   <% end %>
   {{end}}

    location / {
      {{if .PushState}}
        if (!-e $request_filename) {
          rewrite ^(.*)$ / break;
        }
      {{end}}

        index index.html index.htm Default.htm;

      {{if .DirectoryIndex}}
        autoindex on;
      {{end}}

      {{if .BasicAuth}}
        auth_basic "Restricted";  #For Basic Auth
        auth_basic_user_file <%= ENV["APP_ROOT"] %>/nginx/conf/.htpasswd;
      {{end}}

      {{if .SSI}}
        ssi on;
      {{end}}

      {{if .HSTS}}
        add_header Strict-Transport-Security "max-age=31536000{{if .HSTSIncludeSubDomains}}; includeSubDomains{{end}}{{if .HSTSPreload}}; preload{{end}}";
      {{end}}

      {{if ne .LocationInclude ""}}
        include {{.LocationInclude}};
      {{end}}

			{{ range $code, $value := .StatusCodes }}
			  error_page {{ $code }} {{ $value }};
		  {{ end }}
    }

    {{if not .HostDotFiles}}
      location ~ /\. {
        deny all;
        return 404;
      }
    {{end}}
  }
}
`
	MimeTypes = `
types {
  text/html html htm shtml;
  text/css css;
  text/xml xml;
  image/gif gif;
  image/jpeg jpeg jpg;
  application/x-javascript js;
  application/atom+xml atom;
  application/rss+xml rss;
  font/ttf ttf;
  font/woff woff;
  font/woff2 woff2;
  text/mathml mml;
  text/plain txt;
  text/vnd.sun.j2me.app-descriptor jad;
  text/vnd.wap.wml wml;
  text/x-component htc;
  text/cache-manifest manifest;
  image/png png;
  image/tiff tif tiff;
  image/vnd.wap.wbmp wbmp;
  image/x-icon ico;
  image/x-jng jng;
  image/x-ms-bmp bmp;
  image/svg+xml svg svgz;
  image/webp webp;
  application/java-archive jar war ear;
  application/mac-binhex40 hqx;
  application/msword doc;
  application/pdf pdf;
  application/postscript ps eps ai;
  application/rtf rtf;
  application/vnd.ms-excel xls;
  application/vnd.ms-powerpoint ppt;
  application/vnd.wap.wmlc wmlc;
  application/vnd.google-earth.kml+xml  kml;
  application/vnd.google-earth.kmz kmz;
  application/x-7z-compressed 7z;
  application/x-cocoa cco;
  application/x-java-archive-diff jardiff;
  application/x-java-jnlp-file jnlp;
  application/x-makeself run;
  application/x-perl pl pm;
  application/x-pilot prc pdb;
  application/x-rar-compressed rar;
  application/x-redhat-package-manager  rpm;
  application/x-sea sea;
  application/x-shockwave-flash swf;
  application/x-stuffit sit;
  application/x-tcl tcl tk;
  application/x-x509-ca-cert der pem crt;
  application/x-xpinstall xpi;
  application/xhtml+xml xhtml;
  application/zip zip;
  application/octet-stream bin exe dll;
  application/octet-stream deb;
  application/octet-stream dmg;
  application/octet-stream eot;
  application/octet-stream iso img;
  application/octet-stream msi msp msm;
  application/json json;
  audio/midi mid midi kar;
  audio/mpeg mp3;
  audio/ogg ogg;
  audio/x-m4a m4a;
  audio/x-realaudio ra;
  video/3gpp 3gpp 3gp;
  video/mp4 mp4;
  video/mpeg mpeg mpg;
  video/quicktime mov;
  video/webm webm;
  video/x-flv flv;
  video/x-m4v m4v;
  video/x-mng mng;
  video/x-ms-asf asx asf;
  video/x-ms-wmv wmv;
  video/x-msvideo avi;
}
`
)
<commit_msg>Preserves x_forwarded_host on https<commit_after>package finalize

const (
	initScript = `
# ------------------------------------------------------------------------------------------------
# Copyright 2013 Jordon Bedwell.
# Apache License.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
# except  in compliance with the License. You may obtain a copy of the License at:
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the
# License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied. See the License for the specific language governing permissions
# and  limitations under the License.
# ------------------------------------------------------------------------------------------------

export APP_ROOT=$HOME
export LD_LIBRARY_PATH=$APP_ROOT/nginx/lib:$LD_LIBRARY_PATH

mv $APP_ROOT/nginx/conf/nginx.conf $APP_ROOT/nginx/conf/nginx.conf.erb
erb $APP_ROOT/nginx/conf/nginx.conf.erb > $APP_ROOT/nginx/conf/nginx.conf

if [[ ! -f $APP_ROOT/nginx/logs/access.log ]]; then
    mkfifo $APP_ROOT/nginx/logs/access.log
fi

if [[ ! -f $APP_ROOT/nginx/logs/error.log ]]; then
    mkfifo $APP_ROOT/nginx/logs/error.log
fi
`

	startLoggingScript = `
cat < $APP_ROOT/nginx/logs/access.log &
(>&2 cat) < $APP_ROOT/nginx/logs/error.log &
`

	startCommand = `#!/bin/sh
set -ex
$APP_ROOT/start_logging.sh
nginx -p $APP_ROOT/nginx -c $APP_ROOT/nginx/conf/nginx.conf
`

	nginxConfTemplate = `
worker_processes 1;
daemon off;

error_log <%= ENV["APP_ROOT"] %>/nginx/logs/error.log;
events { worker_connections 1024; }

http {
  charset utf-8;
  log_format cloudfoundry '$http_x_forwarded_for - $http_referer - [$time_local] "$request" $status $body_bytes_sent';
  access_log <%= ENV["APP_ROOT"] %>/nginx/logs/access.log cloudfoundry;
  default_type application/octet-stream;
  include mime.types;
  sendfile on;

  gzip on;
  gzip_disable "msie6";
  gzip_comp_level 6;
  gzip_min_length 1100;
  gzip_buffers 16 8k;
  gzip_proxied any;
  gunzip on;
  gzip_static always;
  gzip_types text/plain text/css text/js text/xml text/javascript application/javascript application/x-javascript application/json application/xml application/xml+rss;
  gzip_vary on;

  tcp_nopush on;
  keepalive_timeout 30;
  port_in_redirect off; # Ensure that redirects don't include the internal container PORT - <%= ENV["PORT"] %>
  server_tokens off;

  server {
    listen <%= ENV["PORT"] %>;
    server_name localhost;

    root <%= ENV["APP_ROOT"] %>/public;

   {{if .ForceHTTPS}}
	 set $updated_host $host;
	 if ($http_x_forwarded_host != "") {
       set $updated_host $http_x_forwarded_host;
     }

     if ($http_x_forwarded_proto != "https") {
	   return 301 https://$updated_host$request_uri;
	 }
   {{else}}
	<% if ENV["FORCE_HTTPS"] %>
	 set $updated_host $host;
	 if ($http_x_forwarded_host != "") {
       set $updated_host $http_x_forwarded_host;
     }

     if ($http_x_forwarded_proto != "https") {
	   return 301 https://$updated_host$request_uri;
	 }
	<% end %>
   {{end}}

    location / {
      {{if .PushState}}
        if (!-e $request_filename) {
          rewrite ^(.*)$ / break;
        }
      {{end}}

        index index.html index.htm Default.htm;

      {{if .DirectoryIndex}}
        autoindex on;
      {{end}}

      {{if .BasicAuth}}
        auth_basic "Restricted";  #For Basic Auth
        auth_basic_user_file <%= ENV["APP_ROOT"] %>/nginx/conf/.htpasswd;
      {{end}}

      {{if .SSI}}
        ssi on;
      {{end}}

      {{if .HSTS}}
        add_header Strict-Transport-Security "max-age=31536000{{if .HSTSIncludeSubDomains}}; includeSubDomains{{end}}{{if .HSTSPreload}}; preload{{end}}";
      {{end}}

      {{if ne .LocationInclude ""}}
        include {{.LocationInclude}};
      {{end}}

			{{ range $code, $value := .StatusCodes }}
			  error_page {{ $code }} {{ $value }};
		  {{ end }}
    }

    {{if not .HostDotFiles}}
      location ~ /\. {
        deny all;
        return 404;
      }
    {{end}}
  }
}
`
	MimeTypes = `
types {
  text/html html htm shtml;
  text/css css;
  text/xml xml;
  image/gif gif;
  image/jpeg jpeg jpg;
  application/x-javascript js;
  application/atom+xml atom;
  application/rss+xml rss;
  font/ttf ttf;
  font/woff woff;
  font/woff2 woff2;
  text/mathml mml;
  text/plain txt;
  text/vnd.sun.j2me.app-descriptor jad;
  text/vnd.wap.wml wml;
  text/x-component htc;
  text/cache-manifest manifest;
  image/png png;
  image/tiff tif tiff;
  image/vnd.wap.wbmp wbmp;
  image/x-icon ico;
  image/x-jng jng;
  image/x-ms-bmp bmp;
  image/svg+xml svg svgz;
  image/webp webp;
  application/java-archive jar war ear;
  application/mac-binhex40 hqx;
  application/msword doc;
  application/pdf pdf;
  application/postscript ps eps ai;
  application/rtf rtf;
  application/vnd.ms-excel xls;
  application/vnd.ms-powerpoint ppt;
  application/vnd.wap.wmlc wmlc;
  application/vnd.google-earth.kml+xml  kml;
  application/vnd.google-earth.kmz kmz;
  application/x-7z-compressed 7z;
  application/x-cocoa cco;
  application/x-java-archive-diff jardiff;
  application/x-java-jnlp-file jnlp;
  application/x-makeself run;
  application/x-perl pl pm;
  application/x-pilot prc pdb;
  application/x-rar-compressed rar;
  application/x-redhat-package-manager  rpm;
  application/x-sea sea;
  application/x-shockwave-flash swf;
  application/x-stuffit sit;
  application/x-tcl tcl tk;
  application/x-x509-ca-cert der pem crt;
  application/x-xpinstall xpi;
  application/xhtml+xml xhtml;
  application/zip zip;
  application/octet-stream bin exe dll;
  application/octet-stream deb;
  application/octet-stream dmg;
  application/octet-stream eot;
  application/octet-stream iso img;
  application/octet-stream msi msp msm;
  application/json json;
  audio/midi mid midi kar;
  audio/mpeg mp3;
  audio/ogg ogg;
  audio/x-m4a m4a;
  audio/x-realaudio ra;
  video/3gpp 3gpp 3gp;
  video/mp4 mp4;
  video/mpeg mpeg mpg;
  video/quicktime mov;
  video/webm webm;
  video/x-flv flv;
  video/x-m4v m4v;
  video/x-mng mng;
  video/x-ms-asf asx asf;
  video/x-ms-wmv wmv;
  video/x-msvideo avi;
}
`
)
<|endoftext|>


<commit_before>package qdisksync

import (
	"bufio"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync/atomic"
	"time"
)

//read from snapshot and copy
func SyncVolumeData(srcVolume string, destVolume string, bufferSize int64, workerCount int32) {
	runtime.GOMAXPROCS(runtime.NumCPU())
	cacheFile := "qdisksync.cache"
	cacheFileH, err := os.Open(cacheFile)
	if err != nil {
		L.Error("Load cache file `%s' failed", cacheFile)
		return
	}
	defer cacheFileH.Close()

	bReader := bufio.NewScanner(cacheFileH)
	bReader.Split(bufio.ScanLines)
	//init channel
	var allWorkers int32 = 0
	syncStart := time.Now()
	L.Informational("Sync `%s' -> `'%s start from `%s'", srcVolume, destVolume, syncStart.String())
	syncDone := make(chan bool)
	//receive the sync result
	go func() {
		<-syncDone
	}()
	for bReader.Scan() {
		line := bReader.Text()
		//split to name and size
		items := strings.Split(line, "\t")
		if len(items) != 3 {
			L.Error("Line data `%s'' error of cache file `%s'", line, cacheFile)
			continue
		}
		fname := items[0]
		fsize, err := strconv.ParseInt(items[1], 10, 64)
		if err != nil {
			L.Error("File length error `%s' for line `%s'", items[1], line)
			continue
		}
		fperm, err := strconv.ParseInt(items[2], 10, 64)
		if err != nil {
			L.Error("File perm error `%s' for line `%s'", items[2], line)
			continue
		}
		//join the path
		srcFullPath := filepath.Join(srcVolume, fname)
		destFullPath := filepath.Join(destVolume, fname)
		//check src and dest file
		srcFileH, srcErr := os.Open(srcFullPath)
		if srcErr != nil {
			L.Error("Open src file `%s' error `%s'", srcFullPath, srcErr.Error())
			continue
		}
		//create path if necessary
		lastSlashIndex := strings.LastIndex(destFullPath, "/")
		destFullPathBase := destFullPath[:lastSlashIndex]
		if err := os.MkdirAll(destFullPathBase, 0775); err != nil {
			L.Error("Failed to create dir `%s' due to error `%s'", destFullPathBase, err.Error())
			continue
		}
		destFileH, destErr := os.OpenFile(destFullPath, os.O_CREATE|os.O_RDWR, os.FileMode(fperm))
		if destErr != nil {
			L.Error("Open dest file `%s' error `%s'", destFullPath, destErr.Error())
			continue
		}

		//check whether it's time to run copy
		for {
			curWorkers := atomic.LoadInt32(&allWorkers)
			L.Debug("Current Workers: `%d'", curWorkers)
			if curWorkers < workerCount {
				atomic.AddInt32(&allWorkers, 1)
				go copy(srcFileH, destFileH, fsize, bufferSize, srcFullPath, destFullPath, &allWorkers)
				break
			} else {
				//wait some time to avoid infinite cycle
				<-time.After(time.Microsecond * 1)
			}
		}
	}
	for {
		L.Debug("Remaing workers: `%s'", atomic.LoadInt32(&allWorkers))
		if atomic.LoadInt32(&allWorkers) == 0 {
			syncDone <- true
			syncEnd := time.Now()
			L.Informational("Sync `%s' -> `%s' end at `%s'", srcVolume, destVolume, syncEnd.String())
			L.Informational("Sync `%s' -> `%s' last for `%s'", srcVolume, destVolume, time.Since(syncStart))
			break
		} else {
			<-time.After(time.Second * 5)
		}
	}
}

func copy(srcFileH, destFileH *os.File, fsize int64, bufferSize int64, srcFullPath, destFullPath string, allWorkers *int32) {
	defer func() {
		atomic.AddInt32(allWorkers, -1)
		runtime.Gosched()
	}()
	L.Debug("Copying from `%s' to `%s'", srcFullPath, destFullPath)
	buffer := make([]byte, bufferSize)
	var cpErr error
	var cpNum int64
	for {
		numRead, errRead := srcFileH.Read(buffer)
		if errRead == io.EOF {
			break
		} else {
			if errRead != nil {
				L.Error("Read from `%s' error: `%s'", srcFullPath, errRead.Error())
				cpErr = errRead
				break
			} else {
				numWrite, errWrite := destFileH.Write(buffer[:numRead])
				if errWrite != nil {
					L.Error("Write to `%s' error: `%s'", destFullPath, errWrite.Error())
					cpErr = errWrite
					break
				}
				cpNum += int64(numWrite)
			}
		}
	}
	defer srcFileH.Close()
	defer destFileH.Close()
	if cpErr != nil || cpNum != fsize {
		L.Error("Copy from `%s' to `%s' failed, error: `%s'", srcFullPath, destFullPath, cpErr.Error())
	} else {
		L.Debug("Copy from `%s' to `%s' succcess", srcFullPath, destFullPath)
	}

}
<commit_msg>Fix debug msg errorwq<commit_after>package qdisksync

import (
	"bufio"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync/atomic"
	"time"
)

//read from snapshot and copy
func SyncVolumeData(srcVolume string, destVolume string, bufferSize int64, workerCount int32) {
	runtime.GOMAXPROCS(runtime.NumCPU())
	cacheFile := "qdisksync.cache"
	cacheFileH, err := os.Open(cacheFile)
	if err != nil {
		L.Error("Load cache file `%s' failed", cacheFile)
		return
	}
	defer cacheFileH.Close()

	bReader := bufio.NewScanner(cacheFileH)
	bReader.Split(bufio.ScanLines)
	//init channel
	var allWorkers int32 = 0
	syncStart := time.Now()
	L.Informational("Sync `%s' -> `'%s start from `%s'", srcVolume, destVolume, syncStart.String())
	syncDone := make(chan bool)
	//receive the sync result
	go func() {
		<-syncDone
	}()
	for bReader.Scan() {
		line := bReader.Text()
		//split to name and size
		items := strings.Split(line, "\t")
		if len(items) != 3 {
			L.Error("Line data `%s'' error of cache file `%s'", line, cacheFile)
			continue
		}
		fname := items[0]
		fsize, err := strconv.ParseInt(items[1], 10, 64)
		if err != nil {
			L.Error("File length error `%s' for line `%s'", items[1], line)
			continue
		}
		fperm, err := strconv.ParseInt(items[2], 10, 64)
		if err != nil {
			L.Error("File perm error `%s' for line `%s'", items[2], line)
			continue
		}
		//join the path
		srcFullPath := filepath.Join(srcVolume, fname)
		destFullPath := filepath.Join(destVolume, fname)
		//check src and dest file
		srcFileH, srcErr := os.Open(srcFullPath)
		if srcErr != nil {
			L.Error("Open src file `%s' error `%s'", srcFullPath, srcErr.Error())
			continue
		}
		//create path if necessary
		lastSlashIndex := strings.LastIndex(destFullPath, "/")
		destFullPathBase := destFullPath[:lastSlashIndex]
		if err := os.MkdirAll(destFullPathBase, 0775); err != nil {
			L.Error("Failed to create dir `%s' due to error `%s'", destFullPathBase, err.Error())
			continue
		}
		destFileH, destErr := os.OpenFile(destFullPath, os.O_CREATE|os.O_RDWR, os.FileMode(fperm))
		if destErr != nil {
			L.Error("Open dest file `%s' error `%s'", destFullPath, destErr.Error())
			continue
		}

		//check whether it's time to run copy
		for {
			curWorkers := atomic.LoadInt32(&allWorkers)
			L.Debug("Current Workers: `%d'", curWorkers)
			if curWorkers < workerCount {
				atomic.AddInt32(&allWorkers, 1)
				go copy(srcFileH, destFileH, fsize, bufferSize, srcFullPath, destFullPath, &allWorkers)
				break
			} else {
				//wait some time to avoid infinite cycle
				<-time.After(time.Microsecond * 1)
			}
		}
	}
	for {
		L.Debug("Remained workers: `%d'", atomic.LoadInt32(&allWorkers))
		if atomic.LoadInt32(&allWorkers) == 0 {
			syncDone <- true
			syncEnd := time.Now()
			L.Informational("Sync `%s' -> `%s' end at `%s'", srcVolume, destVolume, syncEnd.String())
			L.Informational("Sync `%s' -> `%s' last for `%s'", srcVolume, destVolume, time.Since(syncStart))
			break
		} else {
			<-time.After(time.Second * 5)
		}
	}
}

func copy(srcFileH, destFileH *os.File, fsize int64, bufferSize int64, srcFullPath, destFullPath string, allWorkers *int32) {
	defer func() {
		atomic.AddInt32(allWorkers, -1)
		runtime.Gosched()
	}()
	L.Debug("Copying from `%s' to `%s'", srcFullPath, destFullPath)
	buffer := make([]byte, bufferSize)
	var cpErr error
	var cpNum int64
	for {
		numRead, errRead := srcFileH.Read(buffer)
		if errRead == io.EOF {
			break
		} else {
			if errRead != nil {
				L.Error("Read from `%s' error: `%s'", srcFullPath, errRead.Error())
				cpErr = errRead
				break
			} else {
				numWrite, errWrite := destFileH.Write(buffer[:numRead])
				if errWrite != nil {
					L.Error("Write to `%s' error: `%s'", destFullPath, errWrite.Error())
					cpErr = errWrite
					break
				}
				cpNum += int64(numWrite)
			}
		}
	}
	defer srcFileH.Close()
	defer destFileH.Close()
	if cpErr != nil || cpNum != fsize {
		L.Error("Copy from `%s' to `%s' failed, error: `%s'", srcFullPath, destFullPath, cpErr.Error())
	} else {
		L.Debug("Copy from `%s' to `%s' succcess", srcFullPath, destFullPath)
	}

}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2016 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package probes

import (
	"fmt"
	"sync"

	"github.com/skydive-project/skydive/analyzer"
	"github.com/skydive-project/skydive/api"
	"github.com/skydive-project/skydive/config"
	"github.com/skydive-project/skydive/flow"
	"github.com/skydive-project/skydive/flow/mappings"
	"github.com/skydive-project/skydive/logging"
	"github.com/skydive-project/skydive/probe"
	"github.com/skydive-project/skydive/topology/graph"
)

type FlowProbeBundle struct {
	probe.ProbeBundle
	Graph              *graph.Graph
	FlowTableAllocator *flow.TableAllocator
}

type FlowProbeInterface interface {
	probe.Probe
	RegisterProbe(n *graph.Node, capture *api.Capture, ft *flow.Table) error
	UnregisterProbe(n *graph.Node) error
}

type FlowProbe struct {
	sync.RWMutex
	fpi            FlowProbeInterface
	pipeline       *mappings.FlowMappingPipeline
	flowClientPool *analyzer.FlowClientPool
}

func (fp FlowProbe) Start() {
	fp.fpi.Start()
}

func (fp FlowProbe) Stop() {
	fp.fpi.Stop()
}

func (fp *FlowProbe) RegisterProbe(n *graph.Node, capture *api.Capture, ft *flow.Table) error {
	return fp.fpi.RegisterProbe(n, capture, ft)
}

func (fp *FlowProbe) UnregisterProbe(n *graph.Node) error {
	return fp.fpi.UnregisterProbe(n)
}

func (fp *FlowProbe) AsyncFlowPipeline(flows []*flow.Flow) {
	fp.pipeline.Enhance(flows)

	fp.RLock()
	defer fp.RUnlock()

	fp.flowClientPool.SendFlows(flows)
}

func (fpb *FlowProbeBundle) UnregisterAllProbes() {
	fpb.Graph.Lock()
	defer fpb.Graph.Unlock()

	for _, n := range fpb.Graph.GetNodes(graph.Metadata{}) {
		for _, p := range fpb.ProbeBundle.Probes {
			fprobe := p.(*FlowProbe)
			fprobe.UnregisterProbe(n)
		}
	}
}

func NewFlowProbeBundleFromConfig(tb *probe.ProbeBundle, g *graph.Graph, fta *flow.TableAllocator, fcpool *analyzer.FlowClientPool) *FlowProbeBundle {
	list := config.GetConfig().GetStringSlice("agent.flow.probes")
	logging.GetLogger().Infof("Flow probes: %v", list)

	pipeline := mappings.NewFlowMappingPipeline(mappings.NewGraphFlowEnhancer(g))

	// check that the neutron probe if loaded if so add the neutron flow enhancer
	if tb.GetProbe("neutron") != nil {
		pipeline.AddEnhancer(mappings.NewNeutronFlowEnhancer(g))
	}

	var captureTypes []string
	var fpi FlowProbeInterface
	var err error

	probes := make(map[string]probe.Probe)
	for _, t := range list {
		if _, ok := probes[t]; ok {
			continue
		}

		switch t {
		case "pcapsocket":
			fpi, err = NewPcapSocketProbeHandler(g)
			captureTypes = []string{"pcapsocket"}
		case "ovssflow":
			fpi, err = NewOvsSFlowProbesHandler(tb, g)
			captureTypes = []string{"ovssflow"}
		case "gopacket":
			fpi, err = NewGoPacketProbesHandler(g)
			captureTypes = []string{"afpacket", "pcap"}
		default:
			err = fmt.Errorf("unknown probe type %s", t)
		}

		if err != nil {
			logging.GetLogger().Errorf("failed to create %s probe: %s", t, err.Error())
			continue
		}

		flowProbe := &FlowProbe{fpi: fpi, pipeline: pipeline, flowClientPool: fcpool}
		for _, captureType := range captureTypes {
			probes[captureType] = flowProbe
		}
	}

	p := probe.NewProbeBundle(probes)

	return &FlowProbeBundle{
		ProbeBundle:        *p,
		Graph:              g,
		FlowTableAllocator: fta,
	}
}
<commit_msg>flow : FlowProbe remove useless lock<commit_after>/*
 * Copyright (C) 2016 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package probes

import (
	"fmt"

	"github.com/skydive-project/skydive/analyzer"
	"github.com/skydive-project/skydive/api"
	"github.com/skydive-project/skydive/config"
	"github.com/skydive-project/skydive/flow"
	"github.com/skydive-project/skydive/flow/mappings"
	"github.com/skydive-project/skydive/logging"
	"github.com/skydive-project/skydive/probe"
	"github.com/skydive-project/skydive/topology/graph"
)

type FlowProbeBundle struct {
	probe.ProbeBundle
	Graph              *graph.Graph
	FlowTableAllocator *flow.TableAllocator
}

type FlowProbeInterface interface {
	probe.Probe
	RegisterProbe(n *graph.Node, capture *api.Capture, ft *flow.Table) error
	UnregisterProbe(n *graph.Node) error
}

type FlowProbe struct {
	fpi            FlowProbeInterface
	pipeline       *mappings.FlowMappingPipeline
	flowClientPool *analyzer.FlowClientPool
}

func (fp *FlowProbe) Start() {
	fp.fpi.Start()
}

func (fp *FlowProbe) Stop() {
	fp.fpi.Stop()
}

func (fp *FlowProbe) RegisterProbe(n *graph.Node, capture *api.Capture, ft *flow.Table) error {
	return fp.fpi.RegisterProbe(n, capture, ft)
}

func (fp *FlowProbe) UnregisterProbe(n *graph.Node) error {
	return fp.fpi.UnregisterProbe(n)
}

func (fp *FlowProbe) AsyncFlowPipeline(flows []*flow.Flow) {
	fp.pipeline.Enhance(flows)
	fp.flowClientPool.SendFlows(flows)
}

func (fpb *FlowProbeBundle) UnregisterAllProbes() {
	fpb.Graph.Lock()
	defer fpb.Graph.Unlock()

	for _, n := range fpb.Graph.GetNodes(graph.Metadata{}) {
		for _, p := range fpb.ProbeBundle.Probes {
			fprobe := p.(*FlowProbe)
			fprobe.UnregisterProbe(n)
		}
	}
}

func NewFlowProbeBundleFromConfig(tb *probe.ProbeBundle, g *graph.Graph, fta *flow.TableAllocator, fcpool *analyzer.FlowClientPool) *FlowProbeBundle {
	list := config.GetConfig().GetStringSlice("agent.flow.probes")
	logging.GetLogger().Infof("Flow probes: %v", list)

	pipeline := mappings.NewFlowMappingPipeline(mappings.NewGraphFlowEnhancer(g))

	// check that the neutron probe if loaded if so add the neutron flow enhancer
	if tb.GetProbe("neutron") != nil {
		pipeline.AddEnhancer(mappings.NewNeutronFlowEnhancer(g))
	}

	var captureTypes []string
	var fpi FlowProbeInterface
	var err error

	probes := make(map[string]probe.Probe)
	for _, t := range list {
		if _, ok := probes[t]; ok {
			continue
		}

		switch t {
		case "pcapsocket":
			fpi, err = NewPcapSocketProbeHandler(g)
			captureTypes = []string{"pcapsocket"}
		case "ovssflow":
			fpi, err = NewOvsSFlowProbesHandler(tb, g)
			captureTypes = []string{"ovssflow"}
		case "gopacket":
			fpi, err = NewGoPacketProbesHandler(g)
			captureTypes = []string{"afpacket", "pcap"}
		default:
			err = fmt.Errorf("unknown probe type %s", t)
		}

		if err != nil {
			logging.GetLogger().Errorf("failed to create %s probe: %s", t, err.Error())
			continue
		}

		flowProbe := &FlowProbe{fpi: fpi, pipeline: pipeline, flowClientPool: fcpool}
		for _, captureType := range captureTypes {
			probes[captureType] = flowProbe
		}
	}

	p := probe.NewProbeBundle(probes)

	return &FlowProbeBundle{
		ProbeBundle:        *p,
		Graph:              g,
		FlowTableAllocator: fta,
	}
}
<|endoftext|>


<commit_before>package flying_test

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"syscall"

	"github.com/concourse/testflight/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Flying", func() {
	var tmpdir string
	var fixture string

	BeforeEach(func() {
		var err error

		tmpdir, err = ioutil.TempDir("", "fly-test")
		Expect(err).NotTo(HaveOccurred())

		fixture = filepath.Join(tmpdir, "fixture")

		err = os.MkdirAll(fixture, 0755)
		Expect(err).NotTo(HaveOccurred())

		err = ioutil.WriteFile(
			filepath.Join(fixture, "run"),
			[]byte(`#!/bin/sh
echo some output
echo FOO is $FOO
echo ARGS are "$@"
exit 0
`),
			0755,
		)
		Expect(err).NotTo(HaveOccurred())

		err = ioutil.WriteFile(
			filepath.Join(fixture, "task.yml"),
			[]byte(`---
platform: linux

image_resource:
  type: docker-image
  source: {repository: busybox}

inputs:
- name: fixture

outputs:
- name: output-1
- name: output-2

params:
  FOO: 1

run:
  path: fixture/run
`),
			0644,
		)
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		os.RemoveAll(tmpdir)
	})

	It("works", func() {
		fly := exec.Command(flyBin, "-t", targetedConcourse, "execute", "-c", "task.yml", "--", "SOME", "ARGS")
		fly.Dir = fixture

		session := helpers.StartFly(fly)

		Eventually(session).Should(gexec.Exit(0))

		Expect(session).To(gbytes.Say("some output"))
		Expect(session).To(gbytes.Say("FOO is 1"))
		Expect(session).To(gbytes.Say("ARGS are SOME ARGS"))
	})

	Describe("hijacking", func() {
		It("executes an interactive command in a running task's container", func() {
			err := ioutil.WriteFile(
				filepath.Join(fixture, "run"),
				[]byte(`#!/bin/sh
mkfifo /tmp/fifo
echo waiting
cat < /tmp/fifo
`),
				0755,
			)
			Expect(err).NotTo(HaveOccurred())

			fly := exec.Command(flyBin, "-t", targetedConcourse, "execute", "-c", "task.yml")
			fly.Dir = fixture

			flyS := helpers.StartFly(fly)

			Eventually(flyS).Should(gbytes.Say("executing build"))

			buildRegex := regexp.MustCompile(`executing build (\d+)`)
			matches := buildRegex.FindSubmatch(flyS.Out.Contents())
			buildID := string(matches[1])

			Eventually(flyS).Should(gbytes.Say("waiting"))

			env := exec.Command(flyBin, "-t", targetedConcourse, "hijack", "-b", buildID, "-s", "one-off", "--", "env")
			envS := helpers.StartFly(env)
			<-envS.Exited
			Expect(envS.ExitCode()).To(Equal(0))
			Expect(envS.Out).To(gbytes.Say("FOO=1"))

			hijack := exec.Command(flyBin, "-t", targetedConcourse, "hijack", "-b", buildID, "-s", "one-off", "--", "sh", "-c", "echo marco > /tmp/fifo")
			hijackS := helpers.StartFly(hijack)
			Eventually(flyS).Should(gbytes.Say("marco"))
			Eventually(hijackS).Should(gexec.Exit())
			Eventually(flyS).Should(gexec.Exit(0))
		})
	})

	Describe("pulling down outputs", func() {
		It("works", func() {
			err := ioutil.WriteFile(
				filepath.Join(fixture, "run"),
				[]byte(`#!/bin/sh
echo hello > output-1/file-1
echo world > output-2/file-2
`),
				0755,
			)
			Expect(err).NotTo(HaveOccurred())

			fly := exec.Command(flyBin, "-t", targetedConcourse, "execute", "-c", "task.yml", "-o", "output-1=./output-1", "-o", "output-2=./output-2")
			fly.Dir = fixture

			session := helpers.StartFly(fly)
			<-session.Exited

			Expect(session.ExitCode()).To(Equal(0))

			file1 := filepath.Join(fixture, "output-1", "file-1")
			file2 := filepath.Join(fixture, "output-2", "file-2")

			Expect(ioutil.ReadFile(file1)).To(Equal([]byte("hello\n")))
			Expect(ioutil.ReadFile(file2)).To(Equal([]byte("world\n")))
		})
	})

	Describe("aborting", func() {
		It("terminates the running task", func() {
			err := ioutil.WriteFile(
				filepath.Join(fixture, "run"),
				[]byte(`#!/bin/sh
trap "echo task got sigterm; exit 1" SIGTERM
sleep 1000 &
echo waiting
wait
`),
				0755,
			)
			Expect(err).NotTo(HaveOccurred())

			fly := exec.Command(flyBin, "-t", targetedConcourse, "execute", "-c", "task.yml")
			fly.Dir = fixture

			flyS := helpers.StartFly(fly)

			Eventually(flyS).Should(gbytes.Say("waiting"))

			flyS.Signal(syscall.SIGTERM)

			Eventually(flyS).Should(gbytes.Say("task got sigterm"))

			// build should have been aborted
			Eventually(flyS).Should(gexec.Exit(3))
		})
	})
})
<commit_msg>Add test to verify files being uploaded during fly execute<commit_after>package flying_test

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"syscall"

	"github.com/concourse/testflight/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Flying", func() {
	var tmpdir string
	var fixture, input1, input2 string

	BeforeEach(func() {
		var err error

		tmpdir, err = ioutil.TempDir("", "fly-test")
		Expect(err).NotTo(HaveOccurred())

		fixture = filepath.Join(tmpdir, "fixture")
		input1 = filepath.Join(tmpdir, "input-1")
		input2 = filepath.Join(tmpdir, "input-2")

		err = os.MkdirAll(fixture, 0755)
		Expect(err).NotTo(HaveOccurred())

		err = os.MkdirAll(input1, 0755)
		Expect(err).NotTo(HaveOccurred())

		err = os.MkdirAll(input2, 0755)
		Expect(err).NotTo(HaveOccurred())

		err = ioutil.WriteFile(
			filepath.Join(fixture, "run"),
			[]byte(`#!/bin/sh
echo some output
echo FOO is $FOO
echo ARGS are "$@"
exit 0
`),
			0755,
		)
		Expect(err).NotTo(HaveOccurred())

		err = ioutil.WriteFile(
			filepath.Join(tmpdir, "task.yml"),
			[]byte(`---
platform: linux

image_resource:
  type: docker-image
  source: {repository: busybox}

inputs:
- name: fixture
- name: input-1
- name: input-2

outputs:
- name: output-1
- name: output-2

params:
  FOO: 1

run:
  path: fixture/run
`),
			0644,
		)
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		os.RemoveAll(tmpdir)
	})

	It("works", func() {
		fly := exec.Command(flyBin, "-t", targetedConcourse, "execute", "-c", "task.yml", "-i", "fixture=./fixture", "-i", "input-1=./input-1", "-i", "input-2=./input-2", "--", "SOME", "ARGS")
		fly.Dir = tmpdir

		session := helpers.StartFly(fly)

		Eventually(session).Should(gexec.Exit(0))

		Expect(session).To(gbytes.Say("some output"))
		Expect(session).To(gbytes.Say("FOO is 1"))
		Expect(session).To(gbytes.Say("ARGS are SOME ARGS"))
	})

	Describe("hijacking", func() {
		It("executes an interactive command in a running task's container", func() {
			err := ioutil.WriteFile(
				filepath.Join(fixture, "run"),
				[]byte(`#!/bin/sh
mkfifo /tmp/fifo
echo waiting
cat < /tmp/fifo
`),
				0755,
			)
			Expect(err).NotTo(HaveOccurred())

			fly := exec.Command(flyBin, "-t", targetedConcourse, "execute", "-c", "task.yml", "-i", "fixture=./fixture", "-i", "input-1=./input-1", "-i", "input-2=./input-2")
			fly.Dir = tmpdir

			flyS := helpers.StartFly(fly)

			Eventually(flyS).Should(gbytes.Say("executing build"))

			buildRegex := regexp.MustCompile(`executing build (\d+)`)
			matches := buildRegex.FindSubmatch(flyS.Out.Contents())
			buildID := string(matches[1])

			Eventually(flyS).Should(gbytes.Say("waiting"))

			env := exec.Command(flyBin, "-t", targetedConcourse, "hijack", "-b", buildID, "-s", "one-off", "--", "env")
			envS := helpers.StartFly(env)
			<-envS.Exited
			Expect(envS.ExitCode()).To(Equal(0))
			Expect(envS.Out).To(gbytes.Say("FOO=1"))

			hijack := exec.Command(flyBin, "-t", targetedConcourse, "hijack", "-b", buildID, "-s", "one-off", "--", "sh", "-c", "echo marco > /tmp/fifo")
			hijackS := helpers.StartFly(hijack)
			Eventually(flyS).Should(gbytes.Say("marco"))
			Eventually(hijackS).Should(gexec.Exit())
			Eventually(flyS).Should(gexec.Exit(0))
		})
	})

	Describe("uploading inputs with and without -x", func() {
		BeforeEach(func() {
			gitIgnorePath := filepath.Join(input1, ".gitignore")

			err := ioutil.WriteFile(gitIgnorePath, []byte(`*.exist`), 0644)
			Expect(err).NotTo(HaveOccurred())

			fileToBeIgnoredPath := filepath.Join(input1, "expect-not-to.exist")
			err = ioutil.WriteFile(fileToBeIgnoredPath, []byte(`ignored file content`), 0644)
			Expect(err).NotTo(HaveOccurred())

			fileToBeIncludedPath := filepath.Join(input2, "expect-to.exist")
			err = ioutil.WriteFile(fileToBeIncludedPath, []byte(`included file content`), 0644)
			Expect(err).NotTo(HaveOccurred())

			file1 := filepath.Join(input1, "file-1")
			err = ioutil.WriteFile(file1, []byte(`file-1 contents`), 0644)
			Expect(err).NotTo(HaveOccurred())

			file2 := filepath.Join(input2, "file-2")
			err = ioutil.WriteFile(file2, []byte(`file-2 contents`), 0644)
			Expect(err).NotTo(HaveOccurred())

			err = os.Mkdir(filepath.Join(input1, ".git"), 0755)
			Expect(err).NotTo(HaveOccurred())

			err = os.Mkdir(filepath.Join(input1, ".git/refs"), 0755)
			Expect(err).NotTo(HaveOccurred())

			err = os.Mkdir(filepath.Join(input1, ".git/objects"), 0755)
			Expect(err).NotTo(HaveOccurred())

			gitHEADPath := filepath.Join(input1, ".git/HEAD")
			err = ioutil.WriteFile(gitHEADPath, []byte(`ref: refs/heads/master`), 0644)
			Expect(err).NotTo(HaveOccurred())

			err = ioutil.WriteFile(
				filepath.Join(fixture, "run"),
				[]byte(`#!/bin/sh
cp -a input-1/. output-1/
cp -a input-2/. output-2/
`),
				0755,
			)
			Expect(err).NotTo(HaveOccurred())
		})

		It("uploads git repo input and non git repo input, IGNORING things in the .gitignore for git repo inputs", func() {
			fly := exec.Command(flyBin, "-t", targetedConcourse, "execute", "-c", "task.yml", "-i", "fixture=./fixture", "-i", "input-1=./input-1", "-i", "input-2=./input-2", "-o", "output-1=./output-1", "-o", "output-2=./output-2")
			fly.Dir = tmpdir

			session := helpers.StartFly(fly)
			<-session.Exited

			Expect(session.ExitCode()).To(Equal(0))

			fileToBeIgnoredPath := filepath.Join(tmpdir, "output-1", "expect-not-to.exist")
			fileToBeIncludedPath := filepath.Join(tmpdir, "output-2", "expect-to.exist")
			file1 := filepath.Join(tmpdir, "output-1", "file-1")
			file2 := filepath.Join(tmpdir, "output-2", "file-2")

			_, err := ioutil.ReadFile(fileToBeIgnoredPath)
			Expect(err).To(HaveOccurred())

			Expect(ioutil.ReadFile(fileToBeIncludedPath)).To(Equal([]byte("included file content")))
			Expect(ioutil.ReadFile(file1)).To(Equal([]byte("file-1 contents")))
			Expect(ioutil.ReadFile(file2)).To(Equal([]byte("file-2 contents")))
		})

		It("uploads git repo input and non git repo input, INCLUDING things in the .gitignore for git repo inputs", func() {
			fly := exec.Command(flyBin, "-t", targetedConcourse, "execute", "-x", "-c", "task.yml", "-i", "fixture=./fixture", "-i", "input-1=./input-1", "-i", "input-2=./input-2", "-o", "output-1=./output-1", "-o", "output-2=./output-2")
			fly.Dir = tmpdir

			session := helpers.StartFly(fly)
			<-session.Exited

			Expect(session.ExitCode()).To(Equal(0))

			fileToBeIgnoredPath := filepath.Join(tmpdir, "output-1", "expect-not-to.exist")
			fileToBeIncludedPath := filepath.Join(tmpdir, "output-2", "expect-to.exist")
			file1 := filepath.Join(tmpdir, "output-1", "file-1")
			file2 := filepath.Join(tmpdir, "output-2", "file-2")

			Expect(ioutil.ReadFile(fileToBeIgnoredPath)).To(Equal([]byte("ignored file content")))
			Expect(ioutil.ReadFile(fileToBeIncludedPath)).To(Equal([]byte("included file content")))
			Expect(ioutil.ReadFile(file1)).To(Equal([]byte("file-1 contents")))
			Expect(ioutil.ReadFile(file2)).To(Equal([]byte("file-2 contents")))
		})
	})

	Describe("pulling down outputs", func() {
		It("works", func() {
			err := ioutil.WriteFile(
				filepath.Join(fixture, "run"),
				[]byte(`#!/bin/sh
echo hello > output-1/file-1
echo world > output-2/file-2
`),
				0755,
			)
			Expect(err).NotTo(HaveOccurred())

			fly := exec.Command(flyBin, "-t", targetedConcourse, "execute", "-c", "task.yml", "-i", "fixture=./fixture", "-i", "input-1=./input-1", "-i", "input-2=./input-2", "-o", "output-1=./output-1", "-o", "output-2=./output-2")
			fly.Dir = tmpdir

			session := helpers.StartFly(fly)
			<-session.Exited

			Expect(session.ExitCode()).To(Equal(0))

			file1 := filepath.Join(tmpdir, "output-1", "file-1")
			file2 := filepath.Join(tmpdir, "output-2", "file-2")

			Expect(ioutil.ReadFile(file1)).To(Equal([]byte("hello\n")))
			Expect(ioutil.ReadFile(file2)).To(Equal([]byte("world\n")))
		})
	})

	Describe("aborting", func() {
		It("terminates the running task", func() {
			err := ioutil.WriteFile(
				filepath.Join(fixture, "run"),
				[]byte(`#!/bin/sh
trap "echo task got sigterm; exit 1" SIGTERM
sleep 1000 &
echo waiting
wait
`),
				0755,
			)
			Expect(err).NotTo(HaveOccurred())

			fly := exec.Command(flyBin, "-t", targetedConcourse, "execute", "-c", "task.yml", "-i", "fixture=./fixture", "-i", "input-1=./input-1", "-i", "input-2=./input-2")
			fly.Dir = tmpdir

			flyS := helpers.StartFly(fly)

			Eventually(flyS).Should(gbytes.Say("waiting"))

			flyS.Signal(syscall.SIGTERM)

			Eventually(flyS).Should(gbytes.Say("task got sigterm"))

			// build should have been aborted
			Eventually(flyS).Should(gexec.Exit(3))
		})
	})
})
<|endoftext|>