<commit_before>package docker_helpers

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"sync"

	"github.com/Sirupsen/logrus"
)

type logWriter struct {
	log    func(args ...interface{})
	reader *bufio.Reader
}

func (l *logWriter) write(line string) {
	line = strings.TrimRight(line, "\n")

	if len(line) <= 0 {
		return
	}

	l.log(line)
}

func (l *logWriter) watch() {
	for {
		line, err := l.reader.ReadString('\n')
		if err == nil || err == io.EOF {
			l.write(line)
			if err == io.EOF {
				return
			}
		} else {
			logrus.WithError(err).Errorln("Problem while reading command output")
		}
	}
}

func newLogWriter(logFunction func(args ...interface{}), reader io.Reader) {
	writer := &logWriter{
		log:    logFunction,
		reader: bufio.NewReader(reader),
	}

	go writer.watch()
}

func stdoutLogWriter(cmd *exec.Cmd, fields logrus.Fields) {
	log := logrus.WithFields(fields)
	reader, _ := cmd.StdoutPipe()

	newLogWriter(log.Infoln, reader)
}

func stderrLogWriter(cmd *exec.Cmd, fields logrus.Fields) {
	log := logrus.WithFields(fields)
	reader, _ := cmd.StderrPipe()

	newLogWriter(log.Errorln, reader)
}

type machineCommand struct {
	lsCmd   *exec.Cmd
	lsLock  sync.Mutex
	lsCond  *sync.Cond
	lsData  []byte
	lsError error
}

func (m *machineCommand) ls() (data []byte, err error) {
	m.lsLock.Lock()
	defer m.lsLock.Unlock()

	if m.lsCond == nil {
		m.lsCond = sync.NewCond(&m.lsLock)
	}

	if m.lsCmd == nil {
		m.lsCmd = exec.Command("docker-machine", "ls", "-q")
		m.lsCmd.Env = os.Environ()
		go func() {
			m.lsData, m.lsError = m.lsCmd.Output()
			m.lsCmd = nil
			m.lsCond.Broadcast()
		}()
	}

	m.lsCond.Wait()

	return m.lsData, m.lsError
}

func (m *machineCommand) Create(driver, name string, opts ...string) error {
	args := []string{
		"create",
		"--driver", driver,
	}
	for _, opt := range opts {
		args = append(args, "--"+opt)
	}
	args = append(args, name)

	cmd := exec.Command("docker-machine", args...)
	cmd.Env = os.Environ()

	fields := logrus.Fields{
		"operation": "create",
		"driver":    driver,
		"name":      name,
	}
	stdoutLogWriter(cmd, fields)
	stderrLogWriter(cmd, fields)

	logrus.Debugln("Executing", cmd.Path, cmd.Args)
	return cmd.Run()
}

func (m *machineCommand) Provision(name string) error {
	cmd := exec.Command("docker-machine", "provision", name)
	cmd.Env = os.Environ()

	fields := logrus.Fields{
		"operation": "provision",
		"name":      name,
	}
	stdoutLogWriter(cmd, fields)
	stderrLogWriter(cmd, fields)

	return cmd.Run()
}

func (m *machineCommand) Remove(name string) error {
	cmd := exec.Command("docker-machine", "rm", "-y", name)
	cmd.Env = os.Environ()

	fields := logrus.Fields{
		"operation": "remove",
		"name":      name,
	}
	stdoutLogWriter(cmd, fields)
	stderrLogWriter(cmd, fields)

	return cmd.Run()
}

func (m *machineCommand) List(nodeFilter string) (machines []string, err error) {
	data, err := m.ls()
	if err != nil {
		return
	}

	reader := bufio.NewReader(bytes.NewReader(data))
	for {
		var line string

		line, err = reader.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return
		}

		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}

		var query string
		if n, _ := fmt.Sscanf(line, nodeFilter, &query); n != 1 {
			continue
		}

		machines = append(machines, line)
	}
}

func (m *machineCommand) get(args ...string) (out string, err error) {
	// Execute docker-machine to fetch IP
	cmd := exec.Command("docker-machine", args...)
	cmd.Env = os.Environ()
	data, err := cmd.Output()
	if err != nil {
		return
	}

	// Save the IP
	out = strings.TrimSpace(string(data))
	if out == "" {
		err = fmt.Errorf("failed to get %v", args)
	}
	return
}

func (m *machineCommand) IP(name string) (string, error) {
	return m.get("ip", name)
}

func (m *machineCommand) URL(name string) (string, error) {
	return m.get("url", name)
}

func (m *machineCommand) CertPath(name string) (string, error) {
	return m.get("inspect", name, "-f", "{{.HostOptions.AuthOptions.StorePath}}")
}

func (m *machineCommand) Status(name string) (string, error) {
	return m.get("status", name)
}

func (m *machineCommand) Exist(name string) bool {
	cmd := exec.Command("docker-machine", "inspect", name)
	cmd.Env = os.Environ()

	fields := logrus.Fields{
		"operation": "exists",
		"name":      name,
	}
	stderrLogWriter(cmd, fields)

	return cmd.Run() == nil
}

func (m *machineCommand) CanConnect(name string) bool {
	// Execute docker-machine config which actively ask the machine if it is up and online
	cmd := exec.Command("docker-machine", "config", name)
	cmd.Env = os.Environ()
	err := cmd.Run()
	if err == nil {
		return true
	}
	return false
}

func (m *machineCommand) Credentials(name string) (dc DockerCredentials, err error) {
	if !m.CanConnect(name) {
		err = errors.New("Can't connect")
		return
	}

	dc.TLSVerify = true
	dc.Host, err = m.URL(name)
	if err == nil {
		dc.CertPath, err = m.CertPath(name)
	}
	return
}

func NewMachineCommand() Machine {
	return &machineCommand{}
}
<commit_msg>Exit watch loop when read error occurs<commit_after>package docker_helpers

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"sync"

	"github.com/Sirupsen/logrus"
)

type logWriter struct {
	log    func(args ...interface{})
	reader *bufio.Reader
}

func (l *logWriter) write(line string) {
	line = strings.TrimRight(line, "\n")

	if len(line) <= 0 {
		return
	}

	l.log(line)
}

func (l *logWriter) watch() {
	for {
		line, err := l.reader.ReadString('\n')
		if err == nil || err == io.EOF {
			l.write(line)
			if err == io.EOF {
				return
			}
		} else {
			logrus.WithError(err).Errorln("Problem while reading command output")
			return
		}
	}
}

func newLogWriter(logFunction func(args ...interface{}), reader io.Reader) {
	writer := &logWriter{
		log:    logFunction,
		reader: bufio.NewReader(reader),
	}

	go writer.watch()
}

func stdoutLogWriter(cmd *exec.Cmd, fields logrus.Fields) {
	log := logrus.WithFields(fields)
	reader, _ := cmd.StdoutPipe()

	newLogWriter(log.Infoln, reader)
}

func stderrLogWriter(cmd *exec.Cmd, fields logrus.Fields) {
	log := logrus.WithFields(fields)
	reader, _ := cmd.StderrPipe()

	newLogWriter(log.Errorln, reader)
}

type machineCommand struct {
	lsCmd   *exec.Cmd
	lsLock  sync.Mutex
	lsCond  *sync.Cond
	lsData  []byte
	lsError error
}

func (m *machineCommand) ls() (data []byte, err error) {
	m.lsLock.Lock()
	defer m.lsLock.Unlock()

	if m.lsCond == nil {
		m.lsCond = sync.NewCond(&m.lsLock)
	}

	if m.lsCmd == nil {
		m.lsCmd = exec.Command("docker-machine", "ls", "-q")
		m.lsCmd.Env = os.Environ()
		go func() {
			m.lsData, m.lsError = m.lsCmd.Output()
			m.lsCmd = nil
			m.lsCond.Broadcast()
		}()
	}

	m.lsCond.Wait()

	return m.lsData, m.lsError
}

func (m *machineCommand) Create(driver, name string, opts ...string) error {
	args := []string{
		"create",
		"--driver", driver,
	}
	for _, opt := range opts {
		args = append(args, "--"+opt)
	}
	args = append(args, name)

	cmd := exec.Command("docker-machine", args...)
	cmd.Env = os.Environ()

	fields := logrus.Fields{
		"operation": "create",
		"driver":    driver,
		"name":      name,
	}
	stdoutLogWriter(cmd, fields)
	stderrLogWriter(cmd, fields)

	logrus.Debugln("Executing", cmd.Path, cmd.Args)
	return cmd.Run()
}

func (m *machineCommand) Provision(name string) error {
	cmd := exec.Command("docker-machine", "provision", name)
	cmd.Env = os.Environ()

	fields := logrus.Fields{
		"operation": "provision",
		"name":      name,
	}
	stdoutLogWriter(cmd, fields)
	stderrLogWriter(cmd, fields)

	return cmd.Run()
}

func (m *machineCommand) Remove(name string) error {
	cmd := exec.Command("docker-machine", "rm", "-y", name)
	cmd.Env = os.Environ()

	fields := logrus.Fields{
		"operation": "remove",
		"name":      name,
	}
	stdoutLogWriter(cmd, fields)
	stderrLogWriter(cmd, fields)

	return cmd.Run()
}

func (m *machineCommand) List(nodeFilter string) (machines []string, err error) {
	data, err := m.ls()
	if err != nil {
		return
	}

	reader := bufio.NewReader(bytes.NewReader(data))
	for {
		var line string

		line, err = reader.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return
		}

		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}

		var query string
		if n, _ := fmt.Sscanf(line, nodeFilter, &query); n != 1 {
			continue
		}

		machines = append(machines, line)
	}
}

func (m *machineCommand) get(args ...string) (out string, err error) {
	// Execute docker-machine to fetch IP
	cmd := exec.Command("docker-machine", args...)
	cmd.Env = os.Environ()
	data, err := cmd.Output()
	if err != nil {
		return
	}

	// Save the IP
	out = strings.TrimSpace(string(data))
	if out == "" {
		err = fmt.Errorf("failed to get %v", args)
	}
	return
}

func (m *machineCommand) IP(name string) (string, error) {
	return m.get("ip", name)
}

func (m *machineCommand) URL(name string) (string, error) {
	return m.get("url", name)
}

func (m *machineCommand) CertPath(name string) (string, error) {
	return m.get("inspect", name, "-f", "{{.HostOptions.AuthOptions.StorePath}}")
}

func (m *machineCommand) Status(name string) (string, error) {
	return m.get("status", name)
}

func (m *machineCommand) Exist(name string) bool {
	cmd := exec.Command("docker-machine", "inspect", name)
	cmd.Env = os.Environ()

	fields := logrus.Fields{
		"operation": "exists",
		"name":      name,
	}
	stderrLogWriter(cmd, fields)

	return cmd.Run() == nil
}

func (m *machineCommand) CanConnect(name string) bool {
	// Execute docker-machine config which actively ask the machine if it is up and online
	cmd := exec.Command("docker-machine", "config", name)
	cmd.Env = os.Environ()
	err := cmd.Run()
	if err == nil {
		return true
	}
	return false
}

func (m *machineCommand) Credentials(name string) (dc DockerCredentials, err error) {
	if !m.CanConnect(name) {
		err = errors.New("Can't connect")
		return
	}

	dc.TLSVerify = true
	dc.Host, err = m.URL(name)
	if err == nil {
		dc.CertPath, err = m.CertPath(name)
	}
	return
}

func NewMachineCommand() Machine {
	return &machineCommand{}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"time"

	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/kubernetes/pkg/api/v1"
	batchinternal "k8s.io/kubernetes/pkg/apis/batch"
	batch "k8s.io/kubernetes/pkg/apis/batch/v1"
	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
	"k8s.io/kubernetes/pkg/kubectl"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	// How long to wait for a job to finish.
	jobTimeout = 15 * time.Minute

	// Job selector name
	jobSelectorKey = "job"
)

var _ = framework.KubeDescribe("Job", func() {
	f := framework.NewDefaultFramework("job")
	parallelism := int32(2)
	completions := int32(4)
	lotsOfFailures := int32(5) // more than completions

	// Simplest case: all pods succeed promptly
	It("should run a job to completion when tasks succeed", func() {
		By("Creating a job")
		job := newTestJob("succeed", "all-succeed", v1.RestartPolicyNever, parallelism, completions)
		job, err := createJob(f.ClientSet, f.Namespace.Name, job)
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring job reaches completions")
		err = waitForJobFinish(f.ClientSet, f.Namespace.Name, job.Name, completions)
		Expect(err).NotTo(HaveOccurred())
	})

	// Pods sometimes fail, but eventually succeed.
	It("should run a job to completion when tasks sometimes fail and are locally restarted", func() {
		By("Creating a job")
		// One failure, then a success, local restarts.
		// We can't use the random failure approach used by the
		// non-local test below, because kubelet will throttle
		// frequently failing containers in a given pod, ramping
		// up to 5 minutes between restarts, making test timeouts
		// due to successive failures too likely with a reasonable
		// test timeout.
		job := newTestJob("failOnce", "fail-once-local", v1.RestartPolicyOnFailure, parallelism, completions)
		job, err := createJob(f.ClientSet, f.Namespace.Name, job)
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring job reaches completions")
		err = waitForJobFinish(f.ClientSet, f.Namespace.Name, job.Name, completions)
		Expect(err).NotTo(HaveOccurred())
	})

	// Pods sometimes fail, but eventually succeed, after pod restarts
	It("should run a job to completion when tasks sometimes fail and are not locally restarted", func() {
		By("Creating a job")
		// 50% chance of container success, local restarts.
		// Can't use the failOnce approach because that relies
		// on an emptyDir, which is not preserved across new pods.
		// Worst case analysis: 15 failures, each taking 1 minute to
		// run due to some slowness, 1 in 2^15 chance of happening,
		// causing test flake.  Should be very rare.
		job := newTestJob("randomlySucceedOrFail", "rand-non-local", v1.RestartPolicyNever, parallelism, completions)
		job, err := createJob(f.ClientSet, f.Namespace.Name, job)
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring job reaches completions")
		err = waitForJobFinish(f.ClientSet, f.Namespace.Name, job.Name, completions)
		Expect(err).NotTo(HaveOccurred())
	})

	It("should keep restarting failed pods", func() {
		By("Creating a job")
		job := newTestJob("fail", "all-fail", v1.RestartPolicyNever, parallelism, completions)
		job, err := createJob(f.ClientSet, f.Namespace.Name, job)
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring job shows many failures")
		err = wait.Poll(framework.Poll, jobTimeout, func() (bool, error) {
			curr, err := getJob(f.ClientSet, f.Namespace.Name, job.Name)
			if err != nil {
				return false, err
			}
			return curr.Status.Failed > lotsOfFailures, nil
		})
	})

	It("should scale a job up", func() {
		startParallelism := int32(1)
		endParallelism := int32(2)
		By("Creating a job")
		job := newTestJob("notTerminate", "scale-up", v1.RestartPolicyNever, startParallelism, completions)
		job, err := createJob(f.ClientSet, f.Namespace.Name, job)
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring active pods == startParallelism")
		err = waitForAllPodsRunning(f.ClientSet, f.Namespace.Name, job.Name, startParallelism)
		Expect(err).NotTo(HaveOccurred())

		By("scale job up")
		scaler, err := kubectl.ScalerFor(batchinternal.Kind("Job"), f.InternalClientset)
		Expect(err).NotTo(HaveOccurred())
		waitForScale := kubectl.NewRetryParams(5*time.Second, 1*time.Minute)
		waitForReplicas := kubectl.NewRetryParams(5*time.Second, 5*time.Minute)
		scaler.Scale(f.Namespace.Name, job.Name, uint(endParallelism), nil, waitForScale, waitForReplicas)
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring active pods == endParallelism")
		err = waitForAllPodsRunning(f.ClientSet, f.Namespace.Name, job.Name, endParallelism)
		Expect(err).NotTo(HaveOccurred())
	})

	It("should scale a job down", func() {
		startParallelism := int32(2)
		endParallelism := int32(1)
		By("Creating a job")
		job := newTestJob("notTerminate", "scale-down", v1.RestartPolicyNever, startParallelism, completions)
		job, err := createJob(f.ClientSet, f.Namespace.Name, job)
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring active pods == startParallelism")
		err = waitForAllPodsRunning(f.ClientSet, f.Namespace.Name, job.Name, startParallelism)
		Expect(err).NotTo(HaveOccurred())

		By("scale job down")
		scaler, err := kubectl.ScalerFor(batchinternal.Kind("Job"), f.InternalClientset)
		Expect(err).NotTo(HaveOccurred())
		waitForScale := kubectl.NewRetryParams(5*time.Second, 1*time.Minute)
		waitForReplicas := kubectl.NewRetryParams(5*time.Second, 5*time.Minute)
		err = scaler.Scale(f.Namespace.Name, job.Name, uint(endParallelism), nil, waitForScale, waitForReplicas)
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring active pods == endParallelism")
		err = waitForAllPodsRunning(f.ClientSet, f.Namespace.Name, job.Name, endParallelism)
		Expect(err).NotTo(HaveOccurred())
	})

	It("should delete a job", func() {
		By("Creating a job")
		job := newTestJob("notTerminate", "foo", v1.RestartPolicyNever, parallelism, completions)
		job, err := createJob(f.ClientSet, f.Namespace.Name, job)
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring active pods == parallelism")
		err = waitForAllPodsRunning(f.ClientSet, f.Namespace.Name, job.Name, parallelism)
		Expect(err).NotTo(HaveOccurred())

		By("delete a job")
		reaper, err := kubectl.ReaperFor(batchinternal.Kind("Job"), f.InternalClientset)
		Expect(err).NotTo(HaveOccurred())
		timeout := 1 * time.Minute
		err = reaper.Stop(f.Namespace.Name, job.Name, timeout, metav1.NewDeleteOptions(0))
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring job was deleted")
		_, err = getJob(f.ClientSet, f.Namespace.Name, job.Name)
		Expect(err).To(HaveOccurred())
		Expect(errors.IsNotFound(err)).To(BeTrue())
	})

	It("should fail a job", func() {
		By("Creating a job")
		job := newTestJob("notTerminate", "foo", v1.RestartPolicyNever, parallelism, completions)
		activeDeadlineSeconds := int64(10)
		job.Spec.ActiveDeadlineSeconds = &activeDeadlineSeconds
		job, err := createJob(f.ClientSet, f.Namespace.Name, job)
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring job was failed")
		err = waitForJobFail(f.ClientSet, f.Namespace.Name, job.Name, 20*time.Second)
		if err == wait.ErrWaitTimeout {
			job, err = getJob(f.ClientSet, f.Namespace.Name, job.Name)
			Expect(err).NotTo(HaveOccurred())
			// the job stabilized and won't be synced until modification or full
			// resync happens, we don't want to wait for the latter so we force
			// sync modifying it
			_, err = framework.UpdateJobWithRetries(f.ClientSet, f.Namespace.Name, job.Name, func(update *batch.Job) {
				update.Spec.Parallelism = &completions
			})
			Expect(err).NotTo(HaveOccurred())
			err = waitForJobFail(f.ClientSet, f.Namespace.Name, job.Name, jobTimeout)
		}
		Expect(err).NotTo(HaveOccurred())
	})
})

// newTestJob returns a job which does one of several testing behaviors.
func newTestJob(behavior, name string, rPol v1.RestartPolicy, parallelism, completions int32) *batch.Job {
	job := &batch.Job{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
		},
		Spec: batch.JobSpec{
			Parallelism:    &parallelism,
			Completions:    &completions,
			ManualSelector: newBool(false),
			Template: v1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{jobSelectorKey: name},
				},
				Spec: v1.PodSpec{
					RestartPolicy: rPol,
					Volumes: []v1.Volume{
						{
							Name: "data",
							VolumeSource: v1.VolumeSource{
								EmptyDir: &v1.EmptyDirVolumeSource{},
							},
						},
					},
					Containers: []v1.Container{
						{
							Name:    "c",
							Image:   "gcr.io/google_containers/busybox:1.24",
							Command: []string{},
							VolumeMounts: []v1.VolumeMount{
								{
									MountPath: "/data",
									Name:      "data",
								},
							},
						},
					},
				},
			},
		},
	}
	switch behavior {
	case "notTerminate":
		job.Spec.Template.Spec.Containers[0].Command = []string{"sleep", "1000000"}
	case "fail":
		job.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh", "-c", "exit 1"}
	case "succeed":
		job.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh", "-c", "exit 0"}
	case "randomlySucceedOrFail":
		// Bash's $RANDOM generates pseudorandom int in range 0 - 32767.
		// Dividing by 16384 gives roughly 50/50 chance of success.
		job.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh", "-c", "exit $(( $RANDOM / 16384 ))"}
	case "failOnce":
		// Fail the first the container of the pod is run, and
		// succeed the second time. Checks for file on emptydir.
		// If present, succeed.  If not, create but fail.
		// Note that this cannot be used with RestartNever because
		// it always fails the first time for a pod.
		job.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh", "-c", "if [[ -r /data/foo ]] ; then exit 0 ; else touch /data/foo ; exit 1 ; fi"}
	}
	return job
}

func getJob(c clientset.Interface, ns, name string) (*batch.Job, error) {
	return c.Batch().Jobs(ns).Get(name, metav1.GetOptions{})
}

func createJob(c clientset.Interface, ns string, job *batch.Job) (*batch.Job, error) {
	return c.Batch().Jobs(ns).Create(job)
}

func updateJob(c clientset.Interface, ns string, job *batch.Job) (*batch.Job, error) {
	return c.Batch().Jobs(ns).Update(job)
}

func deleteJob(c clientset.Interface, ns, name string) error {
	return c.Batch().Jobs(ns).Delete(name, nil)
}

// Wait for all pods to become Running.  Only use when pods will run for a long time, or it will be racy.
func waitForAllPodsRunning(c clientset.Interface, ns, jobName string, parallelism int32) error {
	label := labels.SelectorFromSet(labels.Set(map[string]string{jobSelectorKey: jobName}))
	return wait.Poll(framework.Poll, jobTimeout, func() (bool, error) {
		options := metav1.ListOptions{LabelSelector: label.String()}
		pods, err := c.Core().Pods(ns).List(options)
		if err != nil {
			return false, err
		}
		count := int32(0)
		for _, p := range pods.Items {
			if p.Status.Phase == v1.PodRunning {
				count++
			}
		}
		return count == parallelism, nil
	})
}

// Wait for job to reach completions.
func waitForJobFinish(c clientset.Interface, ns, jobName string, completions int32) error {
	return wait.Poll(framework.Poll, jobTimeout, func() (bool, error) {
		curr, err := c.Batch().Jobs(ns).Get(jobName, metav1.GetOptions{})
		if err != nil {
			return false, err
		}
		return curr.Status.Succeeded == completions, nil
	})
}

// Wait for job fail.
func waitForJobFail(c clientset.Interface, ns, jobName string, timeout time.Duration) error {
	return wait.Poll(framework.Poll, timeout, func() (bool, error) {
		curr, err := c.Batch().Jobs(ns).Get(jobName, metav1.GetOptions{})
		if err != nil {
			return false, err
		}
		for _, c := range curr.Status.Conditions {
			if c.Type == batch.JobFailed && c.Status == v1.ConditionTrue {
				return true, nil
			}
		}
		return false, nil
	})
}

func newBool(val bool) *bool {
	p := new(bool)
	*p = val
	return p
}
<commit_msg>Remove half of the job e2e tests.<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"time"

	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/kubernetes/pkg/api/v1"
	batchinternal "k8s.io/kubernetes/pkg/apis/batch"
	batch "k8s.io/kubernetes/pkg/apis/batch/v1"
	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
	"k8s.io/kubernetes/pkg/kubectl"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	// How long to wait for a job to finish.
	jobTimeout = 15 * time.Minute

	// Job selector name
	jobSelectorKey = "job"
)

var _ = framework.KubeDescribe("Job", func() {
	f := framework.NewDefaultFramework("job")
	parallelism := int32(2)
	completions := int32(4)

	// Simplest case: all pods succeed promptly
	It("should run a job to completion when tasks succeed", func() {
		By("Creating a job")
		job := newTestJob("succeed", "all-succeed", v1.RestartPolicyNever, parallelism, completions)
		job, err := createJob(f.ClientSet, f.Namespace.Name, job)
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring job reaches completions")
		err = waitForJobFinish(f.ClientSet, f.Namespace.Name, job.Name, completions)
		Expect(err).NotTo(HaveOccurred())
	})

	// Pods sometimes fail, but eventually succeed.
	It("should run a job to completion when tasks sometimes fail and are locally restarted", func() {
		By("Creating a job")
		// One failure, then a success, local restarts.
		// We can't use the random failure approach used by the
		// non-local test below, because kubelet will throttle
		// frequently failing containers in a given pod, ramping
		// up to 5 minutes between restarts, making test timeouts
		// due to successive failures too likely with a reasonable
		// test timeout.
		job := newTestJob("failOnce", "fail-once-local", v1.RestartPolicyOnFailure, parallelism, completions)
		job, err := createJob(f.ClientSet, f.Namespace.Name, job)
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring job reaches completions")
		err = waitForJobFinish(f.ClientSet, f.Namespace.Name, job.Name, completions)
		Expect(err).NotTo(HaveOccurred())
	})

	// Pods sometimes fail, but eventually succeed, after pod restarts
	It("should run a job to completion when tasks sometimes fail and are not locally restarted", func() {
		By("Creating a job")
		// 50% chance of container success, local restarts.
		// Can't use the failOnce approach because that relies
		// on an emptyDir, which is not preserved across new pods.
		// Worst case analysis: 15 failures, each taking 1 minute to
		// run due to some slowness, 1 in 2^15 chance of happening,
		// causing test flake.  Should be very rare.
		job := newTestJob("randomlySucceedOrFail", "rand-non-local", v1.RestartPolicyNever, parallelism, completions)
		job, err := createJob(f.ClientSet, f.Namespace.Name, job)
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring job reaches completions")
		err = waitForJobFinish(f.ClientSet, f.Namespace.Name, job.Name, completions)
		Expect(err).NotTo(HaveOccurred())
	})

	It("should delete a job", func() {
		By("Creating a job")
		job := newTestJob("notTerminate", "foo", v1.RestartPolicyNever, parallelism, completions)
		job, err := createJob(f.ClientSet, f.Namespace.Name, job)
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring active pods == parallelism")
		err = waitForAllPodsRunning(f.ClientSet, f.Namespace.Name, job.Name, parallelism)
		Expect(err).NotTo(HaveOccurred())

		By("delete a job")
		reaper, err := kubectl.ReaperFor(batchinternal.Kind("Job"), f.InternalClientset)
		Expect(err).NotTo(HaveOccurred())
		timeout := 1 * time.Minute
		err = reaper.Stop(f.Namespace.Name, job.Name, timeout, metav1.NewDeleteOptions(0))
		Expect(err).NotTo(HaveOccurred())

		By("Ensuring job was deleted")
		_, err = getJob(f.ClientSet, f.Namespace.Name, job.Name)
		Expect(err).To(HaveOccurred())
		Expect(errors.IsNotFound(err)).To(BeTrue())
	})
})

// newTestJob returns a job which does one of several testing behaviors.
func newTestJob(behavior, name string, rPol v1.RestartPolicy, parallelism, completions int32) *batch.Job {
	job := &batch.Job{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
		},
		Spec: batch.JobSpec{
			Parallelism:    &parallelism,
			Completions:    &completions,
			ManualSelector: newBool(false),
			Template: v1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{jobSelectorKey: name},
				},
				Spec: v1.PodSpec{
					RestartPolicy: rPol,
					Volumes: []v1.Volume{
						{
							Name: "data",
							VolumeSource: v1.VolumeSource{
								EmptyDir: &v1.EmptyDirVolumeSource{},
							},
						},
					},
					Containers: []v1.Container{
						{
							Name:    "c",
							Image:   "gcr.io/google_containers/busybox:1.24",
							Command: []string{},
							VolumeMounts: []v1.VolumeMount{
								{
									MountPath: "/data",
									Name:      "data",
								},
							},
						},
					},
				},
			},
		},
	}
	switch behavior {
	case "notTerminate":
		job.Spec.Template.Spec.Containers[0].Command = []string{"sleep", "1000000"}
	case "fail":
		job.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh", "-c", "exit 1"}
	case "succeed":
		job.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh", "-c", "exit 0"}
	case "randomlySucceedOrFail":
		// Bash's $RANDOM generates pseudorandom int in range 0 - 32767.
		// Dividing by 16384 gives roughly 50/50 chance of success.
		job.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh", "-c", "exit $(( $RANDOM / 16384 ))"}
	case "failOnce":
		// Fail the first the container of the pod is run, and
		// succeed the second time. Checks for file on emptydir.
		// If present, succeed.  If not, create but fail.
		// Note that this cannot be used with RestartNever because
		// it always fails the first time for a pod.
		job.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh", "-c", "if [[ -r /data/foo ]] ; then exit 0 ; else touch /data/foo ; exit 1 ; fi"}
	}
	return job
}

func getJob(c clientset.Interface, ns, name string) (*batch.Job, error) {
	return c.Batch().Jobs(ns).Get(name, metav1.GetOptions{})
}

func createJob(c clientset.Interface, ns string, job *batch.Job) (*batch.Job, error) {
	return c.Batch().Jobs(ns).Create(job)
}

func updateJob(c clientset.Interface, ns string, job *batch.Job) (*batch.Job, error) {
	return c.Batch().Jobs(ns).Update(job)
}

func deleteJob(c clientset.Interface, ns, name string) error {
	return c.Batch().Jobs(ns).Delete(name, nil)
}

// Wait for all pods to become Running.  Only use when pods will run for a long time, or it will be racy.
func waitForAllPodsRunning(c clientset.Interface, ns, jobName string, parallelism int32) error {
	label := labels.SelectorFromSet(labels.Set(map[string]string{jobSelectorKey: jobName}))
	return wait.Poll(framework.Poll, jobTimeout, func() (bool, error) {
		options := metav1.ListOptions{LabelSelector: label.String()}
		pods, err := c.Core().Pods(ns).List(options)
		if err != nil {
			return false, err
		}
		count := int32(0)
		for _, p := range pods.Items {
			if p.Status.Phase == v1.PodRunning {
				count++
			}
		}
		return count == parallelism, nil
	})
}

// Wait for job to reach completions.
func waitForJobFinish(c clientset.Interface, ns, jobName string, completions int32) error {
	return wait.Poll(framework.Poll, jobTimeout, func() (bool, error) {
		curr, err := c.Batch().Jobs(ns).Get(jobName, metav1.GetOptions{})
		if err != nil {
			return false, err
		}
		return curr.Status.Succeeded == completions, nil
	})
}

// Wait for job fail.
func waitForJobFail(c clientset.Interface, ns, jobName string, timeout time.Duration) error {
	return wait.Poll(framework.Poll, timeout, func() (bool, error) {
		curr, err := c.Batch().Jobs(ns).Get(jobName, metav1.GetOptions{})
		if err != nil {
			return false, err
		}
		for _, c := range curr.Status.Conditions {
			if c.Type == batch.JobFailed && c.Status == v1.ConditionTrue {
				return true, nil
			}
		}
		return false, nil
	})
}

func newBool(val bool) *bool {
	p := new(bool)
	*p = val
	return p
}
<|endoftext|>


<commit_before>package inproc

import (
	"errors"
	"fmt"
	"io"
	"math"
	"net"
	"strconv"
	"sync"
	"time"

	"github.com/anacrolix/missinggo"
)

var (
	mu       sync.Mutex
	cond         = sync.Cond{L: &mu}
	nextPort int = 1
	conns        = map[int]*packetConn{}
)

type addr struct {
	Port int
}

func (addr) Network() string {
	return "inproc"
}

func (me addr) String() string {
	return fmt.Sprintf("127.0.0.1:%d", me.Port)
}

func getPort() (port int) {
	mu.Lock()
	defer mu.Unlock()
	port = nextPort
	nextPort++
	return
}

func resolve(str string) (addr addr) {
	if str != "" {
		h, p, err := net.SplitHostPort(str)
		if err != nil {
			panic(err)
		}
		if h != "localhost" && h != "" {
			panic(h)
		}
		i64, err := strconv.ParseInt(p, 10, 0)
		if err != nil {
			panic(err)
		}
		addr.Port = int(i64)
	}
	if addr.Port == 0 {
		addr.Port = getPort()
	}
	return
}

func ListenPacket(network, address string) (nc net.PacketConn, err error) {
	addr := resolve(address)
	mu.Lock()
	defer mu.Unlock()
	if _, ok := conns[addr.Port]; ok {
		err = errors.New("address in use")
		return
	}
	pc := &packetConn{
		addr:          addr,
		readDeadline:  newCondDeadline(&cond),
		writeDeadline: newCondDeadline(&cond),
	}
	conns[addr.Port] = pc
	nc = pc
	return
}

type packet struct {
	data []byte
	addr addr
}

type packetConn struct {
	closed        bool
	addr          addr
	reads         []packet
	readDeadline  *condDeadline
	writeDeadline *condDeadline
}

func (me *packetConn) Close() error {
	mu.Lock()
	defer mu.Unlock()
	me.closed = true
	delete(conns, me.addr.Port)
	cond.Broadcast()
	return nil
}

func (me *packetConn) LocalAddr() net.Addr {
	return me.addr
}

var errTimeout = errors.New("i/o timeout")

func (me *packetConn) WriteTo(b []byte, na net.Addr) (n int, err error) {
	n = len(b)
	mu.Lock()
	defer mu.Unlock()
	if me.writeDeadline.exceeded() {
		err = errTimeout
		return
	}
	port := missinggo.AddrPort(na)
	c, ok := conns[port]
	if !ok {
		// log.Printf("no conn for port %d", port)
		return
	}
	c.reads = append(c.reads, packet{append([]byte(nil), b...), me.addr})
	cond.Broadcast()
	return
}

func (me *packetConn) ReadFrom(b []byte) (n int, addr net.Addr, err error) {
	mu.Lock()
	defer mu.Unlock()
	for {
		if len(me.reads) != 0 {
			r := me.reads[0]
			me.reads = me.reads[1:]
			n = copy(b, r.data)
			addr = r.addr
			// log.Println(addr)
			return
		}
		if me.closed {
			err = io.EOF
			return
		}
		if me.readDeadline.exceeded() {
			err = errTimeout
			return
		}
		cond.Wait()
	}
}

func (me *packetConn) SetDeadline(t time.Time) error {
	me.writeDeadline.setDeadline(t)
	me.readDeadline.setDeadline(t)
	return nil
}

func (me *packetConn) SetReadDeadline(t time.Time) error {
	me.readDeadline.setDeadline(t)
	return nil
}

func (me *packetConn) SetWriteDeadline(t time.Time) error {
	me.writeDeadline.setDeadline(t)
	return nil
}

func newCondDeadline(cond *sync.Cond) (ret *condDeadline) {
	ret = &condDeadline{
		timer: time.AfterFunc(math.MaxInt64, func() {
			mu.Lock()
			ret._exceeded = true
			mu.Unlock()
			cond.Broadcast()
		}),
	}
	ret.setDeadline(time.Time{})
	return
}

type condDeadline struct {
	mu        sync.Mutex
	_exceeded bool
	timer     *time.Timer
}

func (me *condDeadline) setDeadline(t time.Time) {
	me.mu.Lock()
	defer me.mu.Unlock()
	me._exceeded = false
	if t.IsZero() {
		me.timer.Stop()
		return
	}
	me.timer.Reset(t.Sub(time.Now()))
}

func (me *condDeadline) exceeded() bool {
	me.mu.Lock()
	defer me.mu.Unlock()
	return me._exceeded
}
<commit_msg>Expose resolving functions<commit_after>package inproc

import (
	"errors"
	"fmt"
	"io"
	"math"
	"net"
	"strconv"
	"sync"
	"time"

	"github.com/anacrolix/missinggo"
)

var (
	mu       sync.Mutex
	cond         = sync.Cond{L: &mu}
	nextPort int = 1
	conns        = map[int]*packetConn{}
)

type addr struct {
	Port int
}

func (addr) Network() string {
	return "inproc"
}

func (me addr) String() string {
	return fmt.Sprintf("127.0.0.1:%d", me.Port)
}

func getPort() (port int) {
	mu.Lock()
	defer mu.Unlock()
	port = nextPort
	nextPort++
	return
}

func ResolveAddr(network, str string) (net.Addr, error) {
	return ResolveInprocAddr(network, str)
}

func ResolveInprocAddr(network, str string) (addr addr, err error) {
	if str == "" {
		addr.Port = getPort()
		return
	}
	_, p, err := net.SplitHostPort(str)
	if err != nil {
		return
	}
	i64, err := strconv.ParseInt(p, 10, 0)
	if err != nil {
		return
	}
	addr.Port = int(i64)
	if addr.Port == 0 {
		addr.Port = getPort()
	}
	return
}

func ListenPacket(network, addrStr string) (nc net.PacketConn, err error) {
	addr, err := ResolveInprocAddr(network, addrStr)
	if err != nil {
		return
	}
	mu.Lock()
	defer mu.Unlock()
	if _, ok := conns[addr.Port]; ok {
		err = errors.New("address in use")
		return
	}
	pc := &packetConn{
		addr:          addr,
		readDeadline:  newCondDeadline(&cond),
		writeDeadline: newCondDeadline(&cond),
	}
	conns[addr.Port] = pc
	nc = pc
	return
}

type packet struct {
	data []byte
	addr addr
}

type packetConn struct {
	closed        bool
	addr          addr
	reads         []packet
	readDeadline  *condDeadline
	writeDeadline *condDeadline
}

func (me *packetConn) Close() error {
	mu.Lock()
	defer mu.Unlock()
	me.closed = true
	delete(conns, me.addr.Port)
	cond.Broadcast()
	return nil
}

func (me *packetConn) LocalAddr() net.Addr {
	return me.addr
}

var errTimeout = errors.New("i/o timeout")

func (me *packetConn) WriteTo(b []byte, na net.Addr) (n int, err error) {
	n = len(b)
	mu.Lock()
	defer mu.Unlock()
	if me.writeDeadline.exceeded() {
		err = errTimeout
		return
	}
	port := missinggo.AddrPort(na)
	c, ok := conns[port]
	if !ok {
		// log.Printf("no conn for port %d", port)
		return
	}
	c.reads = append(c.reads, packet{append([]byte(nil), b...), me.addr})
	cond.Broadcast()
	return
}

func (me *packetConn) ReadFrom(b []byte) (n int, addr net.Addr, err error) {
	mu.Lock()
	defer mu.Unlock()
	for {
		if len(me.reads) != 0 {
			r := me.reads[0]
			me.reads = me.reads[1:]
			n = copy(b, r.data)
			addr = r.addr
			// log.Println(addr)
			return
		}
		if me.closed {
			err = io.EOF
			return
		}
		if me.readDeadline.exceeded() {
			err = errTimeout
			return
		}
		cond.Wait()
	}
}

func (me *packetConn) SetDeadline(t time.Time) error {
	me.writeDeadline.setDeadline(t)
	me.readDeadline.setDeadline(t)
	return nil
}

func (me *packetConn) SetReadDeadline(t time.Time) error {
	me.readDeadline.setDeadline(t)
	return nil
}

func (me *packetConn) SetWriteDeadline(t time.Time) error {
	me.writeDeadline.setDeadline(t)
	return nil
}

func newCondDeadline(cond *sync.Cond) (ret *condDeadline) {
	ret = &condDeadline{
		timer: time.AfterFunc(math.MaxInt64, func() {
			mu.Lock()
			ret._exceeded = true
			mu.Unlock()
			cond.Broadcast()
		}),
	}
	ret.setDeadline(time.Time{})
	return
}

type condDeadline struct {
	mu        sync.Mutex
	_exceeded bool
	timer     *time.Timer
}

func (me *condDeadline) setDeadline(t time.Time) {
	me.mu.Lock()
	defer me.mu.Unlock()
	me._exceeded = false
	if t.IsZero() {
		me.timer.Stop()
		return
	}
	me.timer.Reset(t.Sub(time.Now()))
}

func (me *condDeadline) exceeded() bool {
	me.mu.Lock()
	defer me.mu.Unlock()
	return me._exceeded
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"
	"github.com/0xor1/gorillaseed/src/server/lib/mux"
	"github.com/0xor1/gorillaseed/src/server/src/api"
)

const (
	domain     = "gorillaseed.net"
	listenPort = "8080"
)

func main() {
	log.Println("Server Starting...")

	r := mux.NewRouter()
	fs := http.FileServer(http.Dir("../client"))

	r.Host(domain).Methods("GET").PathPrefix("/").Handler(fs)
	r.Host("www." + domain).Methods("GET").PathPrefix("/").Handler(fs)

	s1 := r.Host(domain).Methods("POST").Subrouter()
	s2 := r.Host("www." + domain).Methods("POST").Subrouter()

	api.Route(s1)
	api.Route(s2)

	http.Handle("/", r)
	log.Println("Server Listening on Port: " + listenPort)
	http.ListenAndServe(":" + listenPort, nil)
}
<commit_msg>only support one host and redirect all others too that host<commit_after>package main

import (
	"log"
	"net/http"
	"github.com/0xor1/gorillaseed/src/server/lib/mux"
	"github.com/0xor1/gorillaseed/src/server/src/api"
)

const (
	domain     = "gorillaseed.net"
	listenPort = "8080"
)

func main() {
	log.Println("Server Starting...")

	r := mux.NewRouter()
	fs := http.FileServer(http.Dir("../client"))

	r.Host("{sub:.*}.{dom:.*}.{tld:.*}").PathPrefix("/").HandlerFunc(redirect)
	ds := r.Host(domain).Subrouter()
	ds.Methods("GET").PathPrefix("/").Handler(fs)
	apis := ds.Methods("POST").Subrouter()
	api.Route(apis)

	http.Handle("/", r)
	log.Println("Server Listening on Port: " + listenPort)
	http.ListenAndServe(":" + listenPort, nil)
}

func redirect(w http.ResponseWriter, r *http.Request){
	http.Redirect(w, r, "http://" + domain, http.StatusMovedPermanently)
}
<|endoftext|>


<commit_before>package server

import (
	"../network"
	"fmt"
	logPkg "log"
	"os"
)

var log *logPkg.Logger

func init() {
	log = logPkg.New(os.Stdout, "server:", logPkg.LstdFlags)
}

type Server struct {
	setting Setting
	conn    *network.BitXConn
	Database
}

type Setting struct {
	Hello            network.ServerHello
	DatabaseLocation string
	DatabaseType     string
	Port             int
	IP               string
}

func NewServer(s Setting) *Server {
	conn, err := network.ListenUDP(s.IP, s.Port)
	if err != nil {
		panic(err)
	}
	if !conn.StartServerLoop() {
		panic(fmt.Errorf("can't start server:%v", s))
	}
	database := Database(nil)
	switch s.DatabaseType {
	case "simple":
		database = OpenSimpleDatabase(s.DatabaseLocation, 0)
	default:
		panic(fmt.Errorf("unknown database type:%v", s.DatabaseType))
	}

	server := &Server{s, conn, database}

	go server.consume(conn.Receive)

	return server
}

func (s *Server) consume(ps <-chan network.BitXPacket) {
	for p := range ps {
		s.process(p)
	}
}

func (s *Server) process(bp network.BitXPacket) {
	addr := bp.Addr
	rece := bp.Packet
	if rece.Hello != nil {
		log.Printf("got hello:%v from:%v", rece.Hello, addr)
	}
	if rece.GetHelloRequest() {
		log.Printf("req hello from:%v", addr)
		send := &network.Packet{}
		send.Hello = &s.setting.Hello
		s.conn.Send(send, addr)
	}

	if rece.Files != nil {
		for _, f := range rece.Files {
			id := f.Id
			log.Printf("about:%v", id)
		}
	}

}
<commit_msg>log each file message type<commit_after>package server

import (
	"../network"
	"fmt"
	logPkg "log"
	"os"
)

var log *logPkg.Logger

func init() {
	log = logPkg.New(os.Stdout, "server:", logPkg.LstdFlags)
}

type Server struct {
	setting Setting
	conn    *network.BitXConn
	Database
}

type Setting struct {
	Hello            network.ServerHello
	DatabaseLocation string
	DatabaseType     string
	Port             int
	IP               string
}

func NewServer(s Setting) *Server {
	conn, err := network.ListenUDP(s.IP, s.Port)
	if err != nil {
		panic(err)
	}
	if !conn.StartServerLoop() {
		panic(fmt.Errorf("can't start server:%v", s))
	}
	database := Database(nil)
	switch s.DatabaseType {
	case "simple":
		database = OpenSimpleDatabase(s.DatabaseLocation, 0)
	default:
		panic(fmt.Errorf("unknown database type:%v", s.DatabaseType))
	}

	server := &Server{s, conn, database}

	go server.consume(conn.Receive)

	return server
}

func (s *Server) consume(ps <-chan network.BitXPacket) {
	for p := range ps {
		s.process(p)
	}
}

func (s *Server) process(bp network.BitXPacket) {
	addr := bp.Addr
	rece := bp.Packet
	if rece.Hello != nil {
		log.Printf("got hello:%v from:%v", rece.Hello, addr)
	}
	if rece.GetHelloRequest() {
		log.Printf("req hello from:%v", addr)
		send := &network.Packet{}
		send.Hello = &s.setting.Hello
		s.conn.Send(send, addr)
	}

	if rece.Files != nil {
		for _, f := range rece.Files {
			id := f.Id
			log.Printf("about:%v", id)
			for _, ha := range f.HashAsk {
				log.Printf("hash ask:%v", ha)
			}
			for _, hs := range f.HashSend {
				log.Printf("hash send:%v", hs)
			}
			for _, da := range f.DataAsk {
				log.Printf("data ask:%v", da)
			}
			for _, ds := range f.DataSend {
				log.Printf("data send:%v", ds)
			}
		}
	}

}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package podtask

import (
	"fmt"
	"strings"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/pborman/uuid"
	"k8s.io/kubernetes/contrib/mesos/pkg/offers"
	annotation "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta"
	"k8s.io/kubernetes/contrib/mesos/pkg/scheduler/metrics"
	mresource "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/resource"
	"k8s.io/kubernetes/pkg/api"

	log "github.com/golang/glog"
	mesos "github.com/mesos/mesos-go/mesosproto"
	mutil "github.com/mesos/mesos-go/mesosutil"
)

type StateType int

const (
	StatePending StateType = iota
	StateRunning
	StateFinished
	StateUnknown
)

type FlagType string

const (
	Launched = FlagType("launched")
	Bound    = FlagType("bound")
	Deleted  = FlagType("deleted")
)

// A struct that describes a pod task.
type T struct {
	ID          string
	Pod         api.Pod
	Spec        Spec
	Offer       offers.Perishable // thread-safe
	State       StateType
	Flags       map[FlagType]struct{}
	CreateTime  time.Time
	UpdatedTime time.Time // time of the most recent StatusUpdate we've seen from the mesos master

	podStatus  api.PodStatus
	executor   *mesos.ExecutorInfo // readonly
	podKey     string
	launchTime time.Time
	bindTime   time.Time
	mapper     HostPortMappingType
}

type Spec struct {
	SlaveID       string
	AssignedSlave string
	CPU           mresource.CPUShares
	Memory        mresource.MegaBytes
	PortMap       []HostPortMapping
	Ports         []uint64
	Data          []byte
}

// mostly-clone this pod task. the clone will actually share the some fields:
//   - executor    // OK because it's read only
//   - Offer       // OK because it's guarantees safe concurrent access
func (t *T) Clone() *T {
	if t == nil {
		return nil
	}

	// shallow-copy
	clone := *t

	// deep copy
	(&t.Spec).copyTo(&clone.Spec)
	clone.Flags = map[FlagType]struct{}{}
	for k := range t.Flags {
		clone.Flags[k] = struct{}{}
	}
	return &clone
}

func (old *Spec) copyTo(new *Spec) {
	if len(old.PortMap) > 0 {
		new.PortMap = append(([]HostPortMapping)(nil), old.PortMap...)
	}
	if len(old.Ports) > 0 {
		new.Ports = append(([]uint64)(nil), old.Ports...)
	}
	if len(old.Data) > 0 {
		new.Data = append(([]byte)(nil), old.Data...)
	}
}

func (t *T) HasAcceptedOffer() bool {
	return t.Spec.SlaveID != ""
}

func (t *T) GetOfferId() string {
	if t.Offer == nil {
		return ""
	}
	return t.Offer.Details().Id.GetValue()
}

func generateTaskName(pod *api.Pod) string {
	ns := pod.Namespace
	if ns == "" {
		ns = api.NamespaceDefault
	}
	return fmt.Sprintf("%s.%s.pods", pod.Name, ns)
}

func (t *T) BuildTaskInfo() *mesos.TaskInfo {
	info := &mesos.TaskInfo{
		Name:     proto.String(generateTaskName(&t.Pod)),
		TaskId:   mutil.NewTaskID(t.ID),
		SlaveId:  mutil.NewSlaveID(t.Spec.SlaveID),
		Executor: t.executor,
		Data:     t.Spec.Data,
		Resources: []*mesos.Resource{
			mutil.NewScalarResource("cpus", float64(t.Spec.CPU)),
			mutil.NewScalarResource("mem", float64(t.Spec.Memory)),
		},
	}
	if portsResource := rangeResource("ports", t.Spec.Ports); portsResource != nil {
		info.Resources = append(info.Resources, portsResource)
	}
	return info
}

// Fill the Spec in the T, should be called during k8s scheduling, before binding.
func (t *T) FillFromDetails(details *mesos.Offer) error {
	if details == nil {
		//programming error
		panic("offer details are nil")
	}

	// compute used resources
	cpu := mresource.PodCPULimit(&t.Pod)
	mem := mresource.PodMemLimit(&t.Pod)
	log.V(3).Infof("Recording offer(s) %s/%s against pod %v: cpu: %.2f, mem: %.2f MB", details.Id, t.Pod.Namespace, t.Pod.Name, cpu, mem)

	t.Spec = Spec{
		SlaveID:       details.GetSlaveId().GetValue(),
		AssignedSlave: *details.Hostname,
		CPU:           cpu,
		Memory:        mem,
	}

	// fill in port mapping
	if mapping, err := t.mapper.Generate(t, details); err != nil {
		t.Reset()
		return err
	} else {
		ports := []uint64{}
		for _, entry := range mapping {
			ports = append(ports, entry.OfferPort)
		}
		t.Spec.PortMap = mapping
		t.Spec.Ports = ports
	}

	// hostname needs of the executor needs to match that of the offer, otherwise
	// the kubelet node status checker/updater is very unhappy
	const HOSTNAME_OVERRIDE_FLAG = "--hostname-override="
	hostname := details.GetHostname() // required field, non-empty
	hostnameOverride := HOSTNAME_OVERRIDE_FLAG + hostname

	argv := t.executor.Command.Arguments
	overwrite := false
	for i, arg := range argv {
		if strings.HasPrefix(arg, HOSTNAME_OVERRIDE_FLAG) {
			overwrite = true
			argv[i] = hostnameOverride
			break
		}
	}
	if !overwrite {
		t.executor.Command.Arguments = append(argv, hostnameOverride)
	}
	return nil
}

// Clear offer-related details from the task, should be called if/when an offer
// has already been assigned to a task but for some reason is no longer valid.
func (t *T) Reset() {
	log.V(3).Infof("Clearing offer(s) from pod %v", t.Pod.Name)
	t.Offer = nil
	t.Spec = Spec{}
}

func (t *T) AcceptOffer(offer *mesos.Offer) bool {
	if offer == nil {
		return false
	}

	// if the user has specified a target host, make sure this offer is for that host
	if t.Pod.Spec.NodeName != "" && offer.GetHostname() != t.Pod.Spec.NodeName {
		return false
	}

	// check ports
	if _, err := t.mapper.Generate(t, offer); err != nil {
		log.V(3).Info(err)
		return false
	}

	// find offered cpu and mem
	var (
		offeredCpus mresource.CPUShares
		offeredMem  mresource.MegaBytes
	)
	for _, resource := range offer.Resources {
		if resource.GetName() == "cpus" {
			offeredCpus = mresource.CPUShares(*resource.GetScalar().Value)
		}

		if resource.GetName() == "mem" {
			offeredMem = mresource.MegaBytes(*resource.GetScalar().Value)
		}
	}

	// calculate cpu and mem sum over all containers of the pod
	// TODO (@sttts): also support pod.spec.resources.limit.request
	// TODO (@sttts): take into account the executor resources
	cpu := mresource.PodCPULimit(&t.Pod)
	mem := mresource.PodMemLimit(&t.Pod)
	log.V(4).Infof("trying to match offer with pod %v/%v: cpus: %.2f mem: %.2f MB", t.Pod.Namespace, t.Pod.Name, cpu, mem)
	if (cpu > offeredCpus) || (mem > offeredMem) {
		log.V(3).Infof("not enough resources for pod %v/%v: cpus: %.2f mem: %.2f MB", t.Pod.Namespace, t.Pod.Name, cpu, mem)
		return false
	}

	return true
}

func (t *T) Set(f FlagType) {
	t.Flags[f] = struct{}{}
	if Launched == f {
		t.launchTime = time.Now()
		queueWaitTime := t.launchTime.Sub(t.CreateTime)
		metrics.QueueWaitTime.Observe(metrics.InMicroseconds(queueWaitTime))
	}
}

func (t *T) Has(f FlagType) (exists bool) {
	_, exists = t.Flags[f]
	return
}

func New(ctx api.Context, id string, pod api.Pod, executor *mesos.ExecutorInfo) (*T, error) {
	if executor == nil {
		return nil, fmt.Errorf("illegal argument: executor was nil")
	}
	key, err := MakePodKey(ctx, pod.Name)
	if err != nil {
		return nil, err
	}
	if id == "" {
		id = "pod." + uuid.NewUUID().String()
	}
	task := &T{
		ID:       id,
		Pod:      pod,
		State:    StatePending,
		podKey:   key,
		mapper:   MappingTypeForPod(&pod),
		Flags:    make(map[FlagType]struct{}),
		executor: proto.Clone(executor).(*mesos.ExecutorInfo),
	}
	task.CreateTime = time.Now()
	return task, nil
}

func (t *T) SaveRecoveryInfo(dict map[string]string) {
	dict[annotation.TaskIdKey] = t.ID
	dict[annotation.SlaveIdKey] = t.Spec.SlaveID
	dict[annotation.OfferIdKey] = t.Offer.Details().Id.GetValue()
	dict[annotation.ExecutorIdKey] = t.executor.ExecutorId.GetValue()
}

// reconstruct a task from metadata stashed in a pod entry. there are limited pod states that
// support reconstruction. if we expect to be able to reconstruct state but encounter errors
// in the process then those errors are returned. if the pod is in a seemingly valid state but
// otherwise does not support task reconstruction return false. if we're able to reconstruct
// state then return a reconstructed task and true.
//
// at this time task reconstruction is only supported for pods that have been annotated with
// binding metadata, which implies that they've previously been associated with a task and
// that mesos knows about it.
//
// assumes that the pod data comes from the k8s registry and reflects the desired state.
//
func RecoverFrom(pod api.Pod) (*T, bool, error) {
	// we only expect annotations if pod has been bound, which implies that it has already
	// been scheduled and launched
	if pod.Spec.NodeName == "" && len(pod.Annotations) == 0 {
		log.V(1).Infof("skipping recovery for unbound pod %v/%v", pod.Namespace, pod.Name)
		return nil, false, nil
	}

	// only process pods that are not in a terminal state
	switch pod.Status.Phase {
	case api.PodPending, api.PodRunning, api.PodUnknown: // continue
	default:
		log.V(1).Infof("skipping recovery for terminal pod %v/%v", pod.Namespace, pod.Name)
		return nil, false, nil
	}

	ctx := api.WithNamespace(api.NewDefaultContext(), pod.Namespace)
	key, err := MakePodKey(ctx, pod.Name)
	if err != nil {
		return nil, false, err
	}

	//TODO(jdef) recover ports (and other resource requirements?) from the pod spec as well

	now := time.Now()
	t := &T{
		Pod:        pod,
		CreateTime: now,
		podKey:     key,
		State:      StatePending, // possibly running? mesos will tell us during reconciliation
		Flags:      make(map[FlagType]struct{}),
		mapper:     MappingTypeForPod(&pod),
		launchTime: now,
		bindTime:   now,
	}
	var (
		offerId string
	)
	for _, k := range []string{
		annotation.BindingHostKey,
		annotation.TaskIdKey,
		annotation.SlaveIdKey,
		annotation.OfferIdKey,
		annotation.ExecutorIdKey,
	} {
		v, found := pod.Annotations[k]
		if !found {
			return nil, false, fmt.Errorf("incomplete metadata: missing value for pod annotation: %v", k)
		}
		switch k {
		case annotation.BindingHostKey:
			t.Spec.AssignedSlave = v
		case annotation.SlaveIdKey:
			t.Spec.SlaveID = v
		case annotation.OfferIdKey:
			offerId = v
		case annotation.TaskIdKey:
			t.ID = v
		case annotation.ExecutorIdKey:
			// this is nowhere near sufficient to re-launch a task, but we really just
			// want this for tracking
			t.executor = &mesos.ExecutorInfo{ExecutorId: mutil.NewExecutorID(v)}
		}
	}
	t.Offer = offers.Expired(offerId, t.Spec.AssignedSlave, 0)
	t.Flags[Launched] = struct{}{}
	t.Flags[Bound] = struct{}{}
	return t, true, nil
}
<commit_msg>Address review comments<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package podtask

import (
	"fmt"
	"strings"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/pborman/uuid"
	"k8s.io/kubernetes/contrib/mesos/pkg/offers"
	annotation "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta"
	"k8s.io/kubernetes/contrib/mesos/pkg/scheduler/metrics"
	mresource "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/resource"
	"k8s.io/kubernetes/pkg/api"

	log "github.com/golang/glog"
	mesos "github.com/mesos/mesos-go/mesosproto"
	mutil "github.com/mesos/mesos-go/mesosutil"
)

type StateType int

const (
	StatePending StateType = iota
	StateRunning
	StateFinished
	StateUnknown
)

type FlagType string

const (
	Launched = FlagType("launched")
	Bound    = FlagType("bound")
	Deleted  = FlagType("deleted")
)

// A struct that describes a pod task.
type T struct {
	ID          string
	Pod         api.Pod
	Spec        Spec
	Offer       offers.Perishable // thread-safe
	State       StateType
	Flags       map[FlagType]struct{}
	CreateTime  time.Time
	UpdatedTime time.Time // time of the most recent StatusUpdate we've seen from the mesos master

	podStatus  api.PodStatus
	executor   *mesos.ExecutorInfo // readonly
	podKey     string
	launchTime time.Time
	bindTime   time.Time
	mapper     HostPortMappingType
}

type Spec struct {
	SlaveID       string
	AssignedSlave string
	CPU           mresource.CPUShares
	Memory        mresource.MegaBytes
	PortMap       []HostPortMapping
	Ports         []uint64
	Data          []byte
}

// mostly-clone this pod task. the clone will actually share the some fields:
//   - executor    // OK because it's read only
//   - Offer       // OK because it's guarantees safe concurrent access
func (t *T) Clone() *T {
	if t == nil {
		return nil
	}

	// shallow-copy
	clone := *t

	// deep copy
	(&t.Spec).copyTo(&clone.Spec)
	clone.Flags = map[FlagType]struct{}{}
	for k := range t.Flags {
		clone.Flags[k] = struct{}{}
	}
	return &clone
}

func (old *Spec) copyTo(new *Spec) {
	if len(old.PortMap) > 0 {
		new.PortMap = append(([]HostPortMapping)(nil), old.PortMap...)
	}
	if len(old.Ports) > 0 {
		new.Ports = append(([]uint64)(nil), old.Ports...)
	}
	if len(old.Data) > 0 {
		new.Data = append(([]byte)(nil), old.Data...)
	}
}

func (t *T) HasAcceptedOffer() bool {
	return t.Spec.SlaveID != ""
}

func (t *T) GetOfferId() string {
	if t.Offer == nil {
		return ""
	}
	return t.Offer.Details().Id.GetValue()
}

func generateTaskName(pod *api.Pod) string {
	ns := pod.Namespace
	if ns == "" {
		ns = api.NamespaceDefault
	}
	return fmt.Sprintf("%s.%s.pods", pod.Name, ns)
}

func (t *T) BuildTaskInfo() *mesos.TaskInfo {
	info := &mesos.TaskInfo{
		Name:     proto.String(generateTaskName(&t.Pod)),
		TaskId:   mutil.NewTaskID(t.ID),
		SlaveId:  mutil.NewSlaveID(t.Spec.SlaveID),
		Executor: t.executor,
		Data:     t.Spec.Data,
		Resources: []*mesos.Resource{
			mutil.NewScalarResource("cpus", float64(t.Spec.CPU)),
			mutil.NewScalarResource("mem", float64(t.Spec.Memory)),
		},
	}
	if portsResource := rangeResource("ports", t.Spec.Ports); portsResource != nil {
		info.Resources = append(info.Resources, portsResource)
	}
	return info
}

// Fill the Spec in the T, should be called during k8s scheduling, before binding.
func (t *T) FillFromDetails(details *mesos.Offer) error {
	if details == nil {
		//programming error
		panic("offer details are nil")
	}

	// compute used resources
	cpu := mresource.PodCPULimit(&t.Pod)
	mem := mresource.PodMemLimit(&t.Pod)
	log.V(3).Infof("Recording offer(s) %s/%s against pod %v: cpu: %.2f, mem: %.2f MB", details.Id, t.Pod.Namespace, t.Pod.Name, cpu, mem)

	t.Spec = Spec{
		SlaveID:       details.GetSlaveId().GetValue(),
		AssignedSlave: details.GetHostname(),
		CPU:           cpu,
		Memory:        mem,
	}

	// fill in port mapping
	if mapping, err := t.mapper.Generate(t, details); err != nil {
		t.Reset()
		return err
	} else {
		ports := []uint64{}
		for _, entry := range mapping {
			ports = append(ports, entry.OfferPort)
		}
		t.Spec.PortMap = mapping
		t.Spec.Ports = ports
	}

	// hostname needs of the executor needs to match that of the offer, otherwise
	// the kubelet node status checker/updater is very unhappy
	const HOSTNAME_OVERRIDE_FLAG = "--hostname-override="
	hostname := details.GetHostname() // required field, non-empty
	hostnameOverride := HOSTNAME_OVERRIDE_FLAG + hostname

	argv := t.executor.Command.Arguments
	overwrite := false
	for i, arg := range argv {
		if strings.HasPrefix(arg, HOSTNAME_OVERRIDE_FLAG) {
			overwrite = true
			argv[i] = hostnameOverride
			break
		}
	}
	if !overwrite {
		t.executor.Command.Arguments = append(argv, hostnameOverride)
	}
	return nil
}

// Clear offer-related details from the task, should be called if/when an offer
// has already been assigned to a task but for some reason is no longer valid.
func (t *T) Reset() {
	log.V(3).Infof("Clearing offer(s) from pod %v", t.Pod.Name)
	t.Offer = nil
	t.Spec = Spec{}
}

func (t *T) AcceptOffer(offer *mesos.Offer) bool {
	if offer == nil {
		return false
	}

	// if the user has specified a target host, make sure this offer is for that host
	if t.Pod.Spec.NodeName != "" && offer.GetHostname() != t.Pod.Spec.NodeName {
		return false
	}

	// check ports
	if _, err := t.mapper.Generate(t, offer); err != nil {
		log.V(3).Info(err)
		return false
	}

	// find offered cpu and mem
	var (
		offeredCpus mresource.CPUShares
		offeredMem  mresource.MegaBytes
	)
	for _, resource := range offer.Resources {
		if resource.GetName() == "cpus" {
			offeredCpus = mresource.CPUShares(*resource.GetScalar().Value)
		}

		if resource.GetName() == "mem" {
			offeredMem = mresource.MegaBytes(*resource.GetScalar().Value)
		}
	}

	// calculate cpu and mem sum over all containers of the pod
	// TODO (@sttts): also support pod.spec.resources.limit.request
	// TODO (@sttts): take into account the executor resources
	cpu := mresource.PodCPULimit(&t.Pod)
	mem := mresource.PodMemLimit(&t.Pod)
	log.V(4).Infof("trying to match offer with pod %v/%v: cpus: %.2f mem: %.2f MB", t.Pod.Namespace, t.Pod.Name, cpu, mem)
	if (cpu > offeredCpus) || (mem > offeredMem) {
		log.V(3).Infof("not enough resources for pod %v/%v: cpus: %.2f mem: %.2f MB", t.Pod.Namespace, t.Pod.Name, cpu, mem)
		return false
	}

	return true
}

func (t *T) Set(f FlagType) {
	t.Flags[f] = struct{}{}
	if Launched == f {
		t.launchTime = time.Now()
		queueWaitTime := t.launchTime.Sub(t.CreateTime)
		metrics.QueueWaitTime.Observe(metrics.InMicroseconds(queueWaitTime))
	}
}

func (t *T) Has(f FlagType) (exists bool) {
	_, exists = t.Flags[f]
	return
}

func New(ctx api.Context, id string, pod api.Pod, executor *mesos.ExecutorInfo) (*T, error) {
	if executor == nil {
		return nil, fmt.Errorf("illegal argument: executor was nil")
	}
	key, err := MakePodKey(ctx, pod.Name)
	if err != nil {
		return nil, err
	}
	if id == "" {
		id = "pod." + uuid.NewUUID().String()
	}
	task := &T{
		ID:       id,
		Pod:      pod,
		State:    StatePending,
		podKey:   key,
		mapper:   MappingTypeForPod(&pod),
		Flags:    make(map[FlagType]struct{}),
		executor: proto.Clone(executor).(*mesos.ExecutorInfo),
	}
	task.CreateTime = time.Now()
	return task, nil
}

func (t *T) SaveRecoveryInfo(dict map[string]string) {
	dict[annotation.TaskIdKey] = t.ID
	dict[annotation.SlaveIdKey] = t.Spec.SlaveID
	dict[annotation.OfferIdKey] = t.Offer.Details().Id.GetValue()
	dict[annotation.ExecutorIdKey] = t.executor.ExecutorId.GetValue()
}

// reconstruct a task from metadata stashed in a pod entry. there are limited pod states that
// support reconstruction. if we expect to be able to reconstruct state but encounter errors
// in the process then those errors are returned. if the pod is in a seemingly valid state but
// otherwise does not support task reconstruction return false. if we're able to reconstruct
// state then return a reconstructed task and true.
//
// at this time task reconstruction is only supported for pods that have been annotated with
// binding metadata, which implies that they've previously been associated with a task and
// that mesos knows about it.
//
// assumes that the pod data comes from the k8s registry and reflects the desired state.
//
func RecoverFrom(pod api.Pod) (*T, bool, error) {
	// we only expect annotations if pod has been bound, which implies that it has already
	// been scheduled and launched
	if pod.Spec.NodeName == "" && len(pod.Annotations) == 0 {
		log.V(1).Infof("skipping recovery for unbound pod %v/%v", pod.Namespace, pod.Name)
		return nil, false, nil
	}

	// only process pods that are not in a terminal state
	switch pod.Status.Phase {
	case api.PodPending, api.PodRunning, api.PodUnknown: // continue
	default:
		log.V(1).Infof("skipping recovery for terminal pod %v/%v", pod.Namespace, pod.Name)
		return nil, false, nil
	}

	ctx := api.WithNamespace(api.NewDefaultContext(), pod.Namespace)
	key, err := MakePodKey(ctx, pod.Name)
	if err != nil {
		return nil, false, err
	}

	//TODO(jdef) recover ports (and other resource requirements?) from the pod spec as well

	now := time.Now()
	t := &T{
		Pod:        pod,
		CreateTime: now,
		podKey:     key,
		State:      StatePending, // possibly running? mesos will tell us during reconciliation
		Flags:      make(map[FlagType]struct{}),
		mapper:     MappingTypeForPod(&pod),
		launchTime: now,
		bindTime:   now,
	}
	var (
		offerId string
	)
	for _, k := range []string{
		annotation.BindingHostKey,
		annotation.TaskIdKey,
		annotation.SlaveIdKey,
		annotation.OfferIdKey,
		annotation.ExecutorIdKey,
	} {
		v, found := pod.Annotations[k]
		if !found {
			return nil, false, fmt.Errorf("incomplete metadata: missing value for pod annotation: %v", k)
		}
		switch k {
		case annotation.BindingHostKey:
			t.Spec.AssignedSlave = v
		case annotation.SlaveIdKey:
			t.Spec.SlaveID = v
		case annotation.OfferIdKey:
			offerId = v
		case annotation.TaskIdKey:
			t.ID = v
		case annotation.ExecutorIdKey:
			// this is nowhere near sufficient to re-launch a task, but we really just
			// want this for tracking
			t.executor = &mesos.ExecutorInfo{ExecutorId: mutil.NewExecutorID(v)}
		}
	}
	t.Offer = offers.Expired(offerId, t.Spec.AssignedSlave, 0)
	t.Flags[Launched] = struct{}{}
	t.Flags[Bound] = struct{}{}
	return t, true, nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*

Gopack program is a variant of the Plan 9 ar tool.  The original is documented at

	http://plan9.bell-labs.com/magic/man2html/1/ar

It adds a special Go-specific section __.PKGDEF that collects all the
Go type information from the files in the archive; that section is
used by the compiler when importing the package during compilation.

Usage: gopack [uvnbailo][mrxtdpq] archive files ...

The new option 'g' causes gopack to maintain the __.PKGDEF section
as files are added to the archive.

*/
package documentation
<commit_msg>gopack documentation: fixed typo (Removed extraneous "program".)<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*

Gopack is a variant of the Plan 9 ar tool.  The original is documented at

	http://plan9.bell-labs.com/magic/man2html/1/ar

It adds a special Go-specific section __.PKGDEF that collects all the
Go type information from the files in the archive; that section is
used by the compiler when importing the package during compilation.

Usage: gopack [uvnbailo][mrxtdpq] archive files ...

The new option 'g' causes gopack to maintain the __.PKGDEF section
as files are added to the archive.

*/
package documentation
<|endoftext|>


<commit_before>package fs

import (
	"io"
	"os"
	"syscall"

	"github.com/containerd/containerd/sys"
	"github.com/containerd/continuity/sysx"
	"github.com/pkg/errors"
	"golang.org/x/sys/unix"
)

func copyFileInfo(fi os.FileInfo, name string) error {
	st := fi.Sys().(*syscall.Stat_t)
	if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil {
		return errors.Wrapf(err, "failed to chown %s", name)
	}

	if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
		if err := os.Chmod(name, fi.Mode()); err != nil {
			return errors.Wrapf(err, "failed to chmod %s", name)
		}
	}

	timespec := []unix.Timespec{unix.Timespec(sys.StatAtime(st)), unix.Timespec(sys.StatMtime(st))}
	if err := unix.UtimesNanoAt(unix.AT_FDCWD, name, timespec, unix.AT_SYMLINK_NOFOLLOW); err != nil {
		return errors.Wrapf(err, "failed to utime %s", name)
	}

	return nil
}

func copyFileContent(dst, src *os.File) error {
	st, err := src.Stat()
	if err != nil {
		return errors.Wrap(err, "unable to stat source")
	}

	n, err := sysx.CopyFileRange(src.Fd(), nil, dst.Fd(), nil, int(st.Size()), 0)
	if err != nil {
		if err != unix.ENOSYS && err != unix.EXDEV {
			return errors.Wrap(err, "copy file range failed")
		}

		buf := bufferPool.Get().([]byte)
		_, err = io.CopyBuffer(dst, src, buf)
		bufferPool.Put(buf)
		return err
	}

	if int64(n) != st.Size() {
		return errors.Wrapf(err, "short copy: %d of %d", int64(n), st.Size())
	}

	return nil
}

func copyXAttrs(dst, src string) error {
	xattrKeys, err := sysx.LListxattr(src)
	if err != nil {
		return errors.Wrapf(err, "failed to list xattrs on %s", src)
	}
	for _, xattr := range xattrKeys {
		data, err := sysx.LGetxattr(src, xattr)
		if err != nil {
			return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src)
		}
		if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil {
			return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst)
		}
	}

	return nil
}

func copyDevice(dst string, fi os.FileInfo) error {
	st, ok := fi.Sys().(*syscall.Stat_t)
	if !ok {
		return errors.New("unsupported stat type")
	}
	return unix.Mknod(dst, uint32(fi.Mode()), int(st.Rdev))
}
<commit_msg>Use CopyFileRange from golang.org/x/sys/unix<commit_after>package fs

import (
	"io"
	"os"
	"syscall"

	"github.com/containerd/containerd/sys"
	"github.com/containerd/continuity/sysx"
	"github.com/pkg/errors"
	"golang.org/x/sys/unix"
)

func copyFileInfo(fi os.FileInfo, name string) error {
	st := fi.Sys().(*syscall.Stat_t)
	if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil {
		return errors.Wrapf(err, "failed to chown %s", name)
	}

	if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
		if err := os.Chmod(name, fi.Mode()); err != nil {
			return errors.Wrapf(err, "failed to chmod %s", name)
		}
	}

	timespec := []unix.Timespec{unix.Timespec(sys.StatAtime(st)), unix.Timespec(sys.StatMtime(st))}
	if err := unix.UtimesNanoAt(unix.AT_FDCWD, name, timespec, unix.AT_SYMLINK_NOFOLLOW); err != nil {
		return errors.Wrapf(err, "failed to utime %s", name)
	}

	return nil
}

func copyFileContent(dst, src *os.File) error {
	st, err := src.Stat()
	if err != nil {
		return errors.Wrap(err, "unable to stat source")
	}

	n, err := unix.CopyFileRange(int(src.Fd()), nil, int(dst.Fd()), nil, int(st.Size()), 0)
	if err != nil {
		if err != unix.ENOSYS && err != unix.EXDEV {
			return errors.Wrap(err, "copy file range failed")
		}

		buf := bufferPool.Get().([]byte)
		_, err = io.CopyBuffer(dst, src, buf)
		bufferPool.Put(buf)
		return err
	}

	if int64(n) != st.Size() {
		return errors.Wrapf(err, "short copy: %d of %d", int64(n), st.Size())
	}

	return nil
}

func copyXAttrs(dst, src string) error {
	xattrKeys, err := sysx.LListxattr(src)
	if err != nil {
		return errors.Wrapf(err, "failed to list xattrs on %s", src)
	}
	for _, xattr := range xattrKeys {
		data, err := sysx.LGetxattr(src, xattr)
		if err != nil {
			return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src)
		}
		if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil {
			return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst)
		}
	}

	return nil
}

func copyDevice(dst string, fi os.FileInfo) error {
	st, ok := fi.Sys().(*syscall.Stat_t)
	if !ok {
		return errors.New("unsupported stat type")
	}
	return unix.Mknod(dst, uint32(fi.Mode()), int(st.Rdev))
}
<|endoftext|>


<commit_before>/* Copyright 2017 Google Inc.
 * https://github.com/NeilFraser/CodeCity
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package flatpack

import (
	"fmt"
	"math"
	"reflect"
	"unsafe"
)

// This file provides some utility functions used in testing the rest
// of the package.

func recEqual(i1, i2 interface{}, disjoint bool) bool {
	return recValueEqual(reflect.ValueOf(i1), reflect.ValueOf(i2), disjoint)
func RecEqual(x, y interface{}, disjoint bool) bool {
	return recValueEqual(reflect.ValueOf(x), reflect.ValueOf(y), disjoint)
}

func recValueEqual(v1, v2 reflect.Value, disjoint bool) bool {
	return recEq(v1, v2, disjoint, make(map[unsafe.Pointer]unsafe.Pointer), make(map[unsafe.Pointer]unsafe.Pointer))
}

// recEq recursively tests for structural equality of v1 and
// v2.
//
// If disjoint is true, then v1 and v2 must not share any substructure.
//
// v1s and v2s keep track of previously-visited values; in v1s the
// keys are values of v1 already seen while the values are the
// corresponding v2 values; in v2s it is vice versa.  We keep maps
// for both directions because otherwise certain cases of structural
// dissimilarity (or shared substructure between v1 and v2) might not
// otherwise be detected.
//
// FXIME: Slices shoudl have better check for shared backing
func recEq(v1, v2 reflect.Value, disjoint bool, v1s, v2s map[unsafe.Pointer]unsafe.Pointer) bool {
	if !v1.IsValid() {
		return !v2.IsValid()
	} else if !v2.IsValid() {
		return false // Already know v1 valid.
	}
	if v1.Type() != v2.Type() {
		return false
	}

	switch v1.Kind() {
	case reflect.Map, reflect.Ptr, reflect.Slice:
		v1p := unsafe.Pointer(v1.Pointer())
		v2p := unsafe.Pointer(v2.Pointer())

		// Check for disjointness if requested:
		if disjoint {
			if v1.Kind() == reflect.Slice {
				if overlap(v1, v2) {
					return false
				}
			} else {
				_, v2in1 := v1s[v2p]
				_, v1in2 := v2s[v1p]
				if (!v1.IsNil() && v1p == v2p) || v2in1 || v1in2 {
					return false
				}
			}
		}

		// Check if we have previously visited this v1 (and if so whether
		// we have the correct corresponding v2):
		v1o, seen1 := v1s[v1p]
		v2o, seen2 := v2s[v2p]
		if seen1 {
			return seen2 && v1o == v2p && v2o == v1p
		} else if seen2 {
			return false
		}
		v1s[v1p] = v2p
		v2s[v2p] = v1p

	}

	switch v1.Kind() {
	case reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
		reflect.String:
		return v1.Interface() == v2.Interface()
	case reflect.Array:
		for i, l := 0, v1.Len(); i < l; i++ {
			if !recEq(v1.Index(i), v2.Index(i), disjoint, v1s, v2s) {
				return false
			}
		}
	case reflect.Complex64, reflect.Complex128:
		c1, c2 := v1.Complex(), v2.Complex()
		return same(real(c1), real(c2)) && same(imag(c1), imag(c2))
	case reflect.Float32, reflect.Float64:
		return same(v1.Float(), v2.Float())
	case reflect.Interface, reflect.Ptr:
		return recEq(v1.Elem(), v2.Elem(), disjoint, v1s, v2s)
	case reflect.Map:
		if v1.IsNil() != v2.IsNil() || v1.Len() != v2.Len() {
			return false
		}
		for _, k := range v1.MapKeys() {
			v1v := v1.MapIndex(k)
			v2v := v2.MapIndex(k)
			if !v1v.IsValid() || !v2v.IsValid() || !recEq(v1v, v2v, disjoint, v1s, v2s) {
				return false
			}
		}
	case reflect.Slice:
		if v1.IsNil() != v2.IsNil() || v1.Len() != v2.Len() {
			return false
		}
		for i, n := 0, v1.Len(); i < n; i++ {
			if !recEq(v1.Index(i), v2.Index(i), disjoint, v1s, v2s) {
				return false
			}
		}
	case reflect.Struct:
		if !v1.CanAddr() {
			vv := reflect.New(v1.Type()).Elem()
			vv.Set(v1)
			v1 = vv
		}
		if !v2.CanAddr() {
			vv := reflect.New(v2.Type()).Elem()
			vv.Set(v2)
			v2 = vv
		}
		for i, n := 0, v1.NumField(); i < n; i++ {
			if !recEq(defeat(v1.Field(i)), defeat(v2.Field(i)), disjoint, v1s, v2s) {
				return false
			}
		}
	case reflect.Func:
		// Because you can't meanigfully produce a disjoint f2 that
		// does the same as f1 at runtime (and would have no way to in
		// general to verify the functions were equivalent even if you
		// could create one), we just test for pointer equality (and
		// *not* disjointness):
		return unsafe.Pointer(v1.Pointer()) == unsafe.Pointer(v2.Pointer())
	case reflect.Chan, reflect.UnsafePointer:
		panic(fmt.Errorf("Comparison of %s not implemented", v1.Kind()))
	default:
		panic(fmt.Errorf("Invalid Kind %s", v1.Kind()))
	}
	return true
}

// same returns true iff its float64 args represent the same logical
// (rather than mathematical) value.  So:
//     same(NaN, NaN) == true
//     same(0, -0) == false
// and otherwise:
//      same(x, y) == (x == y)
func same(f1, f2 float64) bool {
	if f1 == f2 {
		if f1 == 0 && f2 == 0 {
			return math.Signbit(f1) == math.Signbit(f2)
		}
		return true
	}
	if math.IsNaN(f1) && math.IsNaN(f2) {
		return true
	}
	return false
}

// overlap makes a best-effort attempt to determine if the backing
// array for its arguments (which must be the reflect.Value
// representations of slices) overlap, and returns true if they do.
func overlap(s1, s2 reflect.Value) bool {
	// Possible false positives...  :-)
	return s1.Pointer() == s2.Pointer() && s1.Cap() > 0 && s2.Cap() > 0
}
<commit_msg>Fold in overlap check for slices<commit_after>/* Copyright 2017 Google Inc.
 * https://github.com/NeilFraser/CodeCity
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package flatpack

import (
	"fmt"
	"math"
	"reflect"
	"unsafe"
)

// This file provides some utility functions used in testing the rest
// of the package.

func recEqual(i1, i2 interface{}, disjoint bool) bool {
	return recValueEqual(reflect.ValueOf(i1), reflect.ValueOf(i2), disjoint)
func RecEqual(x, y interface{}, disjoint bool) bool {
	return recValueEqual(reflect.ValueOf(x), reflect.ValueOf(y), disjoint)
}

func recValueEqual(v1, v2 reflect.Value, disjoint bool) bool {
	return recEq(v1, v2, disjoint, make(map[unsafe.Pointer]unsafe.Pointer), make(map[unsafe.Pointer]unsafe.Pointer))
}

// recEq recursively tests for structural equality of v1 and
// v2.
//
// If disjoint is true, then v1 and v2 must not share any substructure.
//
// v1s and v2s keep track of previously-visited values; in v1s the
// keys are values of v1 already seen while the values are the
// corresponding v2 values; in v2s it is vice versa.  We keep maps
// for both directions because otherwise certain cases of structural
// dissimilarity (or shared substructure between v1 and v2) might not
// otherwise be detected.
//
// FXIME: Slices shoudl have better check for shared backing
func recEq(v1, v2 reflect.Value, disjoint bool, v1s, v2s map[unsafe.Pointer]unsafe.Pointer) bool {
	if !v1.IsValid() {
		return !v2.IsValid()
	} else if !v2.IsValid() {
		return false // Already know v1 valid.
	}
	if v1.Type() != v2.Type() {
		return false
	}

	switch v1.Kind() {
	case reflect.Map, reflect.Ptr, reflect.Slice:
		v1p := unsafe.Pointer(v1.Pointer())
		v2p := unsafe.Pointer(v2.Pointer())

		// Check for disjointness if requested:
		if disjoint {
			// (But ignore zero-capacity slices.)
			if v1.Kind() != reflect.Slice || v1.Cap() > 0 && v2.Cap() > 0 {
				_, v2in1 := v1s[v2p]
				_, v1in2 := v2s[v1p]
				if (!v1.IsNil() && v1p == v2p) || v2in1 || v1in2 {
					return false
				}
			}
		}

		// Check if we have previously visited this v1 (and if so whether
		// we have the correct corresponding v2):
		v1o, seen1 := v1s[v1p]
		v2o, seen2 := v2s[v2p]
		if seen1 {
			return seen2 && v1o == v2p && v2o == v1p
		} else if seen2 {
			return false
		}
		v1s[v1p] = v2p
		v2s[v2p] = v1p

	}

	switch v1.Kind() {
	case reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
		reflect.String:
		return v1.Interface() == v2.Interface()
	case reflect.Array:
		for i, l := 0, v1.Len(); i < l; i++ {
			if !recEq(v1.Index(i), v2.Index(i), disjoint, v1s, v2s) {
				return false
			}
		}
	case reflect.Complex64, reflect.Complex128:
		c1, c2 := v1.Complex(), v2.Complex()
		return same(real(c1), real(c2)) && same(imag(c1), imag(c2))
	case reflect.Float32, reflect.Float64:
		return same(v1.Float(), v2.Float())
	case reflect.Interface, reflect.Ptr:
		return recEq(v1.Elem(), v2.Elem(), disjoint, v1s, v2s)
	case reflect.Map:
		if v1.IsNil() != v2.IsNil() || v1.Len() != v2.Len() {
			return false
		}
		for _, k := range v1.MapKeys() {
			v1v := v1.MapIndex(k)
			v2v := v2.MapIndex(k)
			if !v1v.IsValid() || !v2v.IsValid() || !recEq(v1v, v2v, disjoint, v1s, v2s) {
				return false
			}
		}
	case reflect.Slice:
		if v1.IsNil() != v2.IsNil() || v1.Len() != v2.Len() {
			return false
		}
		for i, n := 0, v1.Len(); i < n; i++ {
			if !recEq(v1.Index(i), v2.Index(i), disjoint, v1s, v2s) {
				return false
			}
		}
	case reflect.Struct:
		if !v1.CanAddr() {
			vv := reflect.New(v1.Type()).Elem()
			vv.Set(v1)
			v1 = vv
		}
		if !v2.CanAddr() {
			vv := reflect.New(v2.Type()).Elem()
			vv.Set(v2)
			v2 = vv
		}
		for i, n := 0, v1.NumField(); i < n; i++ {
			if !recEq(defeat(v1.Field(i)), defeat(v2.Field(i)), disjoint, v1s, v2s) {
				return false
			}
		}
	case reflect.Func:
		// Because you can't meanigfully produce a disjoint f2 that
		// does the same as f1 at runtime (and would have no way to in
		// general to verify the functions were equivalent even if you
		// could create one), we just test for pointer equality (and
		// *not* disjointness):
		return unsafe.Pointer(v1.Pointer()) == unsafe.Pointer(v2.Pointer())
	case reflect.Chan, reflect.UnsafePointer:
		panic(fmt.Errorf("Comparison of %s not implemented", v1.Kind()))
	default:
		panic(fmt.Errorf("Invalid Kind %s", v1.Kind()))
	}
	return true
}

// same returns true iff its float64 args represent the same logical
// (rather than mathematical) value.  So:
//     same(NaN, NaN) == true
//     same(0, -0) == false
// and otherwise:
//      same(x, y) == (x == y)
func same(f1, f2 float64) bool {
	if f1 == f2 {
		if f1 == 0 && f2 == 0 {
			return math.Signbit(f1) == math.Signbit(f2)
		}
		return true
	}
	if math.IsNaN(f1) && math.IsNaN(f2) {
		return true
	}
	return false
}
<|endoftext|>