<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"

	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/pkg/version"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
)

const (
	podName = "pfpod"
)

// TODO support other ports besides 80
var (
	portForwardRegexp        = regexp.MustCompile("Forwarding from 127.0.0.1:([0-9]+) -> 80")
	portForwardPortToStdOutV = version.MustParse("v1.3.0-alpha.4")
)

func pfPod(expectedClientData, chunks, chunkSize, chunkIntervalMillis string) *v1.Pod {
	return &v1.Pod{
		ObjectMeta: v1.ObjectMeta{
			Name:   podName,
			Labels: map[string]string{"name": podName},
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:  "portforwardtester",
					Image: "gcr.io/google_containers/portforwardtester:1.2",
					Env: []v1.EnvVar{
						{
							Name:  "BIND_PORT",
							Value: "80",
						},
						{
							Name:  "EXPECTED_CLIENT_DATA",
							Value: expectedClientData,
						},
						{
							Name:  "CHUNKS",
							Value: chunks,
						},
						{
							Name:  "CHUNK_SIZE",
							Value: chunkSize,
						},
						{
							Name:  "CHUNK_INTERVAL",
							Value: chunkIntervalMillis,
						},
					},
				},
			},
			RestartPolicy: v1.RestartPolicyNever,
		},
	}
}

type portForwardCommand struct {
	cmd  *exec.Cmd
	port int
}

// Stop attempts to gracefully stop `kubectl port-forward`, only killing it if necessary.
// This helps avoid spdy goroutine leaks in the Kubelet.
func (c *portForwardCommand) Stop() {
	// SIGINT signals that kubectl port-forward should gracefully terminate
	if err := c.cmd.Process.Signal(syscall.SIGINT); err != nil {
		framework.Logf("error sending SIGINT to kubectl port-forward: %v", err)
	}

	// try to wait for a clean exit
	done := make(chan error)
	go func() {
		done <- c.cmd.Wait()
	}()

	expired := time.NewTimer(wait.ForeverTestTimeout)
	defer expired.Stop()

	select {
	case err := <-done:
		if err == nil {
			// success
			return
		}
		framework.Logf("error waiting for kubectl port-forward to exit: %v", err)
	case <-expired.C:
		framework.Logf("timed out waiting for kubectl port-forward to exit")
	}

	framework.Logf("trying to forcibly kill kubectl port-forward")
	framework.TryKill(c.cmd)
}

func runPortForward(ns, podName string, port int) *portForwardCommand {
	cmd := framework.KubectlCmd("port-forward", fmt.Sprintf("--namespace=%v", ns), podName, fmt.Sprintf(":%d", port))
	// This is somewhat ugly but is the only way to retrieve the port that was picked
	// by the port-forward command. We don't want to hard code the port as we have no
	// way of guaranteeing we can pick one that isn't in use, particularly on Jenkins.
	framework.Logf("starting port-forward command and streaming output")
	stdout, stderr, err := framework.StartCmdAndStreamOutput(cmd)
	if err != nil {
		framework.Failf("Failed to start port-forward command: %v", err)
	}

	buf := make([]byte, 128)

	// After v1.3.0-alpha.4 (#17030), kubectl port-forward outputs port
	// info to stdout, not stderr, so for version-skewed tests, look there
	// instead.
	var portOutput io.ReadCloser
	if useStdOut, err := framework.KubectlVersionGTE(portForwardPortToStdOutV); err != nil {
		framework.Failf("Failed to get kubectl version: %v", err)
	} else if useStdOut {
		portOutput = stdout
	} else {
		portOutput = stderr
	}

	var n int
	framework.Logf("reading from `kubectl port-forward` command's stdout")
	if n, err = portOutput.Read(buf); err != nil {
		framework.Failf("Failed to read from kubectl port-forward stdout: %v", err)
	}
	portForwardOutput := string(buf[:n])
	match := portForwardRegexp.FindStringSubmatch(portForwardOutput)
	if len(match) != 2 {
		framework.Failf("Failed to parse kubectl port-forward output: %s", portForwardOutput)
	}

	listenPort, err := strconv.Atoi(match[1])
	if err != nil {
		framework.Failf("Error converting %s to an int: %v", match[1], err)
	}

	return &portForwardCommand{
		cmd:  cmd,
		port: listenPort,
	}
}

var _ = framework.KubeDescribe("Port forwarding", func() {
	f := framework.NewDefaultFramework("port-forwarding")

	framework.KubeDescribe("With a server that expects a client request", func() {
		It("should support a client that connects, sends no data, and disconnects [Conformance]", func() {
			By("creating the target pod")
			pod := pfPod("abc", "1", "1", "1")
			if _, err := f.ClientSet.Core().Pods(f.Namespace.Name).Create(pod); err != nil {
				framework.Failf("Couldn't create pod: %v", err)
			}
			if err := f.WaitForPodRunning(pod.Name); err != nil {
				framework.Failf("Pod did not start running: %v", err)
			}
			defer func() {
				logs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester")
				if err != nil {
					framework.Logf("Error getting pod log: %v", err)
				} else {
					framework.Logf("Pod log:\n%s", logs)
				}
			}()

			By("Running 'kubectl port-forward'")
			cmd := runPortForward(f.Namespace.Name, pod.Name, 80)
			defer cmd.Stop()

			By("Dialing the local port")
			conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", cmd.port))
			if err != nil {
				framework.Failf("Couldn't connect to port %d: %v", cmd.port, err)
			}

			By("Closing the connection to the local port")
			conn.Close()

			By("Waiting for the target pod to stop running")
			if err := f.WaitForPodNoLongerRunning(pod.Name); err != nil {
				framework.Failf("Pod did not stop running: %v", err)
			}

			By("Verifying logs")
			logOutput, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester")
			if err != nil {
				framework.Failf("Error retrieving pod logs: %v", err)
			}
			verifyLogMessage(logOutput, "Accepted client connection")
			verifyLogMessage(logOutput, "Expected to read 3 bytes from client, but got 0 instead")
		})

		It("should support a client that connects, sends data, and disconnects [Conformance] [Flaky]", func() {
			By("creating the target pod")
			pod := pfPod("abc", "10", "10", "100")
			if _, err := f.ClientSet.Core().Pods(f.Namespace.Name).Create(pod); err != nil {
				framework.Failf("Couldn't create pod: %v", err)
			}
			if err := f.WaitForPodRunning(pod.Name); err != nil {
				framework.Failf("Pod did not start running: %v", err)
			}
			defer func() {
				logs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester")
				if err != nil {
					framework.Logf("Error getting pod log: %v", err)
				} else {
					framework.Logf("Pod log:\n%s", logs)
				}
			}()

			By("Running 'kubectl port-forward'")
			cmd := runPortForward(f.Namespace.Name, pod.Name, 80)
			defer cmd.Stop()

			By("Dialing the local port")
			addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("127.0.0.1:%d", cmd.port))
			if err != nil {
				framework.Failf("Error resolving tcp addr: %v", err)
			}
			conn, err := net.DialTCP("tcp", nil, addr)
			if err != nil {
				framework.Failf("Couldn't connect to port %d: %v", cmd.port, err)
			}
			defer func() {
				By("Closing the connection to the local port")
				conn.Close()
			}()

			By("Sending the expected data to the local port")
			fmt.Fprint(conn, "abc")

			By("Closing the write half of the client's connection")
			conn.CloseWrite()

			By("Reading data from the local port")
			fromServer, err := ioutil.ReadAll(conn)
			if err != nil {
				framework.Failf("Unexpected error reading data from the server: %v", err)
			}

			if e, a := strings.Repeat("x", 100), string(fromServer); e != a {
				framework.Failf("Expected %q from server, got %q", e, a)
			}

			By("Waiting for the target pod to stop running")
			if err := f.WaitForPodNoLongerRunning(pod.Name); err != nil {
				framework.Failf("Pod did not stop running: %v", err)
			}

			By("Verifying logs")
			logOutput, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester")
			if err != nil {
				framework.Failf("Error retrieving pod logs: %v", err)
			}
			verifyLogMessage(logOutput, "^Accepted client connection$")
			verifyLogMessage(logOutput, "^Received expected client data$")
			verifyLogMessage(logOutput, "^Done$")
		})
	})
	framework.KubeDescribe("With a server that expects no client request", func() {
		It("should support a client that connects, sends no data, and disconnects [Conformance] [Flaky]", func() {
			By("creating the target pod")
			pod := pfPod("", "10", "10", "100")
			if _, err := f.ClientSet.Core().Pods(f.Namespace.Name).Create(pod); err != nil {
				framework.Failf("Couldn't create pod: %v", err)
			}
			if err := f.WaitForPodRunning(pod.Name); err != nil {
				framework.Failf("Pod did not start running: %v", err)
			}
			defer func() {
				logs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester")
				if err != nil {
					framework.Logf("Error getting pod log: %v", err)
				} else {
					framework.Logf("Pod log:\n%s", logs)
				}
			}()

			By("Running 'kubectl port-forward'")
			cmd := runPortForward(f.Namespace.Name, pod.Name, 80)
			defer cmd.Stop()

			By("Dialing the local port")
			conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", cmd.port))
			if err != nil {
				framework.Failf("Couldn't connect to port %d: %v", cmd.port, err)
			}
			defer func() {
				By("Closing the connection to the local port")
				conn.Close()
			}()

			By("Reading data from the local port")
			fromServer, err := ioutil.ReadAll(conn)
			if err != nil {
				framework.Failf("Unexpected error reading data from the server: %v", err)
			}

			if e, a := strings.Repeat("x", 100), string(fromServer); e != a {
				framework.Failf("Expected %q from server, got %q", e, a)
			}

			By("Waiting for the target pod to stop running")
			if err := f.WaitForPodNoLongerRunning(pod.Name); err != nil {
				framework.Failf("Pod did not stop running: %v", err)
			}

			By("Verifying logs")
			logOutput, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester")
			if err != nil {
				framework.Failf("Error retrieving pod logs: %v", err)
			}
			verifyLogMessage(logOutput, "Accepted client connection")
			verifyLogMessage(logOutput, "Done")
		})
	})
})

func verifyLogMessage(log, expected string) {
	re := regexp.MustCompile(expected)
	lines := strings.Split(log, "\n")
	for i := range lines {
		if re.MatchString(lines[i]) {
			return
		}
	}
	framework.Failf("Missing %q from log: %s", expected, log)
}
<commit_msg>e2e: mark 3rd portforwarding test as flaky<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"

	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/pkg/version"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
)

const (
	podName = "pfpod"
)

// TODO support other ports besides 80
var (
	portForwardRegexp        = regexp.MustCompile("Forwarding from 127.0.0.1:([0-9]+) -> 80")
	portForwardPortToStdOutV = version.MustParse("v1.3.0-alpha.4")
)

func pfPod(expectedClientData, chunks, chunkSize, chunkIntervalMillis string) *v1.Pod {
	return &v1.Pod{
		ObjectMeta: v1.ObjectMeta{
			Name:   podName,
			Labels: map[string]string{"name": podName},
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:  "portforwardtester",
					Image: "gcr.io/google_containers/portforwardtester:1.2",
					Env: []v1.EnvVar{
						{
							Name:  "BIND_PORT",
							Value: "80",
						},
						{
							Name:  "EXPECTED_CLIENT_DATA",
							Value: expectedClientData,
						},
						{
							Name:  "CHUNKS",
							Value: chunks,
						},
						{
							Name:  "CHUNK_SIZE",
							Value: chunkSize,
						},
						{
							Name:  "CHUNK_INTERVAL",
							Value: chunkIntervalMillis,
						},
					},
				},
			},
			RestartPolicy: v1.RestartPolicyNever,
		},
	}
}

type portForwardCommand struct {
	cmd  *exec.Cmd
	port int
}

// Stop attempts to gracefully stop `kubectl port-forward`, only killing it if necessary.
// This helps avoid spdy goroutine leaks in the Kubelet.
func (c *portForwardCommand) Stop() {
	// SIGINT signals that kubectl port-forward should gracefully terminate
	if err := c.cmd.Process.Signal(syscall.SIGINT); err != nil {
		framework.Logf("error sending SIGINT to kubectl port-forward: %v", err)
	}

	// try to wait for a clean exit
	done := make(chan error)
	go func() {
		done <- c.cmd.Wait()
	}()

	expired := time.NewTimer(wait.ForeverTestTimeout)
	defer expired.Stop()

	select {
	case err := <-done:
		if err == nil {
			// success
			return
		}
		framework.Logf("error waiting for kubectl port-forward to exit: %v", err)
	case <-expired.C:
		framework.Logf("timed out waiting for kubectl port-forward to exit")
	}

	framework.Logf("trying to forcibly kill kubectl port-forward")
	framework.TryKill(c.cmd)
}

func runPortForward(ns, podName string, port int) *portForwardCommand {
	cmd := framework.KubectlCmd("port-forward", fmt.Sprintf("--namespace=%v", ns), podName, fmt.Sprintf(":%d", port))
	// This is somewhat ugly but is the only way to retrieve the port that was picked
	// by the port-forward command. We don't want to hard code the port as we have no
	// way of guaranteeing we can pick one that isn't in use, particularly on Jenkins.
	framework.Logf("starting port-forward command and streaming output")
	stdout, stderr, err := framework.StartCmdAndStreamOutput(cmd)
	if err != nil {
		framework.Failf("Failed to start port-forward command: %v", err)
	}

	buf := make([]byte, 128)

	// After v1.3.0-alpha.4 (#17030), kubectl port-forward outputs port
	// info to stdout, not stderr, so for version-skewed tests, look there
	// instead.
	var portOutput io.ReadCloser
	if useStdOut, err := framework.KubectlVersionGTE(portForwardPortToStdOutV); err != nil {
		framework.Failf("Failed to get kubectl version: %v", err)
	} else if useStdOut {
		portOutput = stdout
	} else {
		portOutput = stderr
	}

	var n int
	framework.Logf("reading from `kubectl port-forward` command's stdout")
	if n, err = portOutput.Read(buf); err != nil {
		framework.Failf("Failed to read from kubectl port-forward stdout: %v", err)
	}
	portForwardOutput := string(buf[:n])
	match := portForwardRegexp.FindStringSubmatch(portForwardOutput)
	if len(match) != 2 {
		framework.Failf("Failed to parse kubectl port-forward output: %s", portForwardOutput)
	}

	listenPort, err := strconv.Atoi(match[1])
	if err != nil {
		framework.Failf("Error converting %s to an int: %v", match[1], err)
	}

	return &portForwardCommand{
		cmd:  cmd,
		port: listenPort,
	}
}

var _ = framework.KubeDescribe("Port forwarding", func() {
	f := framework.NewDefaultFramework("port-forwarding")

	framework.KubeDescribe("With a server that expects a client request", func() {
		It("should support a client that connects, sends no data, and disconnects [Conformance] [Flaky]", func() {
			By("creating the target pod")
			pod := pfPod("abc", "1", "1", "1")
			if _, err := f.ClientSet.Core().Pods(f.Namespace.Name).Create(pod); err != nil {
				framework.Failf("Couldn't create pod: %v", err)
			}
			if err := f.WaitForPodRunning(pod.Name); err != nil {
				framework.Failf("Pod did not start running: %v", err)
			}
			defer func() {
				logs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester")
				if err != nil {
					framework.Logf("Error getting pod log: %v", err)
				} else {
					framework.Logf("Pod log:\n%s", logs)
				}
			}()

			By("Running 'kubectl port-forward'")
			cmd := runPortForward(f.Namespace.Name, pod.Name, 80)
			defer cmd.Stop()

			By("Dialing the local port")
			conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", cmd.port))
			if err != nil {
				framework.Failf("Couldn't connect to port %d: %v", cmd.port, err)
			}

			By("Closing the connection to the local port")
			conn.Close()

			By("Waiting for the target pod to stop running")
			if err := f.WaitForPodNoLongerRunning(pod.Name); err != nil {
				framework.Failf("Pod did not stop running: %v", err)
			}

			By("Verifying logs")
			logOutput, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester")
			if err != nil {
				framework.Failf("Error retrieving pod logs: %v", err)
			}
			verifyLogMessage(logOutput, "Accepted client connection")
			verifyLogMessage(logOutput, "Expected to read 3 bytes from client, but got 0 instead")
		})

		It("should support a client that connects, sends data, and disconnects [Conformance] [Flaky]", func() {
			By("creating the target pod")
			pod := pfPod("abc", "10", "10", "100")
			if _, err := f.ClientSet.Core().Pods(f.Namespace.Name).Create(pod); err != nil {
				framework.Failf("Couldn't create pod: %v", err)
			}
			if err := f.WaitForPodRunning(pod.Name); err != nil {
				framework.Failf("Pod did not start running: %v", err)
			}
			defer func() {
				logs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester")
				if err != nil {
					framework.Logf("Error getting pod log: %v", err)
				} else {
					framework.Logf("Pod log:\n%s", logs)
				}
			}()

			By("Running 'kubectl port-forward'")
			cmd := runPortForward(f.Namespace.Name, pod.Name, 80)
			defer cmd.Stop()

			By("Dialing the local port")
			addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("127.0.0.1:%d", cmd.port))
			if err != nil {
				framework.Failf("Error resolving tcp addr: %v", err)
			}
			conn, err := net.DialTCP("tcp", nil, addr)
			if err != nil {
				framework.Failf("Couldn't connect to port %d: %v", cmd.port, err)
			}
			defer func() {
				By("Closing the connection to the local port")
				conn.Close()
			}()

			By("Sending the expected data to the local port")
			fmt.Fprint(conn, "abc")

			By("Closing the write half of the client's connection")
			conn.CloseWrite()

			By("Reading data from the local port")
			fromServer, err := ioutil.ReadAll(conn)
			if err != nil {
				framework.Failf("Unexpected error reading data from the server: %v", err)
			}

			if e, a := strings.Repeat("x", 100), string(fromServer); e != a {
				framework.Failf("Expected %q from server, got %q", e, a)
			}

			By("Waiting for the target pod to stop running")
			if err := f.WaitForPodNoLongerRunning(pod.Name); err != nil {
				framework.Failf("Pod did not stop running: %v", err)
			}

			By("Verifying logs")
			logOutput, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester")
			if err != nil {
				framework.Failf("Error retrieving pod logs: %v", err)
			}
			verifyLogMessage(logOutput, "^Accepted client connection$")
			verifyLogMessage(logOutput, "^Received expected client data$")
			verifyLogMessage(logOutput, "^Done$")
		})
	})
	framework.KubeDescribe("With a server that expects no client request", func() {
		It("should support a client that connects, sends no data, and disconnects [Conformance] [Flaky]", func() {
			By("creating the target pod")
			pod := pfPod("", "10", "10", "100")
			if _, err := f.ClientSet.Core().Pods(f.Namespace.Name).Create(pod); err != nil {
				framework.Failf("Couldn't create pod: %v", err)
			}
			if err := f.WaitForPodRunning(pod.Name); err != nil {
				framework.Failf("Pod did not start running: %v", err)
			}
			defer func() {
				logs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester")
				if err != nil {
					framework.Logf("Error getting pod log: %v", err)
				} else {
					framework.Logf("Pod log:\n%s", logs)
				}
			}()

			By("Running 'kubectl port-forward'")
			cmd := runPortForward(f.Namespace.Name, pod.Name, 80)
			defer cmd.Stop()

			By("Dialing the local port")
			conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", cmd.port))
			if err != nil {
				framework.Failf("Couldn't connect to port %d: %v", cmd.port, err)
			}
			defer func() {
				By("Closing the connection to the local port")
				conn.Close()
			}()

			By("Reading data from the local port")
			fromServer, err := ioutil.ReadAll(conn)
			if err != nil {
				framework.Failf("Unexpected error reading data from the server: %v", err)
			}

			if e, a := strings.Repeat("x", 100), string(fromServer); e != a {
				framework.Failf("Expected %q from server, got %q", e, a)
			}

			By("Waiting for the target pod to stop running")
			if err := f.WaitForPodNoLongerRunning(pod.Name); err != nil {
				framework.Failf("Pod did not stop running: %v", err)
			}

			By("Verifying logs")
			logOutput, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, "portforwardtester")
			if err != nil {
				framework.Failf("Error retrieving pod logs: %v", err)
			}
			verifyLogMessage(logOutput, "Accepted client connection")
			verifyLogMessage(logOutput, "Done")
		})
	})
})

func verifyLogMessage(log, expected string) {
	re := regexp.MustCompile(expected)
	lines := strings.Split(log, "\n")
	for i := range lines {
		if re.MatchString(lines[i]) {
			return
		}
	}
	framework.Failf("Missing %q from log: %s", expected, log)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// This test requires Rescheduler to be enabled.
var _ = framework.KubeDescribe("Rescheduler [Serial]", func() {
	f := framework.NewDefaultFramework("rescheduler")
	var ns string
	var totalMillicores int

	BeforeEach(func() {
		framework.SkipUnlessProviderIs("gce")
		ns = f.Namespace.Name
		nodes := framework.GetReadySchedulableNodesOrDie(f.Client)
		nodeCount := len(nodes.Items)
		Expect(nodeCount).NotTo(BeZero())

		cpu := nodes.Items[0].Status.Capacity[api.ResourceCPU]
		totalMillicores = int((&cpu).MilliValue()) * nodeCount
	})

	It("should ensure that critical pod is scheduled in case there is no resources available", func() {
		By("reserving all available cpu")
		err := reserveAllCpu(f, "reserve-all-cpu", totalMillicores)
		defer framework.DeleteRCAndPods(f.Client, ns, "reserve-all-cpu")
		framework.ExpectNoError(err)

		By("creating a new instance of DNS and waiting for DNS to be scheduled")
		label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-dns"}))
		listOpts := api.ListOptions{LabelSelector: label}
		rcs, err := f.Client.ReplicationControllers(api.NamespaceSystem).List(listOpts)
		framework.ExpectNoError(err)
		Expect(len(rcs.Items)).Should(Equal(1))

		rc := rcs.Items[0]
		replicas := uint(rc.Spec.Replicas)

		err = framework.ScaleRC(f.Client, api.NamespaceSystem, rc.Name, replicas+1, true)
		defer framework.ExpectNoError(framework.ScaleRC(f.Client, api.NamespaceSystem, rc.Name, replicas, true))
		framework.ExpectNoError(err)
	})
})

func reserveAllCpu(f *framework.Framework, id string, millicores int) error {
	timeout := 5 * time.Minute
	replicas := millicores / 100

	ReserveCpu(f, id, 1, 100)
	framework.ExpectNoError(framework.ScaleRC(f.Client, f.Namespace.Name, id, uint(replicas), false))

	for start := time.Now(); time.Since(start) < timeout; time.Sleep(10 * time.Second) {
		pods, err := framework.GetPodsInNamespace(f.Client, f.Namespace.Name, framework.ImagePullerLabels)
		if err != nil {
			return err
		}

		if len(pods) != replicas {
			continue
		}

		allRunningOrUnschedulable := true
		for _, pod := range pods {
			if !podRunningOrUnschedulable(pod) {
				allRunningOrUnschedulable = false
				break
			}
		}
		if allRunningOrUnschedulable {
			return nil
		}
	}
	return fmt.Errorf("Pod name %s: Gave up waiting %v for %d pods to come up", id, timeout, replicas)
}

func podRunningOrUnschedulable(pod *api.Pod) bool {
	_, cond := api.GetPodCondition(&pod.Status, api.PodScheduled)
	if cond != nil && cond.Status == api.ConditionFalse && cond.Reason == "Unschedulable" {
		return true
	}
	running, _ := framework.PodRunningReady(pod)
	return running
}
<commit_msg>Enabled Rescheduler e2e for gke<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// This test requires Rescheduler to be enabled.
var _ = framework.KubeDescribe("Rescheduler [Serial]", func() {
	f := framework.NewDefaultFramework("rescheduler")
	var ns string
	var totalMillicores int

	BeforeEach(func() {
		framework.SkipUnlessProviderIs("gce", "gke")
		ns = f.Namespace.Name
		nodes := framework.GetReadySchedulableNodesOrDie(f.Client)
		nodeCount := len(nodes.Items)
		Expect(nodeCount).NotTo(BeZero())

		cpu := nodes.Items[0].Status.Capacity[api.ResourceCPU]
		totalMillicores = int((&cpu).MilliValue()) * nodeCount
	})

	It("should ensure that critical pod is scheduled in case there is no resources available", func() {
		By("reserving all available cpu")
		err := reserveAllCpu(f, "reserve-all-cpu", totalMillicores)
		defer framework.DeleteRCAndPods(f.Client, ns, "reserve-all-cpu")
		framework.ExpectNoError(err)

		By("creating a new instance of DNS and waiting for DNS to be scheduled")
		label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-dns"}))
		listOpts := api.ListOptions{LabelSelector: label}
		rcs, err := f.Client.ReplicationControllers(api.NamespaceSystem).List(listOpts)
		framework.ExpectNoError(err)
		Expect(len(rcs.Items)).Should(Equal(1))

		rc := rcs.Items[0]
		replicas := uint(rc.Spec.Replicas)

		err = framework.ScaleRC(f.Client, api.NamespaceSystem, rc.Name, replicas+1, true)
		defer framework.ExpectNoError(framework.ScaleRC(f.Client, api.NamespaceSystem, rc.Name, replicas, true))
		framework.ExpectNoError(err)
	})
})

func reserveAllCpu(f *framework.Framework, id string, millicores int) error {
	timeout := 5 * time.Minute
	replicas := millicores / 100

	ReserveCpu(f, id, 1, 100)
	framework.ExpectNoError(framework.ScaleRC(f.Client, f.Namespace.Name, id, uint(replicas), false))

	for start := time.Now(); time.Since(start) < timeout; time.Sleep(10 * time.Second) {
		pods, err := framework.GetPodsInNamespace(f.Client, f.Namespace.Name, framework.ImagePullerLabels)
		if err != nil {
			return err
		}

		if len(pods) != replicas {
			continue
		}

		allRunningOrUnschedulable := true
		for _, pod := range pods {
			if !podRunningOrUnschedulable(pod) {
				allRunningOrUnschedulable = false
				break
			}
		}
		if allRunningOrUnschedulable {
			return nil
		}
	}
	return fmt.Errorf("Pod name %s: Gave up waiting %v for %d pods to come up", id, timeout, replicas)
}

func podRunningOrUnschedulable(pod *api.Pod) bool {
	_, cond := api.GetPodCondition(&pod.Status, api.PodScheduled)
	if cond != nil && cond.Status == api.ConditionFalse && cond.Reason == "Unschedulable" {
		return true
	}
	running, _ := framework.PodRunningReady(pod)
	return running
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/*
Package e2e implements support for running kubeadm e2e tests or kubernetes e2e test.

It takes care of building upstream test suites if necessary, and provides "sane" defaults
for simplifying test invokation.
*/
package e2e

import (
	"fmt"

	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"

	kindexec "sigs.k8s.io/kind/pkg/exec"
)

// Option is an Runner configuration option supplied to NewRunner
type Option func(*Runner)

// KubeRoot option sets the kubernetes checkout folder
func KubeRoot(kubeRoot string) Option {
	return func(r *Runner) {
		r.kubeRoot = kubeRoot
	}
}

// WithGinkgoFlags option sets flags for ginkgo
func WithGinkgoFlags(ginkgoFlags GinkgoFlags) Option {
	return func(r *Runner) {
		r.ginkgoFlags = ginkgoFlags
	}
}

// WithSuiteFlags option sets flags for the test program
func WithSuiteFlags(suiteFlags SuiteFlags) Option {
	return func(r *Runner) {
		r.suiteFlags = suiteFlags
	}
}

// Runner defines attributes for a Kubernetes artifact extractor
type Runner struct {
	testBinary     string
	makeBinaryGoal string
	kubeRoot       string
	ginkgoFlags    GinkgoFlags
	suiteFlags     SuiteFlags
}

// NewKubernetesTestRunner returns a new E2E (Kubernetes) test runner configured with the given options
func NewKubernetesTestRunner(options ...Option) (runner *Runner, err error) {
	return newTestRunner("e2e.test", "test/e2e/e2e.test", options...)
}

// NewKubeadmTestRunner returns a new E2E kubeadm test runner configured with the given options
func NewKubeadmTestRunner(options ...Option) (runner *Runner, err error) {
	return newTestRunner("e2e_kubeadm.test", "test/e2e_kubeadm/e2e_kubeadm.test", options...)
}

// newTestRunner returns a new test runner - using ginkgo and the given test binary - configured with the given options
func newTestRunner(testBinary, makeBinaryGoal string, options ...Option) (runner *Runner, err error) {
	runner = &Runner{
		testBinary:     testBinary,
		makeBinaryGoal: makeBinaryGoal,
	}

	// apply user options
	for _, option := range options {
		option(runner)
	}

	// sets kubeRoot if not provided by the user
	if runner.kubeRoot == "" {
		runner.kubeRoot, err = findKubeRoot()
		if err != nil {
			return nil, errors.Wrap(err, "")
		}
		log.Infof("using Kubernetes checkout in %s folder", runner.kubeRoot)
	}

	return runner, nil
}

// Run executes tests as defined by the selected runner options.
// it takes care of building ginkgo and upstream test suites if necessary,
func (r *Runner) Run() error {
	// find a ginkgo binary or build it if it not exists
	ginkgoBinary, err := getOrBuildBinary(r.kubeRoot, "ginkgo", "vendor/github.com/onsi/ginkgo/ginkgo")
	if err != nil {
		return err
	}

	// find the binary with the test suites to be executes or build it if it not exists
	testBinary, err := getOrBuildBinary(r.kubeRoot, r.testBinary, r.makeBinaryGoal)
	if err != nil {
		return err
	}

	// prepare args to be passed to ginkgo test runner:
	// ginkgo [ginkgo-flags] [test-suite-binary] -- [test-suite-flags]
	var args []string
	for k, v := range r.ginkgoFlags {
		args = append(args, fmt.Sprintf("--%s=%s", k, v))
	}
	args = append(args, testBinary, "--")
	for k, v := range r.suiteFlags {
		args = append(args, fmt.Sprintf("--%s=%s", k, v))
	}

	log.Debugf("invoking ginkgo with followiong args: %s", args)

	// executes the command.
	// TODO: switch to an executor that supports timeout/cancellation
	cmd := kindexec.Command(ginkgoBinary, args...)
	kindexec.InheritOutput(cmd)
	err = cmd.Run()
	if err != nil {
		return errors.Wrap(err, "error running test")
	}

	fmt.Println(args)

	return nil
}
<commit_msg>fix misspelled words in runner.go<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/*
Package e2e implements support for running kubeadm e2e tests or kubernetes e2e test.

It takes care of building upstream test suites if necessary, and provides "sane" defaults
for simplifying test invokation.
*/
package e2e

import (
	"fmt"

	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"

	kindexec "sigs.k8s.io/kind/pkg/exec"
)

// Option is an Runner configuration option supplied to NewRunner
type Option func(*Runner)

// KubeRoot option sets the kubernetes checkout folder
func KubeRoot(kubeRoot string) Option {
	return func(r *Runner) {
		r.kubeRoot = kubeRoot
	}
}

// WithGinkgoFlags option sets flags for ginkgo
func WithGinkgoFlags(ginkgoFlags GinkgoFlags) Option {
	return func(r *Runner) {
		r.ginkgoFlags = ginkgoFlags
	}
}

// WithSuiteFlags option sets flags for the test program
func WithSuiteFlags(suiteFlags SuiteFlags) Option {
	return func(r *Runner) {
		r.suiteFlags = suiteFlags
	}
}

// Runner defines attributes for a Kubernetes artifact extractor
type Runner struct {
	testBinary     string
	makeBinaryGoal string
	kubeRoot       string
	ginkgoFlags    GinkgoFlags
	suiteFlags     SuiteFlags
}

// NewKubernetesTestRunner returns a new E2E (Kubernetes) test runner configured with the given options
func NewKubernetesTestRunner(options ...Option) (runner *Runner, err error) {
	return newTestRunner("e2e.test", "test/e2e/e2e.test", options...)
}

// NewKubeadmTestRunner returns a new E2E kubeadm test runner configured with the given options
func NewKubeadmTestRunner(options ...Option) (runner *Runner, err error) {
	return newTestRunner("e2e_kubeadm.test", "test/e2e_kubeadm/e2e_kubeadm.test", options...)
}

// newTestRunner returns a new test runner - using ginkgo and the given test binary - configured with the given options
func newTestRunner(testBinary, makeBinaryGoal string, options ...Option) (runner *Runner, err error) {
	runner = &Runner{
		testBinary:     testBinary,
		makeBinaryGoal: makeBinaryGoal,
	}

	// apply user options
	for _, option := range options {
		option(runner)
	}

	// sets kubeRoot if not provided by the user
	if runner.kubeRoot == "" {
		runner.kubeRoot, err = findKubeRoot()
		if err != nil {
			return nil, errors.Wrap(err, "")
		}
		log.Infof("using Kubernetes checkout in %s folder", runner.kubeRoot)
	}

	return runner, nil
}

// Run executes tests as defined by the selected runner options.
// it takes care of building ginkgo and upstream test suites if necessary,
func (r *Runner) Run() error {
	// find a ginkgo binary or build it if it not exists
	ginkgoBinary, err := getOrBuildBinary(r.kubeRoot, "ginkgo", "vendor/github.com/onsi/ginkgo/ginkgo")
	if err != nil {
		return err
	}

	// find the binary with the test suites to be executes or build it if it not exists
	testBinary, err := getOrBuildBinary(r.kubeRoot, r.testBinary, r.makeBinaryGoal)
	if err != nil {
		return err
	}

	// prepare args to be passed to ginkgo test runner:
	// ginkgo [ginkgo-flags] [test-suite-binary] -- [test-suite-flags]
	var args []string
	for k, v := range r.ginkgoFlags {
		args = append(args, fmt.Sprintf("--%s=%s", k, v))
	}
	args = append(args, testBinary, "--")
	for k, v := range r.suiteFlags {
		args = append(args, fmt.Sprintf("--%s=%s", k, v))
	}

	log.Debugf("invoking ginkgo with following args: %s", args)

	// executes the command.
	// TODO: switch to an executor that supports timeout/cancellation
	cmd := kindexec.Command(ginkgoBinary, args...)
	kindexec.InheritOutput(cmd)
	err = cmd.Run()
	if err != nil {
		return errors.Wrap(err, "error running test")
	}

	fmt.Println(args)

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// On-cloud data structures

package arq

import (
	"fmt"
	"os"
	"time"
)

// plist data structures

type computerInfo struct {
	UserName string `plist:"userName"`
	ComputerName string `plist:"computerName"`
}

type folderInfo struct {
	BucketUUID string
	BucketName string
	ComputerUUID string
	LocalPath string
	LocalMountPoint string
	// don't care about IgnoredRelativePaths or Excludes
}

type reflog struct {
	OldHeadSHA1 string `plist:"oldHeadSHA1"`
	NewHeadSHA1 string `plist:"newHeadSHA1"`
}

// binary data structures

type score [20]byte

type sscore struct {
	Score      score `arq:"HexScore"`
	StretchKey bool  // v4+
}

type tag string

type commit struct {
	Tag tag                 `arq:"CommitV005"`
	Author              string
	Comment             string
	ParentCommits       []sscore
	Tree                sscore
	Location            string
	MergeCommonAncestor sscore
	CreateTime          time.Time
	Failed              []failed // v3+
	BucketXML           []byte   // v5+
}

type tree struct {
	Tag tag           `arq:"TreeV015"`
	CompressXattr bool
	CompressACL   bool
	Xattr         sscore
	XattrSize     uint64
	ACL           sscore
	Uid           int32
	Gid           int32
	Mode          int32
	Mtime         unixTime
	Flags         int64
	FinderFlags   int32
	XFinderFlags  int32
	StDev         int32
	StIno         int32
	StNlink       uint32
	StRdev        int32
	Ctime         unixTime
	StBlocks      int64
	StBlksize     uint32
	AggrSize      uint64
	WTF           [16]byte   // what is this?
	Nodes         []*nameNode `arq:"count32"`
}

type nameNode struct {
	Name string
	Node node
}

type node struct {
	IsTree            bool
	CompressData      bool
	CompressXattr     bool
	CompressACL       bool
	Blob              []sscore `arq:"count32"`
	UncompressedSize  uint64
	Thumbnail         sscore
	Preview           sscore
	Xattr             sscore
	XattrSize         uint64
	ACL               sscore
	Uid               int32
	Gid               int32
	Mode              int32
	Mtime             unixTime
	Flags             int64
	FinderFlags       int32
	XFinderFlags      int32
	FinderFileType    string
	FinderFileCreator string
	IsExtHidden       bool
	StDev             int32
	StIno             int32
	StNlink           uint32
	StRdev            int32
	Ctime             unixTime
	CreateTime        unixTime
	StBlocks          int64
	StBlksize         uint32
}

func fileMode(m int32) os.FileMode {
	const (
		// Darwin file mode.
 		S_IFBLK                     = 0x6000
 		S_IFCHR                     = 0x2000
 		S_IFDIR                     = 0x4000
 		S_IFIFO                     = 0x1000
 		S_IFLNK                     = 0xa000
 		S_IFMT                      = 0xf000
 		S_IFREG                     = 0x8000
 		S_IFSOCK                    = 0xc000
 		S_IFWHT                     = 0xe000
 		S_ISGID                     = 0x400
 		S_ISTXT                     = 0x200
 		S_ISUID                     = 0x800
 		S_ISVTX                     = 0x200
	)
	mode := os.FileMode(m & 0777)
	switch m & S_IFMT {
	case S_IFBLK, S_IFWHT:
		mode |= os.ModeDevice
	case S_IFCHR:
		mode |= os.ModeDevice | os.ModeCharDevice
	case S_IFDIR:
		mode |= os.ModeDir
	case S_IFIFO:
		mode |= os.ModeNamedPipe
	case S_IFLNK:
		mode |= os.ModeSymlink
	case S_IFREG:
		// nothing to do
	case S_IFSOCK:
		mode |= os.ModeSocket
	}
	if m&S_ISGID != 0 {
		mode |= os.ModeSetgid
	}
	if m&S_ISUID != 0 {
		mode |= os.ModeSetuid
	}
	if m&S_ISVTX != 0 {
		mode |= os.ModeSticky
	}
	return mode
}

type unixTime struct {
	Sec  int64
	Nsec int64
}

func (t *unixTime) Time() time.Time {
	return time.Unix(t.Sec, t.Nsec)
}

type failed struct {
	Path  string
	Error string
}

type ientry struct {
	File   string
	Offset int64
	Size   int64
	Score  score
}

func (s score) Equal(t score) bool {
	for i := range s {
		if s[i] != t[i] {
			return false
		}
	}
	return true
}

func (s score) String() string {
	return fmt.Sprintf("%x", s[:])
}

func binaryScore(b []byte) score {
	if len(b) < 20 {
		panic("BinaryScore: not enough data")
	}
	var sc score
	copy(sc[:], b)
	return sc
}

func hexScore(b string) score {
	if len(b) < 40 {
		panic("HexScore: not enough data")
	}
	var sc score
	for i := 0; i < 40; i++ {
		ch := b[i]
		if '0' <= ch && ch <= '9' {
			ch -= '0'
		} else if 'a' <= ch && ch <= 'f' {
			ch -= 'a' - 10
		} else {
			panic("HexScore: invalid lower hex digit")
		}
		if i%2 == 0 {
			ch <<= 4
		}
		sc[i/2] |= ch
	}
	return sc
}

func (ss sscore) String() string {
	str := ss.Score.String()
	if ss.StretchKey {
		str += "Y"
	}
	return str
}
<commit_msg>arq: mystery solved<commit_after>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// On-cloud data structures

package arq

import (
	"fmt"
	"os"
	"time"
)

// plist data structures

type computerInfo struct {
	UserName string `plist:"userName"`
	ComputerName string `plist:"computerName"`
}

type folderInfo struct {
	BucketUUID string
	BucketName string
	ComputerUUID string
	LocalPath string
	LocalMountPoint string
	// don't care about IgnoredRelativePaths or Excludes
}

type reflog struct {
	OldHeadSHA1 string `plist:"oldHeadSHA1"`
	NewHeadSHA1 string `plist:"newHeadSHA1"`
}

// binary data structures

type score [20]byte

type sscore struct {
	Score      score `arq:"HexScore"`
	StretchKey bool  // v4+
}

type tag string

type commit struct {
	Tag tag                 `arq:"CommitV005"`
	Author              string
	Comment             string
	ParentCommits       []sscore
	Tree                sscore
	Location            string
	MergeCommonAncestor sscore
	CreateTime          time.Time
	Failed              []failed // v3+
	BucketXML           []byte   // v5+
}

type tree struct {
	Tag tag           `arq:"TreeV015"`
	CompressXattr bool
	CompressACL   bool
	Xattr         sscore
	XattrSize     uint64
	ACL           sscore
	Uid           int32
	Gid           int32
	Mode          int32
	Mtime         unixTime
	Flags         int64
	FinderFlags   int32
	XFinderFlags  int32
	StDev         int32
	StIno         int32
	StNlink       uint32
	StRdev        int32
	Ctime         unixTime
	StBlocks      int64
	StBlksize     uint32
	AggrSize      uint64
	Crtime unixTime
	Nodes         []*nameNode `arq:"count32"`
}

type nameNode struct {
	Name string
	Node node
}

type node struct {
	IsTree            bool
	CompressData      bool
	CompressXattr     bool
	CompressACL       bool
	Blob              []sscore `arq:"count32"`
	UncompressedSize  uint64
	Thumbnail         sscore
	Preview           sscore
	Xattr             sscore
	XattrSize         uint64
	ACL               sscore
	Uid               int32
	Gid               int32
	Mode              int32
	Mtime             unixTime
	Flags             int64
	FinderFlags       int32
	XFinderFlags      int32
	FinderFileType    string
	FinderFileCreator string
	IsExtHidden       bool
	StDev             int32
	StIno             int32
	StNlink           uint32
	StRdev            int32
	Ctime             unixTime
	CreateTime        unixTime
	StBlocks          int64
	StBlksize         uint32
}

func fileMode(m int32) os.FileMode {
	const (
		// Darwin file mode.
 		S_IFBLK                     = 0x6000
 		S_IFCHR                     = 0x2000
 		S_IFDIR                     = 0x4000
 		S_IFIFO                     = 0x1000
 		S_IFLNK                     = 0xa000
 		S_IFMT                      = 0xf000
 		S_IFREG                     = 0x8000
 		S_IFSOCK                    = 0xc000
 		S_IFWHT                     = 0xe000
 		S_ISGID                     = 0x400
 		S_ISTXT                     = 0x200
 		S_ISUID                     = 0x800
 		S_ISVTX                     = 0x200
	)
	mode := os.FileMode(m & 0777)
	switch m & S_IFMT {
	case S_IFBLK, S_IFWHT:
		mode |= os.ModeDevice
	case S_IFCHR:
		mode |= os.ModeDevice | os.ModeCharDevice
	case S_IFDIR:
		mode |= os.ModeDir
	case S_IFIFO:
		mode |= os.ModeNamedPipe
	case S_IFLNK:
		mode |= os.ModeSymlink
	case S_IFREG:
		// nothing to do
	case S_IFSOCK:
		mode |= os.ModeSocket
	}
	if m&S_ISGID != 0 {
		mode |= os.ModeSetgid
	}
	if m&S_ISUID != 0 {
		mode |= os.ModeSetuid
	}
	if m&S_ISVTX != 0 {
		mode |= os.ModeSticky
	}
	return mode
}

type unixTime struct {
	Sec  int64
	Nsec int64
}

func (t *unixTime) Time() time.Time {
	return time.Unix(t.Sec, t.Nsec)
}

type failed struct {
	Path  string
	Error string
}

type ientry struct {
	File   string
	Offset int64
	Size   int64
	Score  score
}

func (s score) Equal(t score) bool {
	for i := range s {
		if s[i] != t[i] {
			return false
		}
	}
	return true
}

func (s score) String() string {
	return fmt.Sprintf("%x", s[:])
}

func binaryScore(b []byte) score {
	if len(b) < 20 {
		panic("BinaryScore: not enough data")
	}
	var sc score
	copy(sc[:], b)
	return sc
}

func hexScore(b string) score {
	if len(b) < 40 {
		panic("HexScore: not enough data")
	}
	var sc score
	for i := 0; i < 40; i++ {
		ch := b[i]
		if '0' <= ch && ch <= '9' {
			ch -= '0'
		} else if 'a' <= ch && ch <= 'f' {
			ch -= 'a' - 10
		} else {
			panic("HexScore: invalid lower hex digit")
		}
		if i%2 == 0 {
			ch <<= 4
		}
		sc[i/2] |= ch
	}
	return sc
}

func (ss sscore) String() string {
	str := ss.Score.String()
	if ss.StretchKey {
		str += "Y"
	}
	return str
}
<|endoftext|>


<commit_before>// Copyright 2012-2013 Joshua Tacoma. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package zmqutil

import (
	"log"
	"os"
	"strconv"
	"sync"
	"time"

	zmq "github.com/alecthomas/gozmq"
)

// A Event represents a set of events on a socket.
//
type Event struct {
	Socket *Socket        // socket on which events occurred
	Events zmq.PollEvents // bitmask of events that occurred
	Fault  error          // handlers may set this
}

// A SocketHandler acts on a *Event.
//
// When a SocketHandler returns an error to a poller, the poller will exit.  An
// instance of this interface is returned from *Poller.HandleFunc(), and can be
// passed to *Poller.Unhandle() to unsubscribe.
//
type SocketHandler interface {
	HandleEvent(*Event)
}

func NewMessageHandler(do func(*Event, [][]byte)) SocketHandler {
	return socketHandlerFunc{
		fun: func(e *Event) {
			for {
				m, err := e.Socket.RecvMultipart(zmq.DONTWAIT)
				if err != nil {
					break
				}
				do(e, m)
			}
		},
	}
}

type socketHandlerFunc struct {
	fun func(*Event)
}

func (h socketHandlerFunc) HandleEvent(e *Event) {
	h.fun(e)
}

// A Poller is a ZeroMQ poller running in a goroutine.
//
// The poller will respond to events on sockets by calling handlers that have
// been associated with those events on those sockets through Handle() and
// HandleFunc().
//
// Note: since a Socket is not thread-safe, a Socket being polled by a Poller
// should not be operated on outside the scope of a handler.
//
type Poller struct {
	items   []pollItem  // sockets with their channels
	closing bool        // true if poller is either stopping or stopped
	logger  *log.Logger // changes when SetVerbose is called
	locker  *sync.Mutex // synchronize access to Poller state
}

type pollItem struct {
	socket  *Socket        // socket to poll
	events  zmq.PollEvents // events to poll for
	handler SocketHandler  // func to call when events occur
}

// NewPoller creates a new poller.
//
func NewPoller(context *Context) *Poller {
	return &Poller{
		locker: &sync.Mutex{},
	}
}

// Handle adds a socket event handler to p.
//
func (p *Poller) Handle(s *Socket, e zmq.PollEvents, h SocketHandler) {
	p.locker.Lock()
	defer p.locker.Unlock()
	var exists bool
	for _, existing := range p.items {
		if existing.socket == s && existing.handler == h {
			existing.events = existing.events | e
			exists = true
		}
	}
	if !exists {
		p.items = append(p.items, pollItem{s, e, h})
	}
}

// HandleFunc adds a socket event handler to p.
//
func (p *Poller) HandleFunc(s *Socket, e zmq.PollEvents, h func(*Event)) SocketHandler {
	handler := &socketHandlerFunc{h}
	p.Handle(s, e, handler)
	return handler
}

// Unhandle removes a handler from p for the given socket and socket events.
//
// If there are no remaining handlers for any event on this socket, the socket
// itself will cease to be polled in p.
//
func (p *Poller) Unhandle(s *Socket, e zmq.PollEvents, h SocketHandler) {
	p.locker.Lock()
	defer p.locker.Unlock()
	index := -1
	for i, existing := range p.items {
		if existing.socket == s && existing.handler == h {
			existing.events = existing.events & ^e
			if existing.events == 0 {
				index = i
			}
		}
	}
	if index == len(p.items)-1 {
		p.items = p.items[0:index]
	} else if index >= 0 {
		p.items = append(p.items[0:index], p.items[index+1:len(p.items)]...)
	}
}

// SetVerbose enables (or disables) logging to os.Stdout.
//
func (p *Poller) SetVerbose(verbose bool) error {
	if verbose == (p.logger != nil) {
		return nil
	}
	p.logf("poller.verbose = %t", verbose)
	if verbose && p.logger == nil {
		p.logger = log.New(os.Stdout, "", log.Lmicroseconds)
	} else if !verbose {
		p.logger = nil
	}
	p.logf("poller.verbose = %t", p.logger != nil)
	return nil
}

func (p *Poller) logf(s string, args ...interface{}) {
	if p.logger != nil {
		if s[len(s)-1] != '\n' {
			s += "\n"
		}
		p.logger.Printf("[zmqutil] "+s, args...)
	}
}

// Run calls Poll until an error is returned; then it closes p.
//
func (p *Poller) Run() error {
	for !p.closing {
		if err := p.Poll(-1); err != nil {
			p.logf("poller: closing notification-receiving socket...")
			p.closing = true
			return err
		}
	}
	return nil
}

// Poll polls, with the specified timeout, all sockets for all events that have
// been registered with event handlers.
//
// A negative timeout means forever; otherwise, timeout wll be truncated to
// millisecond precision.
//
// Execution will halt and return first error encountered from polling
// or handling.
//
func (p *Poller) Poll(timeout time.Duration) (err error) {

	// This PollItems construction may become inefficient for large
	// numbers of handlers.
	baseItems := make(zmq.PollItems, 0, len(p.items))
	for _, item := range p.items {
		var exists bool
		for _, base := range baseItems {
			if base.Socket == item.socket.base {
				base.Events = base.Events | item.events
				exists = true
			}
		}
		if !exists {
			baseItems = append(baseItems, zmq.PollItem{
				Socket: item.socket.base,
				Events: item.events,
			})
		}
	}

	_, err = zmq.Poll(baseItems, timeout)

	// Possible errors returned from Poll() are: ETERM, meaning a
	// context was closed; EFAULT, meaning a mistake was made in
	// setting up the PollItems list; and EINTR, meaning a signal
	// was delivered before any events were available.  Here, we
	// treat all errors the same:
	if err != nil {
		return err
	}

	p.logf("poller: events detected.")

	// Check all other sockets, sending any available messages to
	// their associated channels:
	for _, base := range baseItems {
		event := Event{
			Events: base.REvents,
		}
		for _, item := range p.items {
			if item.socket.base == base.Socket && (item.events&base.REvents) != 0 {
				event.Socket = item.socket
				item.handler.HandleEvent(&event)
				if event.Fault != nil {
					return event.Fault
				}
			}
		}
	}

	return nil
}

// newPair returns a PUSH/PULL pair of inproc sockets.
func newPair(c *Context) (send *Socket, recv *Socket, err error) {
	send = c.NewSocket(zmq.PUSH)
	send.SetLinger(0)
	recv = c.NewSocket(zmq.PULL)
	send.SetLinger(0)
	addr := newInprocAddress()
	err = send.Bind(addr)
	if err != nil {
		return
	}
	err = recv.Connect(addr)
	if err != nil {
		return
	}
	return
}

// newInprocAddress returns a unique incproc address.
func newInprocAddress() string {
	inprocNextMutex.Lock()
	defer inprocNextMutex.Unlock()
	inprocNext += 1
	return "inproc://github.com/jtacoma/zmqutil/" + strconv.Itoa(inprocNext-1)
}

var inprocNext = 1
var inprocNextMutex sync.Mutex
<commit_msg>Rename SocketHandler to Handler.<commit_after>// Copyright 2012-2013 Joshua Tacoma. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package zmqutil

import (
	"log"
	"os"
	"strconv"
	"sync"
	"time"

	zmq "github.com/alecthomas/gozmq"
)

// A Event represents a set of events on a socket.
//
type Event struct {
	Socket *Socket        // socket on which events occurred
	Events zmq.PollEvents // bitmask of events that occurred
	Fault  error          // handlers may set this
}

// A Handler acts on a *Event.
//
// When a Handler returns an error to a poller, the poller will exit.  An
// instance of this interface is returned from *Poller.HandleFunc(), and can be
// passed to *Poller.Unhandle() to unsubscribe.
//
type Handler interface {
	HandleEvent(*Event)
}

func NewMessageHandler(do func(*Event, [][]byte)) Handler {
	return socketHandlerFunc{
		fun: func(e *Event) {
			for {
				m, err := e.Socket.RecvMultipart(zmq.DONTWAIT)
				if err != nil {
					break
				}
				do(e, m)
			}
		},
	}
}

type socketHandlerFunc struct {
	fun func(*Event)
}

func (h socketHandlerFunc) HandleEvent(e *Event) {
	h.fun(e)
}

// A Poller is a ZeroMQ poller running in a goroutine.
//
// The poller will respond to events on sockets by calling handlers that have
// been associated with those events on those sockets through Handle() and
// HandleFunc().
//
// Note: since a Socket is not thread-safe, a Socket being polled by a Poller
// should not be operated on outside the scope of a handler.
//
type Poller struct {
	items   []pollItem  // sockets with their channels
	closing bool        // true if poller is either stopping or stopped
	logger  *log.Logger // changes when SetVerbose is called
	locker  *sync.Mutex // synchronize access to Poller state
}

type pollItem struct {
	socket  *Socket        // socket to poll
	events  zmq.PollEvents // events to poll for
	handler Handler        // func to call when events occur
}

// NewPoller creates a new poller.
//
func NewPoller(context *Context) *Poller {
	return &Poller{
		locker: &sync.Mutex{},
	}
}

// Handle adds a socket event handler to p.
//
func (p *Poller) Handle(s *Socket, e zmq.PollEvents, h Handler) {
	p.locker.Lock()
	defer p.locker.Unlock()
	var exists bool
	for _, existing := range p.items {
		if existing.socket == s && existing.handler == h {
			existing.events = existing.events | e
			exists = true
		}
	}
	if !exists {
		p.items = append(p.items, pollItem{s, e, h})
	}
}

// HandleFunc adds a socket event handler to p.
//
func (p *Poller) HandleFunc(s *Socket, e zmq.PollEvents, h func(*Event)) Handler {
	handler := &socketHandlerFunc{h}
	p.Handle(s, e, handler)
	return handler
}

// Unhandle removes a handler from p for the given socket and socket events.
//
// If there are no remaining handlers for any event on this socket, the socket
// itself will cease to be polled in p.
//
func (p *Poller) Unhandle(s *Socket, e zmq.PollEvents, h Handler) {
	p.locker.Lock()
	defer p.locker.Unlock()
	index := -1
	for i, existing := range p.items {
		if existing.socket == s && existing.handler == h {
			existing.events = existing.events & ^e
			if existing.events == 0 {
				index = i
			}
		}
	}
	if index == len(p.items)-1 {
		p.items = p.items[0:index]
	} else if index >= 0 {
		p.items = append(p.items[0:index], p.items[index+1:len(p.items)]...)
	}
}

// SetVerbose enables (or disables) logging to os.Stdout.
//
func (p *Poller) SetVerbose(verbose bool) error {
	if verbose == (p.logger != nil) {
		return nil
	}
	p.logf("poller.verbose = %t", verbose)
	if verbose && p.logger == nil {
		p.logger = log.New(os.Stdout, "", log.Lmicroseconds)
	} else if !verbose {
		p.logger = nil
	}
	p.logf("poller.verbose = %t", p.logger != nil)
	return nil
}

func (p *Poller) logf(s string, args ...interface{}) {
	if p.logger != nil {
		if s[len(s)-1] != '\n' {
			s += "\n"
		}
		p.logger.Printf("[zmqutil] "+s, args...)
	}
}

// Run calls Poll until an error is returned; then it closes p.
//
func (p *Poller) Run() error {
	for !p.closing {
		if err := p.Poll(-1); err != nil {
			p.logf("poller: closing notification-receiving socket...")
			p.closing = true
			return err
		}
	}
	return nil
}

// Poll polls, with the specified timeout, all sockets for all events that have
// been registered with event handlers.
//
// A negative timeout means forever; otherwise, timeout wll be truncated to
// millisecond precision.
//
// Execution will halt and return first error encountered from polling
// or handling.
//
func (p *Poller) Poll(timeout time.Duration) (err error) {

	// This PollItems construction may become inefficient for large
	// numbers of handlers.
	baseItems := make(zmq.PollItems, 0, len(p.items))
	for _, item := range p.items {
		var exists bool
		for _, base := range baseItems {
			if base.Socket == item.socket.base {
				base.Events = base.Events | item.events
				exists = true
			}
		}
		if !exists {
			baseItems = append(baseItems, zmq.PollItem{
				Socket: item.socket.base,
				Events: item.events,
			})
		}
	}

	_, err = zmq.Poll(baseItems, timeout)

	// Possible errors returned from Poll() are: ETERM, meaning a
	// context was closed; EFAULT, meaning a mistake was made in
	// setting up the PollItems list; and EINTR, meaning a signal
	// was delivered before any events were available.  Here, we
	// treat all errors the same:
	if err != nil {
		return err
	}

	p.logf("poller: events detected.")

	// Check all other sockets, sending any available messages to
	// their associated channels:
	for _, base := range baseItems {
		event := Event{
			Events: base.REvents,
		}
		for _, item := range p.items {
			if item.socket.base == base.Socket && (item.events&base.REvents) != 0 {
				event.Socket = item.socket
				item.handler.HandleEvent(&event)
				if event.Fault != nil {
					return event.Fault
				}
			}
		}
	}

	return nil
}

// newPair returns a PUSH/PULL pair of inproc sockets.
func newPair(c *Context) (send *Socket, recv *Socket, err error) {
	send = c.NewSocket(zmq.PUSH)
	send.SetLinger(0)
	recv = c.NewSocket(zmq.PULL)
	send.SetLinger(0)
	addr := newInprocAddress()
	err = send.Bind(addr)
	if err != nil {
		return
	}
	err = recv.Connect(addr)
	if err != nil {
		return
	}
	return
}

// newInprocAddress returns a unique incproc address.
func newInprocAddress() string {
	inprocNextMutex.Lock()
	defer inprocNextMutex.Unlock()
	inprocNext += 1
	return "inproc://github.com/jtacoma/zmqutil/" + strconv.Itoa(inprocNext-1)
}

var inprocNext = 1
var inprocNextMutex sync.Mutex
<|endoftext|>


<commit_before>package googlecompute

import (
	"crypto/rand"
	"crypto/x509"
	"encoding/pem"
	"io/ioutil"
	"os"
	"testing"
)

func testPrivateKeyFile(t *testing.T) string {
	tf, err := ioutil.TempFile("", "packer")
	if err != nil {
		t.Fatalf("bad: %s", err)
	}
	defer tf.Close()

	b := &pem.Block{
		Type:  "RSA PRIVATE KEY",
		Bytes: []byte("what"),
	}

	if err := pem.Encode(tf, b); err != nil {
		t.Fatalf("err: %s", err)
	}

	return tf.Name()
}

func TestProcesssPrivateKeyFile(t *testing.T) {
	path := testPrivateKeyFile(t)
	defer os.Remove(path)

	data, err := processPrivateKeyFile(path, "")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if len(data) <= 0 {
		t.Fatalf("bad: %#v", data)
	}
}

func TestProcessPrivateKeyFile_encrypted(t *testing.T) {
	// Encrypt the file
	b, err := x509.EncryptPEMBlock(rand.Reader,
		"RSA PRIVATE KEY",
		[]byte("what"),
		[]byte("password"),
		x509.PEMCipherAES128)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	tf, err := ioutil.TempFile("", "packer")
	if err != nil {
		t.Fatalf("bad: %s", err)
	}
	defer os.Remove(tf.Name())

	err = pem.Encode(tf, b)
	tf.Close()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	path := tf.Name()

	// Should have an error with a bad password
	if _, err := processPrivateKeyFile(path, "bad"); err == nil {
		t.Fatal("should error")
	}

	if _, err := processPrivateKeyFile(path, "password"); err != nil {
		t.Fatalf("bad: %s", err)
	}
}
<commit_msg>fix intermitent TestProcessPrivateKeyFile_encrypted failure<commit_after>package googlecompute

import (
	"crypto/rand"
	"crypto/x509"
	"encoding/pem"
	"io/ioutil"
	"os"
	"testing"
)

func testPrivateKeyFile(t *testing.T) string {
	tf, err := ioutil.TempFile("", "packer")
	if err != nil {
		t.Fatalf("bad: %s", err)
	}
	defer tf.Close()

	b := &pem.Block{
		Type:  "RSA PRIVATE KEY",
		Bytes: []byte("what"),
	}

	if err := pem.Encode(tf, b); err != nil {
		t.Fatalf("err: %s", err)
	}

	return tf.Name()
}

func TestProcesssPrivateKeyFile(t *testing.T) {
	path := testPrivateKeyFile(t)
	defer os.Remove(path)

	data, err := processPrivateKeyFile(path, "")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if len(data) <= 0 {
		t.Fatalf("bad: %#v", data)
	}
}

func TestProcessPrivateKeyFile_encrypted(t *testing.T) {
	data := []byte("what")
	// Encrypt the file
	b, err := x509.EncryptPEMBlock(rand.Reader,
		"RSA PRIVATE KEY",
		data,
		[]byte("password"),
		x509.PEMCipherAES128)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	tf, err := ioutil.TempFile("", "packer")
	if err != nil {
		t.Fatalf("bad: %s", err)
	}
	defer os.Remove(tf.Name())

	err = pem.Encode(tf, b)
	tf.Close()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	path := tf.Name()

	// Should have an error with a bad password
	if b, err := processPrivateKeyFile(path, "bad"); err == nil {
		if string(b) == string(data) {
			t.Fatal("should error & be different")
		}
		t.Logf(`Decrypt was successfull but the body was wrong.`)
		// Because of deficiencies
		// in the encrypted-PEM format, it's not always possible to detect an incorrect
		// password. In these cases no error will be returned but the decrypted DER
		// bytes will be random noise.
		// https://github.com/golang/go/blob/50bd1c4d4eb4fac8ddeb5f063c099daccfb71b26/src/crypto/x509/pem_decrypt.go#L112-L114
	}

	if _, err := processPrivateKeyFile(path, "password"); err != nil {
		t.Fatalf("bad: %s", err)
	}
}
<|endoftext|>


<commit_before>package vmware

import (
	"fmt"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
	"log"
	"time"
)

type stepPrepareOutputDir struct {
	dir OutputDir
}

func (s *stepPrepareOutputDir) Run(state multistep.StateBag) multistep.StepAction {
	config := state.Get("config").(*config)
	ui := state.Get("ui").(packer.Ui)

	dir := s.outputDir(state)
	dir.SetOutputDir(config.OutputDir)

	exists, err := dir.DirExists()
	if err != nil {
		state.Put("error", err)
		return multistep.ActionHalt
	}

	if exists {
		if config.PackerForce {
			ui.Say("Deleting previous output directory...")
			dir.RemoveAll()
		} else {
			state.Put("error", fmt.Errorf("Output directory '%s' already exists.", config.OutputDir))
			return multistep.ActionHalt
		}
	}

	if err := dir.MkdirAll(); err != nil {
		state.Put("error", err)
		return multistep.ActionHalt
	}

	s.dir = dir

	state.Put("dir", dir)

	return multistep.ActionContinue
}

func (s *stepPrepareOutputDir) Cleanup(state multistep.StateBag) {
	_, cancelled := state.GetOk(multistep.StateCancelled)
	_, halted := state.GetOk(multistep.StateHalted)

	if cancelled || halted {
		ui := state.Get("ui").(packer.Ui)

		if s.dir != nil {
			ui.Say("Deleting output directory...")
			for i := 0; i < 5; i++ {
				err := s.dir.RemoveAll()
				if err == nil {
					break
				}

				log.Printf("Error removing output dir: %s", err)
				time.Sleep(2 * time.Second)
			}
		}
	}
}

func (s *stepPrepareOutputDir) outputDir(state multistep.StateBag) (dir OutputDir) {
	driver := state.Get("driver").(Driver)

	switch d := driver.(type) {
	case OutputDir:
		log.Printf("Using driver as the OutputDir implementation")
		dir = d
	default:
		log.Printf("Using localOutputDir implementation")
		dir = new(localOutputDir)
	}

	return
}
<commit_msg>builder/vmware: style<commit_after>package vmware

import (
	"fmt"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
	"log"
	"time"
)

type stepPrepareOutputDir struct {
	dir OutputDir
}

func (s *stepPrepareOutputDir) Run(state multistep.StateBag) multistep.StepAction {
	config := state.Get("config").(*config)
	ui := state.Get("ui").(packer.Ui)

	dir := s.outputDir(state)
	dir.SetOutputDir(config.OutputDir)

	exists, err := dir.DirExists()
	if err != nil {
		state.Put("error", err)
		return multistep.ActionHalt
	}

	if exists {
		if config.PackerForce {
			ui.Say("Deleting previous output directory...")
			dir.RemoveAll()
		} else {
			state.Put("error", fmt.Errorf(
				"Output directory '%s' already exists.", config.OutputDir))
			return multistep.ActionHalt
		}
	}

	if err := dir.MkdirAll(); err != nil {
		state.Put("error", err)
		return multistep.ActionHalt
	}

	s.dir = dir
	state.Put("dir", dir)
	return multistep.ActionContinue
}

func (s *stepPrepareOutputDir) Cleanup(state multistep.StateBag) {
	_, cancelled := state.GetOk(multistep.StateCancelled)
	_, halted := state.GetOk(multistep.StateHalted)

	if cancelled || halted {
		ui := state.Get("ui").(packer.Ui)

		if s.dir != nil {
			ui.Say("Deleting output directory...")
			for i := 0; i < 5; i++ {
				err := s.dir.RemoveAll()
				if err == nil {
					break
				}

				log.Printf("Error removing output dir: %s", err)
				time.Sleep(2 * time.Second)
			}
		}
	}
}

func (s *stepPrepareOutputDir) outputDir(state multistep.StateBag) (dir OutputDir) {
	driver := state.Get("driver").(Driver)

	switch d := driver.(type) {
	case OutputDir:
		log.Printf("Using driver as the OutputDir implementation")
		dir = d
	default:
		log.Printf("Using localOutputDir implementation")
		dir = new(localOutputDir)
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tester

import (
	"context"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/functional/rpcpb"

	"go.uber.org/zap"
)

//  1. Assume node C is the current leader with most up-to-date data.
//  2. Download snapshot from node C, before destroying node A and B.
//  3. Destroy node A and B, and make the whole cluster inoperable.
//  4. Now node C cannot operate either.
//  5. SIGTERM node C and remove its data directories.
//  6. Restore a new seed member from node C's latest snapshot file.
//  7. Add another member to establish 2-node cluster.
//  8. Add another member to establish 3-node cluster.

type fetchSnapshotAndFailureQuorum struct {
	desc        string
	failureCase rpcpb.FailureCase
	injected    map[int]struct{}
	snapshotted int
}

func (f *fetchSnapshotAndFailureQuorum) Inject(clus *Cluster) error {
	//  1. Assume node C is the current leader with most up-to-date data.
	lead, err := clus.GetLeader()
	if err != nil {
		return err
	}
	f.snapshotted = lead

	//  2. Download snapshot from node C, before destroying node A and B.
	clus.lg.Info(
		"install snapshot on leader node START",
		zap.String("target-endpoint", clus.Members[lead].EtcdClientEndpoint),
		zap.Error(err),
	)
	var resp *rpcpb.Response
	if resp == nil || err != nil {
		resp, err = clus.sendOpWithResp(lead, rpcpb.Operation_FETCH_SNAPSHOT)
		clus.lg.Info(
			"install snapshot on leader node END",
			zap.String("target-endpoint", clus.Members[lead].EtcdClientEndpoint),
			zap.Error(err),
		)
		return err
	}
	resp, err = clus.sendOpWithResp(lead, rpcpb.Operation_FETCH_SNAPSHOT)
	clus.lg.Info(
		"install snapshot on leader node END",
		zap.String("target-endpoint", clus.Members[lead].EtcdClientEndpoint),
		zap.String("member-name", resp.SnapshotInfo.MemberName),
		zap.Strings("member-client-urls", resp.SnapshotInfo.MemberClientURLs),
		zap.String("snapshot-path", resp.SnapshotInfo.SnapshotPath),
		zap.String("snapshot-file-size", resp.SnapshotInfo.SnapshotFileSize),
		zap.String("snapshot-total-size", resp.SnapshotInfo.SnapshotTotalSize),
		zap.Int64("snapshot-total-key", resp.SnapshotInfo.SnapshotTotalKey),
		zap.Int64("snapshot-hash", resp.SnapshotInfo.SnapshotHash),
		zap.Int64("snapshot-revision", resp.SnapshotInfo.SnapshotRevision),
		zap.String("took", resp.SnapshotInfo.Took),
		zap.Error(err),
	)
	if err != nil {
		return err
	}

	cli, err := clus.Members[lead].CreateEtcdClient()
	if err != nil {
		return err
	}
	defer cli.Close()
	var mresp *clientv3.MemberListResponse
	mresp, err = cli.MemberList(context.Background())
	mss := []string{}
	if err == nil && mresp != nil {
		mss = describeMembers(mresp)
	}
	clus.lg.Info(
		"member list before disastrous machine failure",
		zap.String("request-to", clus.Members[lead].EtcdClientEndpoint),
		zap.Strings("members", mss),
		zap.Error(err),
	)
	if err != nil {
		return err
	}

	//  3. Destroy node A and B, and make the whole cluster inoperable.
	for {
		f.injected = pickQuorum(len(clus.Members))
		if _, ok := f.injected[lead]; !ok {
			break
		}
	}

	return nil
}

func (f *fetchSnapshotAndFailureQuorum) Recover(clus *Cluster) error {
	// for idx := range f.injected {
	// 	if err := f.recoverMember(clus, idx); err != nil {
	// 		return err
	// 	}
	// }
	return nil
}

func (f *fetchSnapshotAndFailureQuorum) Desc() string {
	if f.desc != "" {
		return f.desc
	}
	return f.failureCase.String()
}

func (f *fetchSnapshotAndFailureQuorum) FailureCase() rpcpb.FailureCase {
	return f.failureCase
}

func new_FailureCase_SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH(clus *Cluster) Failure {
	f := &fetchSnapshotAndFailureQuorum{
		failureCase: rpcpb.FailureCase_SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH,
		injected:    make(map[int]struct{}),
		snapshotted: -1,
	}
	return &failureDelay{
		Failure:       f,
		delayDuration: clus.GetFailureDelayDuration(),
	}
}
<commit_msg>functional/tester: implement "fetchSnapshotAndFailureQuorum"<commit_after>// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tester

import (
	"context"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/functional/rpcpb"

	"go.uber.org/zap"
)

type fetchSnapshotAndFailureQuorum struct {
	desc        string
	failureCase rpcpb.FailureCase
	injected    map[int]struct{}
	snapshotted int
}

func (f *fetchSnapshotAndFailureQuorum) Inject(clus *Cluster) error {
	//  1. Assume node C is the current leader with most up-to-date data.
	lead, err := clus.GetLeader()
	if err != nil {
		return err
	}
	f.snapshotted = lead

	//  2. Download snapshot from node C, before destroying node A and B.
	clus.lg.Info(
		"install snapshot on leader node START",
		zap.String("target-endpoint", clus.Members[lead].EtcdClientEndpoint),
		zap.Error(err),
	)
	var resp *rpcpb.Response
	if resp == nil || err != nil {
		resp, err = clus.sendOpWithResp(lead, rpcpb.Operation_FETCH_SNAPSHOT)
		clus.lg.Info(
			"install snapshot on leader node END",
			zap.String("target-endpoint", clus.Members[lead].EtcdClientEndpoint),
			zap.Error(err),
		)
		return err
	}
	resp, err = clus.sendOpWithResp(lead, rpcpb.Operation_FETCH_SNAPSHOT)
	clus.lg.Info(
		"install snapshot on leader node END",
		zap.String("target-endpoint", clus.Members[lead].EtcdClientEndpoint),
		zap.String("member-name", resp.SnapshotInfo.MemberName),
		zap.Strings("member-client-urls", resp.SnapshotInfo.MemberClientURLs),
		zap.String("snapshot-path", resp.SnapshotInfo.SnapshotPath),
		zap.String("snapshot-file-size", resp.SnapshotInfo.SnapshotFileSize),
		zap.String("snapshot-total-size", resp.SnapshotInfo.SnapshotTotalSize),
		zap.Int64("snapshot-total-key", resp.SnapshotInfo.SnapshotTotalKey),
		zap.Int64("snapshot-hash", resp.SnapshotInfo.SnapshotHash),
		zap.Int64("snapshot-revision", resp.SnapshotInfo.SnapshotRevision),
		zap.String("took", resp.SnapshotInfo.Took),
		zap.Error(err),
	)
	if err != nil {
		return err
	}

	leaderc, err := clus.Members[lead].CreateEtcdClient()
	if err != nil {
		return err
	}
	defer leaderc.Close()
	var mresp *clientv3.MemberListResponse
	mresp, err = leaderc.MemberList(context.Background())
	mss := []string{}
	if err == nil && mresp != nil {
		mss = describeMembers(mresp)
	}
	clus.lg.Info(
		"member list before disastrous machine failure",
		zap.String("request-to", clus.Members[lead].EtcdClientEndpoint),
		zap.Strings("members", mss),
		zap.Error(err),
	)
	if err != nil {
		return err
	}

	// simulate real life; machine failures may happen
	// after some time since last snapshot save
	time.Sleep(time.Second)

	//  3. Destroy node A and B, and make the whole cluster inoperable.
	for {
		f.injected = pickQuorum(len(clus.Members))
		if _, ok := f.injected[lead]; !ok {
			break
		}
	}
	for idx := range f.injected {
		clus.lg.Info(
			"disastrous machine failure to quorum START",
			zap.String("target-endpoint", clus.Members[idx].EtcdClientEndpoint),
			zap.Error(err),
		)
		err = clus.sendOp(idx, rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA)
		clus.lg.Info(
			"disastrous machine failure to quorum END",
			zap.String("target-endpoint", clus.Members[idx].EtcdClientEndpoint),
			zap.Error(err),
		)
		if err != nil {
			return err
		}
	}

	//  4. Now node C cannot operate either.
	//  5. SIGTERM node C and remove its data directories.
	clus.lg.Info(
		"disastrous machine failure to old leader START",
		zap.String("target-endpoint", clus.Members[lead].EtcdClientEndpoint),
		zap.Error(err),
	)
	err = clus.sendOp(lead, rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA)
	clus.lg.Info(
		"disastrous machine failure to old leader END",
		zap.String("target-endpoint", clus.Members[lead].EtcdClientEndpoint),
		zap.Error(err),
	)
	return err
}

func (f *fetchSnapshotAndFailureQuorum) Recover(clus *Cluster) error {
	//  6. Restore a new seed member from node C's latest snapshot file.

	//  7. Add another member to establish 2-node cluster.

	//  8. Add another member to establish 3-node cluster.

	// for idx := range f.injected {
	// 	if err := f.recoverMember(clus, idx); err != nil {
	// 		return err
	// 	}
	// }
	return nil
}

func (f *fetchSnapshotAndFailureQuorum) Desc() string {
	if f.desc != "" {
		return f.desc
	}
	return f.failureCase.String()
}

func (f *fetchSnapshotAndFailureQuorum) FailureCase() rpcpb.FailureCase {
	return f.failureCase
}

func new_FailureCase_SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH(clus *Cluster) Failure {
	f := &fetchSnapshotAndFailureQuorum{
		failureCase: rpcpb.FailureCase_SIGQUIT_AND_REMOVE_QUORUM_AND_RESTORE_LEADER_SNAPSHOT_FROM_SCRATCH,
		injected:    make(map[int]struct{}),
		snapshotted: -1,
	}
	// simulate real life; machine replacements may happen
	// after some time since disaster
	return &failureDelay{
		Failure:       f,
		delayDuration: clus.GetFailureDelayDuration(),
	}
}
<|endoftext|>


<commit_before>package e2e

import (
	"github.com/ohsu-comp-bio/funnel/proto/tes"
	"testing"
	"time"
)

func TestHelloWorld(t *testing.T) {
	id := run(`
    --cmd 'echo hello world'
  `)
	task := wait(id)

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout")
	}
}

func TestGetTaskView(t *testing.T) {
	var err error
	var task *tes.Task

	id := run(`
    --cmd 'echo hello world'
    --name 'foo'
  `)
	wait(id)

	task = getView(id, tes.TaskView_MINIMAL)

	if task.Id != id {
		t.Fatal("expected task ID in minimal view")
	}
	if task.State != tes.State_COMPLETE {
		t.Fatal("expected complete state")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	task = getView(id, tes.TaskView_BASIC)

	if task.Name != "foo" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	task = getView(id, tes.TaskView_FULL)

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}

	// test http proxy
	task, err = hcli.GetTask(id, "MINIMAL")
	if err != nil {
		t.Fatal(err)
	}

	if task.Id != id {
		t.Fatal("expected task ID in minimal view")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	task, err = hcli.GetTask(id, "BASIC")
	if err != nil {
		t.Fatal(err)
	}

	if task.Name != "foo" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	task, err = hcli.GetTask(id, "FULL")
	if err != nil {
		t.Fatal(err)
	}

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}
}

// TODO this is a bit hacky for now because we're reusing the same
//      server + DB for all the e2e tests, so ListTasks gets the
//      results of all of those. It works for the moment, but
//      should probably run against a clean environment.
func TestListTaskView(t *testing.T) {
	var tasks []*tes.Task
	var task *tes.Task
	var err error

	id := run(`
    --cmd 'echo hello world'
    --name 'foo'
  `)
	wait(id)

	tasks = listView(tes.TaskView_MINIMAL)
	task = tasks[0]

	if task.Id == "" {
		t.Fatal("expected task ID in minimal view")
	}
	if task.State == tes.State_UNKNOWN {
		t.Fatal("expected complete state")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	tasks = listView(tes.TaskView_BASIC)
	task = tasks[0]

	if task.Name == "" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	tasks = listView(tes.TaskView_FULL)
	task = tasks[0]

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}

	// test http proxy
	var r *tes.ListTasksResponse
	r, err = hcli.ListTasks("MINIMAL")
	if err != nil {
		t.Fatal(err)
	}
	task = r.Tasks[0]

	if task.Id == "" {
		t.Fatal("expected task ID in minimal view")
	}
	if task.State == tes.State_UNKNOWN {
		t.Fatal("expected complete state")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	r, err = hcli.ListTasks("BASIC")
	if err != nil {
		t.Fatal(err)
	}
	task = r.Tasks[0]

	if task.Name == "" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	r, err = hcli.ListTasks("FULL")
	if err != nil {
		t.Fatal(err)
	}
	task = r.Tasks[0]

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}
}

// Test that the streaming logs pick up a single character.
// This ensures that the streaming works even when a small
// amount of logs are written (which was once a bug).
func TestSingleCharLog(t *testing.T) {
	id := run(`
    --cmd "sh -c 'echo a; sleep 100'"
  `)
	waitForRunning(id)
	time.Sleep(time.Second * 2)
	task := get(id)
	if task.Logs[0].Logs[0].Stdout != "a\n" {
		t.Fatal("Missing logs")
	}
	cancel(id)
}

// Test that port mappings are being logged.
/* TODO need ports in funnel run
func TestPortLog(t *testing.T) {
  id := run(`
    --cmd 'echo start'
    --cmd 'sleep 10'
  `)
  waitForExec(id, 2)
  task := get(id)
  if task.Logs[0].Logs[0].Ports[0].Host != 5000 {
    t.Fatal("Unexpected port logs")
  }
}
*/

// Test that a completed task cannot change state.
func TestCompleteStateImmutable(t *testing.T) {
	id := run(`
    --cmd 'echo hello'
  `)
	wait(id)
	err := cancel(id)
	if err == nil {
		t.Fatal("expected error")
	}
	task := get(id)
	if task.State != tes.State_COMPLETE {
		t.Fatal("Unexpected state")
	}
}

// Test canceling a task
func TestCancel(t *testing.T) {
	id := run(`
    --cmd 'echo start'
    --cmd 'sleep 1000'
    --cmd 'echo never'
  `)
	waitForExec(id, 1)
	cancel(id)
	// TODO docker details and container ID are very Funnel specific
	//      how could we generalize this to be reusued as TES conformance?
	waitForDockerDestroy(id + "-0")
	task := get(id)
	if task.State != tes.State_CANCELED {
		t.Fatal("Unexpected state")
	}
}

// The task executor logs list should only include entries for steps that
// have been started or completed, i.e. steps that have yet to be started
// won't show up in Task.Logs[0].Logs
func TestExecutorLogLength(t *testing.T) {
	id := run(`
    --cmd 'echo first'
    --cmd 'sleep 10'
    --cmd 'echo done'
  `)
	waitForExec(id, 2)
	task := get(id)
	cancel(id)
	if len(task.Logs[0].Logs) != 2 {
		t.Fatal("Unexpected executor log count")
	}
}

// There was a bug + fix where the task was being marked complete after
// the first step completed, but the correct behavior is to mark the
// task complete after *all* steps have completed.
func TestMarkCompleteBug(t *testing.T) {
	id := run(`
    --cmd 'echo step 1'
    --cmd 'sleep 100'
  `)
	waitForExec(id, 2)
	task := get(id)
	if task.State != tes.State_RUNNING {
		t.Fatal("Unexpected task state")
	}
}
<commit_msg>waitForRunning in TestMarkCompleteBug<commit_after>package e2e

import (
	"github.com/ohsu-comp-bio/funnel/proto/tes"
	"testing"
	"time"
)

func TestHelloWorld(t *testing.T) {
	id := run(`
    --cmd 'echo hello world'
  `)
	task := wait(id)

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout")
	}
}

func TestGetTaskView(t *testing.T) {
	var err error
	var task *tes.Task

	id := run(`
    --cmd 'echo hello world'
    --name 'foo'
  `)
	wait(id)

	task = getView(id, tes.TaskView_MINIMAL)

	if task.Id != id {
		t.Fatal("expected task ID in minimal view")
	}
	if task.State != tes.State_COMPLETE {
		t.Fatal("expected complete state")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	task = getView(id, tes.TaskView_BASIC)

	if task.Name != "foo" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	task = getView(id, tes.TaskView_FULL)

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}

	// test http proxy
	task, err = hcli.GetTask(id, "MINIMAL")
	if err != nil {
		t.Fatal(err)
	}

	if task.Id != id {
		t.Fatal("expected task ID in minimal view")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	task, err = hcli.GetTask(id, "BASIC")
	if err != nil {
		t.Fatal(err)
	}

	if task.Name != "foo" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	task, err = hcli.GetTask(id, "FULL")
	if err != nil {
		t.Fatal(err)
	}

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}
}

// TODO this is a bit hacky for now because we're reusing the same
//      server + DB for all the e2e tests, so ListTasks gets the
//      results of all of those. It works for the moment, but
//      should probably run against a clean environment.
func TestListTaskView(t *testing.T) {
	var tasks []*tes.Task
	var task *tes.Task
	var err error

	id := run(`
    --cmd 'echo hello world'
    --name 'foo'
  `)
	wait(id)

	tasks = listView(tes.TaskView_MINIMAL)
	task = tasks[0]

	if task.Id == "" {
		t.Fatal("expected task ID in minimal view")
	}
	if task.State == tes.State_UNKNOWN {
		t.Fatal("expected complete state")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	tasks = listView(tes.TaskView_BASIC)
	task = tasks[0]

	if task.Name == "" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	tasks = listView(tes.TaskView_FULL)
	task = tasks[0]

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}

	// test http proxy
	var r *tes.ListTasksResponse
	r, err = hcli.ListTasks("MINIMAL")
	if err != nil {
		t.Fatal(err)
	}
	task = r.Tasks[0]

	if task.Id == "" {
		t.Fatal("expected task ID in minimal view")
	}
	if task.State == tes.State_UNKNOWN {
		t.Fatal("expected complete state")
	}
	if task.Name != "" {
		t.Fatal("unexpected task name included in minimal view")
	}
	if task.Logs != nil {
		t.Fatal("unexpected task logs included in minimal view")
	}

	r, err = hcli.ListTasks("BASIC")
	if err != nil {
		t.Fatal(err)
	}
	task = r.Tasks[0]

	if task.Name == "" {
		t.Fatal("expected task name to be included basic view")
	}

	if task.Logs != nil {
		t.Fatal("unexpected task logs included in basic view")
	}

	r, err = hcli.ListTasks("FULL")
	if err != nil {
		t.Fatal(err)
	}
	task = r.Tasks[0]

	if task.Logs[0].Logs[0].Stdout != "hello world\n" {
		t.Fatal("Missing stdout in full view")
	}
}

// Test that the streaming logs pick up a single character.
// This ensures that the streaming works even when a small
// amount of logs are written (which was once a bug).
func TestSingleCharLog(t *testing.T) {
	id := run(`
    --cmd "sh -c 'echo a; sleep 100'"
  `)
	waitForRunning(id)
	time.Sleep(time.Second * 2)
	task := get(id)
	if task.Logs[0].Logs[0].Stdout != "a\n" {
		t.Fatal("Missing logs")
	}
	cancel(id)
}

// Test that port mappings are being logged.
/* TODO need ports in funnel run
func TestPortLog(t *testing.T) {
  id := run(`
    --cmd 'echo start'
    --cmd 'sleep 10'
  `)
  waitForExec(id, 2)
  task := get(id)
  if task.Logs[0].Logs[0].Ports[0].Host != 5000 {
    t.Fatal("Unexpected port logs")
  }
}
*/

// Test that a completed task cannot change state.
func TestCompleteStateImmutable(t *testing.T) {
	id := run(`
    --cmd 'echo hello'
  `)
	wait(id)
	err := cancel(id)
	if err == nil {
		t.Fatal("expected error")
	}
	task := get(id)
	if task.State != tes.State_COMPLETE {
		t.Fatal("Unexpected state")
	}
}

// Test canceling a task
func TestCancel(t *testing.T) {
	id := run(`
    --cmd 'echo start'
    --cmd 'sleep 1000'
    --cmd 'echo never'
  `)
	waitForExec(id, 1)
	cancel(id)
	// TODO docker details and container ID are very Funnel specific
	//      how could we generalize this to be reusued as TES conformance?
	waitForDockerDestroy(id + "-0")
	task := get(id)
	if task.State != tes.State_CANCELED {
		t.Fatal("Unexpected state")
	}
}

// The task executor logs list should only include entries for steps that
// have been started or completed, i.e. steps that have yet to be started
// won't show up in Task.Logs[0].Logs
func TestExecutorLogLength(t *testing.T) {
	id := run(`
    --cmd 'echo first'
    --cmd 'sleep 10'
    --cmd 'echo done'
  `)
	waitForExec(id, 2)
	task := get(id)
	cancel(id)
	if len(task.Logs[0].Logs) != 2 {
		t.Fatal("Unexpected executor log count")
	}
}

// There was a bug + fix where the task was being marked complete after
// the first step completed, but the correct behavior is to mark the
// task complete after *all* steps have completed.
func TestMarkCompleteBug(t *testing.T) {
	id := run(`
    --cmd 'echo step 1'
    --cmd 'sleep 100'
  `)
	waitForRunning(id)
	waitForExec(id, 2)
	task := get(id)
	if task.State != tes.State_RUNNING {
		t.Fatal("Unexpected task state")
	}
	cancel(id)
}
<|endoftext|>


<commit_before>package abode

import (
	"context"
	"errors"
	"fmt"
	"strings"

	"github.com/snikch/api/config"

	"googlemaps.github.io/maps"
)

var client *maps.Client
var addressLineDelimeter = ","

// Address represents a response Address from abode.
type Address struct {
	AddressLine1     *string  `json:"address_line1"`
	AddressLine2     *string  `json:"address_line2"`
	AddressCity      *string  `json:"address_city"`
	AddressState     *string  `json:"address_state"`
	AddressCountry   *string  `json:"address_country"`
	AddressZip       *string  `json:"address_zip"`
	AddressLat       *float64 `json:"address_lat"`
	AddressLng       *float64 `json:"address_lng"`
	FormattedAddress *string  `json:"formatted_address"`
}

// initClient will initalize a Google Maps API client.
func initClient() error {
	var err error
	key := config.PrivateString("GOOGLE_MAPS_API_KEY")
	if key == "" {
		return errors.New("Please configure a `GOOGLE_MAPS_API_KEY`")
	}
	client, err = maps.NewClient(maps.WithAPIKey(key))
	return err
}

// Explode takes a one-line address string, explodes it and returns an *Address
func Explode(address string) (*Address, error) {
	if client == nil {
		err := initClient()
		if err != nil {
			return nil, err
		}
	}

	// Build the API request.
	req := &maps.GeocodingRequest{
		Address: address,
	}

	// Execute the request.
	resp, err := client.Geocode(context.Background(), req)
	if len(resp) < 1 {
		return nil, errors.New("Zero results for given address")
	}
	if err != nil {
		return nil, err
	}

	// Using the first/closest match in our response, grab the values we need.
	components := resp[0].AddressComponents
	formattedAddress := resp[0].FormattedAddress
	lat := resp[0].Geometry.Location.Lat
	lng := resp[0].Geometry.Location.Lng

	// Construct the return *Address{}
	response := &Address{
		AddressLine1:     compose(addressLine1Composition, "", components),
		AddressLine2:     compose(addressLine2Composition, addressLineDelimeter, components),
		AddressCity:      compose(addressCityComposition, addressLineDelimeter, components),
		AddressState:     compose(addressStateComposition, addressLineDelimeter, components),
		AddressCountry:   compose(addressCountryComposition, addressLineDelimeter, components),
		AddressZip:       compose(addressPostalCodeComposition, addressLineDelimeter, components),
		AddressLat:       &lat,
		AddressLng:       &lng,
		FormattedAddress: &formattedAddress,
	}

	return response, err
}

func compose(composition []string, delimeter string, components []maps.AddressComponent) *string {
	var str string
	for _, element := range composition {
		component := getComponentByType(components, element)
		if component != nil && !strings.Contains(str, component.LongName) {
			str = fmt.Sprintf("%s %s%s", str, component.LongName, delimeter)
		}
	}
	if str == "" {
		return nil
	}
	str = strings.TrimPrefix(strings.TrimSuffix(str, delimeter), " ")
	return &str
}
<commit_msg>Tweak to error responses<commit_after>package abode

import (
	"context"
	"errors"
	"fmt"
	"strings"

	"github.com/snikch/api/config"

	"googlemaps.github.io/maps"
)

var client *maps.Client
var addressLineDelimeter = ","

// Address represents a response Address from abode.
type Address struct {
	AddressLine1     *string  `json:"address_line1"`
	AddressLine2     *string  `json:"address_line2"`
	AddressCity      *string  `json:"address_city"`
	AddressState     *string  `json:"address_state"`
	AddressCountry   *string  `json:"address_country"`
	AddressZip       *string  `json:"address_zip"`
	AddressLat       *float64 `json:"address_lat"`
	AddressLng       *float64 `json:"address_lng"`
	FormattedAddress *string  `json:"formatted_address"`
}

// initClient will initalize a Google Maps API client.
func initClient() error {
	var err error
	key := config.PrivateString("GOOGLE_MAPS_API_KEY")
	if key == "" {
		return errors.New("Please configure a `GOOGLE_MAPS_API_KEY`")
	}
	client, err = maps.NewClient(maps.WithAPIKey(key))
	return err
}

// Explode takes a one-line address string, explodes it and returns an *Address
func Explode(address string) (*Address, error) {
	if client == nil {
		err := initClient()
		if err != nil {
			return nil, err
		}
	}

	// Build the API request.
	req := &maps.GeocodingRequest{
		Address: address,
	}

	// Execute the request.
	resp, err := client.Geocode(context.Background(), req)
	if len(resp) < 1 {
		return nil, err
	}
	if err != nil {
		return nil, err
	}

	// Using the first/closest match in our response, grab the values we need.
	components := resp[0].AddressComponents
	formattedAddress := resp[0].FormattedAddress
	lat := resp[0].Geometry.Location.Lat
	lng := resp[0].Geometry.Location.Lng

	// Construct the return *Address{}
	response := &Address{
		AddressLine1:     compose(addressLine1Composition, "", components),
		AddressLine2:     compose(addressLine2Composition, addressLineDelimeter, components),
		AddressCity:      compose(addressCityComposition, addressLineDelimeter, components),
		AddressState:     compose(addressStateComposition, addressLineDelimeter, components),
		AddressCountry:   compose(addressCountryComposition, addressLineDelimeter, components),
		AddressZip:       compose(addressPostalCodeComposition, addressLineDelimeter, components),
		AddressLat:       &lat,
		AddressLng:       &lng,
		FormattedAddress: &formattedAddress,
	}

	return response, err
}

func compose(composition []string, delimeter string, components []maps.AddressComponent) *string {
	var str string
	for _, element := range composition {
		component := getComponentByType(components, element)
		if component != nil && !strings.Contains(str, component.LongName) {
			str = fmt.Sprintf("%s %s%s", str, component.LongName, delimeter)
		}
	}
	if str == "" {
		return nil
	}
	str = strings.TrimPrefix(strings.TrimSuffix(str, delimeter), " ")
	return &str
}
<|endoftext|>