<commit_before>// Copyright 2017-2021 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
)

// The 5.4 CI job is intended to catch BPF complexity regressions and as such
// doesn't need to execute this test suite.
var _ = SkipDescribeIf(helpers.RunsOn54Kernel, "K8sBookInfoDemoTest", func() {
	var (
		kubectl        *helpers.Kubectl
		ciliumFilename string
	)

	BeforeAll(func() {
		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)
		ciliumFilename = helpers.TimestampFilename("cilium.yaml")
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
	})

	AfterEach(func() {
		ExpectAllPodsTerminated(kubectl)
	})

	AfterAll(func() {
		UninstallCiliumFromManifest(kubectl, ciliumFilename)
		kubectl.CloseSSHClient()
	})

	SkipContextIf(func() bool { return helpers.IsIntegration(helpers.CIIntegrationEKS) }, "Bookinfo Demo", func() {
		var (
			bookinfoV1YAML, bookinfoV2YAML string
			resourceYAMLs                  []string
			policyPath                     string
		)

		BeforeAll(func() {
			DeployCiliumAndDNS(kubectl, ciliumFilename)

			bookinfoV1YAML = helpers.ManifestGet(kubectl.BasePath(), "bookinfo-v1.yaml")
			bookinfoV2YAML = helpers.ManifestGet(kubectl.BasePath(), "bookinfo-v2.yaml")
			policyPath = helpers.ManifestGet(kubectl.BasePath(), "cnp-specs.yaml")

			resourceYAMLs = []string{bookinfoV1YAML, bookinfoV2YAML}

			for _, resourcePath := range resourceYAMLs {
				By("Creating objects in file %q", resourcePath)
				res := kubectl.Create(resourcePath)
				res.ExpectSuccess("unable to create resource %q", resourcePath)
			}

			By("Waiting for pods to be ready")
			err := kubectl.WaitforPods(helpers.DefaultNamespace, "-l zgroup=bookinfo", helpers.HelperTimeout)
			Expect(err).Should(BeNil(), "Pods are not ready after timeout")
		})

		AfterAll(func() {

			// Explicitly do not check result to avoid having assertions in AfterAll.
			_ = kubectl.Delete(policyPath)

			for _, resourcePath := range resourceYAMLs {
				By("Deleting resource %s", resourcePath)
				// Explicitly do not check result to avoid having assertions in AfterAll.
				_ = kubectl.Delete(resourcePath)
			}
		})

		It("Tests bookinfo demo", func() {

			// We use wget in this test because the Istio apps do not provide curl.
			wgetCommand := fmt.Sprintf("wget --tries=2 --connect-timeout %d", helpers.CurlConnectTimeout)

			version := "version"
			v1 := "v1"

			productPage := "productpage"
			reviews := "reviews"
			ratings := "ratings"
			details := "details"
			dnsChecks := []string{productPage, reviews, ratings, details}
			app := "app"
			health := "health"
			ratingsPath := "ratings/0"

			apiPort := "9080"

			podNameFilter := "{.items[*].metadata.name}"

			// shouldConnect asserts that srcPod can connect to dst.
			shouldConnect := func(srcPod, dst string) {
				By("Checking that %q can connect to %q", srcPod, dst)
				res := kubectl.ExecPodCmd(
					helpers.DefaultNamespace, srcPod, fmt.Sprintf("%s %s", wgetCommand, dst))
				res.ExpectSuccess("Unable to connect from %q to %q", srcPod, dst)
			}

			// shouldNotConnect asserts that srcPod cannot connect to dst.
			shouldNotConnect := func(srcPod, dst string) {
				By("Checking that %q cannot connect to %q", srcPod, dst)
				res := kubectl.ExecPodCmd(
					helpers.DefaultNamespace, srcPod, fmt.Sprintf("%s %s", wgetCommand, dst))
				res.ExpectFail("Was able to connect from %q to %q, but expected no connection: %s", srcPod, dst, res.CombineOutput())
			}

			// formatLabelArgument formats the provided key-value pairs as labels for use in
			// querying Kubernetes.
			formatLabelArgument := func(firstKey, firstValue string, nextLabels ...string) string {
				baseString := fmt.Sprintf("-l %s=%s", firstKey, firstValue)
				if nextLabels == nil {
					return baseString
				} else if len(nextLabels)%2 != 0 {
					Fail("must provide even number of arguments for label key-value pairings")
				} else {
					for i := 0; i < len(nextLabels); i += 2 {
						baseString = fmt.Sprintf("%s,%s=%s", baseString, nextLabels[i], nextLabels[i+1])
					}
				}
				return baseString
			}

			// formatAPI is a helper function which formats a URI to access.
			formatAPI := func(service, port, resource string) string {
				target := fmt.Sprintf(
					"%s.%s.svc.cluster.local:%s",
					service, helpers.DefaultNamespace, port)
				if resource != "" {
					return fmt.Sprintf("%s/%s", target, resource)
				}
				return target
			}

			err := kubectl.CiliumEndpointWaitReady()
			ExpectWithOffset(1, err).To(BeNil(), "Endpoints are not ready after timeout")

			By("Waiting for services to be ready")
			for _, service := range []string{details, ratings, reviews, productPage} {
				err = kubectl.WaitForServiceEndpoints(
					helpers.DefaultNamespace, "", service,
					helpers.HelperTimeout)
				Expect(err).Should(BeNil(), "Service %q is not ready after timeout", service)
			}
			By("Validating DNS without Policy")
			for _, name := range dnsChecks {
				err = kubectl.WaitForKubeDNSEntry(name, helpers.DefaultNamespace)
				Expect(err).To(BeNil(), "DNS entry is not ready after timeout")
			}

			By("All pods should be able to connect without policy")

			reviewsPodV1, err := kubectl.GetPods(helpers.DefaultNamespace, formatLabelArgument(app, reviews, version, v1)).Filter(podNameFilter)
			Expect(err).Should(BeNil(), "cannot get reviewsV1 pods")
			productpagePodV1, err := kubectl.GetPods(helpers.DefaultNamespace, formatLabelArgument(app, productPage, version, v1)).Filter(podNameFilter)
			Expect(err).Should(BeNil(), "cannot get productpageV1 pods")

			shouldConnect(reviewsPodV1.String(), formatAPI(ratings, apiPort, health))
			shouldConnect(reviewsPodV1.String(), formatAPI(ratings, apiPort, ratingsPath))

			shouldConnect(productpagePodV1.String(), formatAPI(details, apiPort, health))
			shouldConnect(productpagePodV1.String(), formatAPI(ratings, apiPort, health))
			shouldConnect(productpagePodV1.String(), formatAPI(ratings, apiPort, ratingsPath))

			policyCmd := "cilium policy get io.cilium.k8s.policy.name=cnp-specs"

			By("Importing policy")

			_, err = kubectl.CiliumPolicyAction(helpers.DefaultNamespace, policyPath, helpers.KubectlCreate, helpers.HelperTimeout)
			Expect(err).Should(BeNil(), "Error creating policy %q", policyPath)

			By("Checking that policies were correctly imported into Cilium")

			ciliumPodK8s1, err := kubectl.GetCiliumPodOnNode(helpers.K8s1)
			Expect(err).Should(BeNil(), "Cannot get cilium pod on k8s1")
			res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPodK8s1, policyCmd)
			res.ExpectSuccess("Policy %s is not imported", policyCmd)

			By("Validating DNS with Policy loaded")
			for _, name := range dnsChecks {
				err = kubectl.WaitForKubeDNSEntry(name, helpers.DefaultNamespace)
				Expect(err).To(BeNil(), "DNS entry is not ready after timeout")
			}

			By("After policy import")
			shouldConnect(reviewsPodV1.String(), formatAPI(ratings, apiPort, health))
			shouldNotConnect(reviewsPodV1.String(), formatAPI(ratings, apiPort, ratingsPath))

			shouldConnect(productpagePodV1.String(), formatAPI(details, apiPort, health))

			shouldNotConnect(productpagePodV1.String(), formatAPI(ratings, apiPort, health))
			shouldNotConnect(productpagePodV1.String(), formatAPI(ratings, apiPort, ratingsPath))
		})
	})

})
<commit_msg>test/Bookinfo: Collect full artifact in case of failure<commit_after>// Copyright 2017-2021 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
)

// The 5.4 CI job is intended to catch BPF complexity regressions and as such
// doesn't need to execute this test suite.
var _ = SkipDescribeIf(helpers.RunsOn54Kernel, "K8sBookInfoDemoTest", func() {
	var (
		kubectl        *helpers.Kubectl
		ciliumFilename string
	)

	BeforeAll(func() {
		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)
		ciliumFilename = helpers.TimestampFilename("cilium.yaml")
	})

	JustAfterEach(func() {
		kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
	})

	AfterEach(func() {
		ExpectAllPodsTerminated(kubectl)
	})

	AfterAll(func() {
		UninstallCiliumFromManifest(kubectl, ciliumFilename)
		kubectl.CloseSSHClient()
	})

	AfterFailed(func() {
		kubectl.CiliumReport("cilium endpoint list")
	})

	SkipContextIf(func() bool { return helpers.IsIntegration(helpers.CIIntegrationEKS) }, "Bookinfo Demo", func() {
		var (
			bookinfoV1YAML, bookinfoV2YAML string
			resourceYAMLs                  []string
			policyPath                     string
		)

		BeforeAll(func() {
			DeployCiliumAndDNS(kubectl, ciliumFilename)

			bookinfoV1YAML = helpers.ManifestGet(kubectl.BasePath(), "bookinfo-v1.yaml")
			bookinfoV2YAML = helpers.ManifestGet(kubectl.BasePath(), "bookinfo-v2.yaml")
			policyPath = helpers.ManifestGet(kubectl.BasePath(), "cnp-specs.yaml")

			resourceYAMLs = []string{bookinfoV1YAML, bookinfoV2YAML}

			for _, resourcePath := range resourceYAMLs {
				By("Creating objects in file %q", resourcePath)
				res := kubectl.Create(resourcePath)
				res.ExpectSuccess("unable to create resource %q", resourcePath)
			}

			By("Waiting for pods to be ready")
			err := kubectl.WaitforPods(helpers.DefaultNamespace, "-l zgroup=bookinfo", helpers.HelperTimeout)
			Expect(err).Should(BeNil(), "Pods are not ready after timeout")
		})

		AfterAll(func() {

			// Explicitly do not check result to avoid having assertions in AfterAll.
			_ = kubectl.Delete(policyPath)

			for _, resourcePath := range resourceYAMLs {
				By("Deleting resource %s", resourcePath)
				// Explicitly do not check result to avoid having assertions in AfterAll.
				_ = kubectl.Delete(resourcePath)
			}
		})

		It("Tests bookinfo demo", func() {

			// We use wget in this test because the Istio apps do not provide curl.
			wgetCommand := fmt.Sprintf("wget --tries=2 --connect-timeout %d", helpers.CurlConnectTimeout)

			version := "version"
			v1 := "v1"

			productPage := "productpage"
			reviews := "reviews"
			ratings := "ratings"
			details := "details"
			dnsChecks := []string{productPage, reviews, ratings, details}
			app := "app"
			health := "health"
			ratingsPath := "ratings/0"

			apiPort := "9080"

			podNameFilter := "{.items[*].metadata.name}"

			// shouldConnect asserts that srcPod can connect to dst.
			shouldConnect := func(srcPod, dst string) {
				By("Checking that %q can connect to %q", srcPod, dst)
				res := kubectl.ExecPodCmd(
					helpers.DefaultNamespace, srcPod, fmt.Sprintf("%s %s", wgetCommand, dst))
				res.ExpectSuccess("Unable to connect from %q to %q", srcPod, dst)
			}

			// shouldNotConnect asserts that srcPod cannot connect to dst.
			shouldNotConnect := func(srcPod, dst string) {
				By("Checking that %q cannot connect to %q", srcPod, dst)
				res := kubectl.ExecPodCmd(
					helpers.DefaultNamespace, srcPod, fmt.Sprintf("%s %s", wgetCommand, dst))
				res.ExpectFail("Was able to connect from %q to %q, but expected no connection: %s", srcPod, dst, res.CombineOutput())
			}

			// formatLabelArgument formats the provided key-value pairs as labels for use in
			// querying Kubernetes.
			formatLabelArgument := func(firstKey, firstValue string, nextLabels ...string) string {
				baseString := fmt.Sprintf("-l %s=%s", firstKey, firstValue)
				if nextLabels == nil {
					return baseString
				} else if len(nextLabels)%2 != 0 {
					Fail("must provide even number of arguments for label key-value pairings")
				} else {
					for i := 0; i < len(nextLabels); i += 2 {
						baseString = fmt.Sprintf("%s,%s=%s", baseString, nextLabels[i], nextLabels[i+1])
					}
				}
				return baseString
			}

			// formatAPI is a helper function which formats a URI to access.
			formatAPI := func(service, port, resource string) string {
				target := fmt.Sprintf(
					"%s.%s.svc.cluster.local:%s",
					service, helpers.DefaultNamespace, port)
				if resource != "" {
					return fmt.Sprintf("%s/%s", target, resource)
				}
				return target
			}

			err := kubectl.CiliumEndpointWaitReady()
			ExpectWithOffset(1, err).To(BeNil(), "Endpoints are not ready after timeout")

			By("Waiting for services to be ready")
			for _, service := range []string{details, ratings, reviews, productPage} {
				err = kubectl.WaitForServiceEndpoints(
					helpers.DefaultNamespace, "", service,
					helpers.HelperTimeout)
				Expect(err).Should(BeNil(), "Service %q is not ready after timeout", service)
			}
			By("Validating DNS without Policy")
			for _, name := range dnsChecks {
				err = kubectl.WaitForKubeDNSEntry(name, helpers.DefaultNamespace)
				Expect(err).To(BeNil(), "DNS entry is not ready after timeout")
			}

			By("All pods should be able to connect without policy")

			reviewsPodV1, err := kubectl.GetPods(helpers.DefaultNamespace, formatLabelArgument(app, reviews, version, v1)).Filter(podNameFilter)
			Expect(err).Should(BeNil(), "cannot get reviewsV1 pods")
			productpagePodV1, err := kubectl.GetPods(helpers.DefaultNamespace, formatLabelArgument(app, productPage, version, v1)).Filter(podNameFilter)
			Expect(err).Should(BeNil(), "cannot get productpageV1 pods")

			shouldConnect(reviewsPodV1.String(), formatAPI(ratings, apiPort, health))
			shouldConnect(reviewsPodV1.String(), formatAPI(ratings, apiPort, ratingsPath))

			shouldConnect(productpagePodV1.String(), formatAPI(details, apiPort, health))
			shouldConnect(productpagePodV1.String(), formatAPI(ratings, apiPort, health))
			shouldConnect(productpagePodV1.String(), formatAPI(ratings, apiPort, ratingsPath))

			policyCmd := "cilium policy get io.cilium.k8s.policy.name=cnp-specs"

			By("Importing policy")

			_, err = kubectl.CiliumPolicyAction(helpers.DefaultNamespace, policyPath, helpers.KubectlCreate, helpers.HelperTimeout)
			Expect(err).Should(BeNil(), "Error creating policy %q", policyPath)

			By("Checking that policies were correctly imported into Cilium")

			ciliumPodK8s1, err := kubectl.GetCiliumPodOnNode(helpers.K8s1)
			Expect(err).Should(BeNil(), "Cannot get cilium pod on k8s1")
			res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPodK8s1, policyCmd)
			res.ExpectSuccess("Policy %s is not imported", policyCmd)

			By("Validating DNS with Policy loaded")
			for _, name := range dnsChecks {
				err = kubectl.WaitForKubeDNSEntry(name, helpers.DefaultNamespace)
				Expect(err).To(BeNil(), "DNS entry is not ready after timeout")
			}

			By("After policy import")
			shouldConnect(reviewsPodV1.String(), formatAPI(ratings, apiPort, health))
			shouldNotConnect(reviewsPodV1.String(), formatAPI(ratings, apiPort, ratingsPath))

			shouldConnect(productpagePodV1.String(), formatAPI(details, apiPort, health))

			shouldNotConnect(productpagePodV1.String(), formatAPI(ratings, apiPort, health))
			shouldNotConnect(productpagePodV1.String(), formatAPI(ratings, apiPort, ratingsPath))
		})
	})

})
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package tests_test

import (
	"flag"
	"time"

	"github.com/google/goexpect"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/tests"

	k8sv1 "k8s.io/api/core/v1"
)

var _ = Describe("Console", func() {

	flag.Parse()

	var virtClient kubecli.KubevirtClient
	var err error

	tests.PanicOnError(err)

	BeforeEach(func() {
		tests.BeforeTestCleanup()
	})

	JustBeforeEach(func() {
		By("Opening new virtClient")
		virtClient, err = kubecli.GetKubevirtClient()
	})

	RunVMIAndWaitForStart := func(vmi *v1.VirtualMachineInstance) {
		By("Creating a new VirtualMachineInstance")
		Expect(virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Error()).To(Succeed())

		By("Waiting until it starts")
		tests.WaitForSuccessfulVMIStartWithTimeout(vmi, 90)
	}

	ExpectConsoleOutput := func(vmi *v1.VirtualMachineInstance, expected string) {
		By("Expecting the VirtualMachineInstance console")
		expecter, _, err := tests.NewConsoleExpecter(virtClient, vmi, 30*time.Second)
		Expect(err).ToNot(HaveOccurred())
		defer func() {
			By("Closing the opened expecter")
			expecter.Close()
		}()

		By("Checking that the console output equals to expected one")
		_, err = expecter.ExpectBatch([]expect.Batcher{
			&expect.BSnd{S: "\n"},
			&expect.BExp{R: expected},
		}, 120*time.Second)
		Expect(err).ToNot(HaveOccurred())
	}

	Describe("A new VirtualMachineInstance", func() {
		Context("with a serial console", func() {
			Context("with a cirros image", func() {
				It("should return that we are running cirros", func() {
					vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.RegistryDiskFor(tests.RegistryDiskCirros), "#!/bin/bash\necho 'hello'\n")
					RunVMIAndWaitForStart(vmi)
					ExpectConsoleOutput(
						vmi,
						"login as 'cirros' user",
					)
				}, 140)
			})

			Context("with a fedora image", func() {
				It("should return that we are running fedora", func() {
					vmi := tests.NewRandomVMIWithEphemeralDiskHighMemory(tests.RegistryDiskFor(tests.RegistryDiskFedora))
					RunVMIAndWaitForStart(vmi)
					ExpectConsoleOutput(
						vmi,
						"Welcome to",
					)
				}, 140)
			})

			It("should be able to reconnect to console multiple times", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(tests.RegistryDiskFor(tests.RegistryDiskAlpine))

				RunVMIAndWaitForStart(vmi)

				for i := 0; i < 5; i++ {
					ExpectConsoleOutput(vmi, "login")
				}
			}, 220)

			It("should wait until the virtual machine is in running state and return a stream interface", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(tests.RegistryDiskFor(tests.RegistryDiskAlpine))
				By("Creating a new VirtualMachineInstance")
				Expect(virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Error()).To(Succeed())

				_, err := virtClient.VirtualMachineInstance(vmi.Namespace).SerialConsole(vmi.Name, 30*time.Second)
				Expect(err).ToNot(HaveOccurred())
			}, 220)

			It("should fail waiting for the virtual machine instance to be running", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(tests.RegistryDiskFor(tests.RegistryDiskAlpine))
				vmi.Spec.Affinity = &k8sv1.Affinity{
					NodeAffinity: &k8sv1.NodeAffinity{
						RequiredDuringSchedulingIgnoredDuringExecution: &k8sv1.NodeSelector{
							NodeSelectorTerms: []k8sv1.NodeSelectorTerm{
								{
									MatchExpressions: []k8sv1.NodeSelectorRequirement{
										{Key: "kubernetes.io/hostname", Operator: k8sv1.NodeSelectorOpIn, Values: []string{"notexist"}},
									},
								},
							},
						},
					},
				}

				By("Creating a new VirtualMachineInstance")
				Expect(virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Error()).To(Succeed())

				_, err := virtClient.VirtualMachineInstance(vmi.Namespace).SerialConsole(vmi.Name, 30*time.Second)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(Equal("Timeout trying to connect to the virtual machine instance"))
			}, 180)

			It("should fail waiting for the expecter", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(tests.RegistryDiskFor(tests.RegistryDiskAlpine))
				vmi.Spec.Affinity = &k8sv1.Affinity{
					NodeAffinity: &k8sv1.NodeAffinity{
						RequiredDuringSchedulingIgnoredDuringExecution: &k8sv1.NodeSelector{
							NodeSelectorTerms: []k8sv1.NodeSelectorTerm{
								{
									MatchExpressions: []k8sv1.NodeSelectorRequirement{
										{Key: "kubernetes.io/hostname", Operator: k8sv1.NodeSelectorOpIn, Values: []string{"notexist"}},
									},
								},
							},
						},
					},
				}

				By("Creating a new VirtualMachineInstance")
				Expect(virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Error()).To(Succeed())

				By("Expecting the VirtualMachineInstance console")
				_, _, err := tests.NewConsoleExpecter(virtClient, vmi, 30*time.Second)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(Equal("Timeout trying to connect to the virtual machine instance"))
			}, 180)
		})
	})
})
<commit_msg>Remove justBeforeEach block<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package tests_test

import (
	"flag"
	"time"

	"github.com/google/goexpect"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/tests"

	k8sv1 "k8s.io/api/core/v1"
)

var _ = Describe("Console", func() {

	flag.Parse()

	virtClient, err := kubecli.GetKubevirtClient()
	tests.PanicOnError(err)

	BeforeEach(func() {
		tests.BeforeTestCleanup()
	})

	RunVMIAndWaitForStart := func(vmi *v1.VirtualMachineInstance) {
		By("Creating a new VirtualMachineInstance")
		Expect(virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Error()).To(Succeed())

		By("Waiting until it starts")
		tests.WaitForSuccessfulVMIStartWithTimeout(vmi, 90)
	}

	ExpectConsoleOutput := func(vmi *v1.VirtualMachineInstance, expected string) {
		By("Expecting the VirtualMachineInstance console")
		expecter, _, err := tests.NewConsoleExpecter(virtClient, vmi, 30*time.Second)
		Expect(err).ToNot(HaveOccurred())
		defer func() {
			By("Closing the opened expecter")
			expecter.Close()
		}()

		By("Checking that the console output equals to expected one")
		_, err = expecter.ExpectBatch([]expect.Batcher{
			&expect.BSnd{S: "\n"},
			&expect.BExp{R: expected},
		}, 120*time.Second)
		Expect(err).ToNot(HaveOccurred())
	}

	Describe("A new VirtualMachineInstance", func() {
		Context("with a serial console", func() {
			Context("with a cirros image", func() {
				It("should return that we are running cirros", func() {
					vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.RegistryDiskFor(tests.RegistryDiskCirros), "#!/bin/bash\necho 'hello'\n")
					RunVMIAndWaitForStart(vmi)
					ExpectConsoleOutput(
						vmi,
						"login as 'cirros' user",
					)
				}, 140)
			})

			Context("with a fedora image", func() {
				It("should return that we are running fedora", func() {
					vmi := tests.NewRandomVMIWithEphemeralDiskHighMemory(tests.RegistryDiskFor(tests.RegistryDiskFedora))
					RunVMIAndWaitForStart(vmi)
					ExpectConsoleOutput(
						vmi,
						"Welcome to",
					)
				}, 140)
			})

			It("should be able to reconnect to console multiple times", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(tests.RegistryDiskFor(tests.RegistryDiskAlpine))

				RunVMIAndWaitForStart(vmi)

				for i := 0; i < 5; i++ {
					ExpectConsoleOutput(vmi, "login")
				}
			}, 220)

			It("should wait until the virtual machine is in running state and return a stream interface", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(tests.RegistryDiskFor(tests.RegistryDiskAlpine))
				By("Creating a new VirtualMachineInstance")
				Expect(virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Error()).To(Succeed())

				_, err := virtClient.VirtualMachineInstance(vmi.Namespace).SerialConsole(vmi.Name, 30*time.Second)
				Expect(err).ToNot(HaveOccurred())
			}, 220)

			It("should fail waiting for the virtual machine instance to be running", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(tests.RegistryDiskFor(tests.RegistryDiskAlpine))
				vmi.Spec.Affinity = &k8sv1.Affinity{
					NodeAffinity: &k8sv1.NodeAffinity{
						RequiredDuringSchedulingIgnoredDuringExecution: &k8sv1.NodeSelector{
							NodeSelectorTerms: []k8sv1.NodeSelectorTerm{
								{
									MatchExpressions: []k8sv1.NodeSelectorRequirement{
										{Key: "kubernetes.io/hostname", Operator: k8sv1.NodeSelectorOpIn, Values: []string{"notexist"}},
									},
								},
							},
						},
					},
				}

				By("Creating a new VirtualMachineInstance")
				Expect(virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Error()).To(Succeed())

				_, err := virtClient.VirtualMachineInstance(vmi.Namespace).SerialConsole(vmi.Name, 30*time.Second)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(Equal("Timeout trying to connect to the virtual machine instance"))
			}, 180)

			It("should fail waiting for the expecter", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(tests.RegistryDiskFor(tests.RegistryDiskAlpine))
				vmi.Spec.Affinity = &k8sv1.Affinity{
					NodeAffinity: &k8sv1.NodeAffinity{
						RequiredDuringSchedulingIgnoredDuringExecution: &k8sv1.NodeSelector{
							NodeSelectorTerms: []k8sv1.NodeSelectorTerm{
								{
									MatchExpressions: []k8sv1.NodeSelectorRequirement{
										{Key: "kubernetes.io/hostname", Operator: k8sv1.NodeSelectorOpIn, Values: []string{"notexist"}},
									},
								},
							},
						},
					},
				}

				By("Creating a new VirtualMachineInstance")
				Expect(virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Error()).To(Succeed())

				By("Expecting the VirtualMachineInstance console")
				_, _, err := tests.NewConsoleExpecter(virtClient, vmi, 30*time.Second)
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(Equal("Timeout trying to connect to the virtual machine instance"))
			}, 180)
		})
	})
})
<|endoftext|>


<commit_before>package util

import (
	"fmt"
	"path/filepath"
	"time"
)

const (
	NUM_WORKERS_PROD     int = 100
	MASTER_NAME              = "build101-m5"
	WORKER_NAME_TEMPLATE     = "build%d-m5"
	GS_HTTP_LINK             = "https://storage.cloud.google.com/"
	LOGS_LINK_PREFIX         = "http://uberchromegw.corp.google.com/i/skia-ct-worker"

	// File names and dir names.
	TIMESTAMP_FILE_NAME          = "TIMESTAMP"
	CHROMIUM_BUILDS_DIR_NAME     = "chromium_builds"
	PAGESETS_DIR_NAME            = "page_sets"
	WEB_ARCHIVES_DIR_NAME        = "webpage_archives"
	SKPS_DIR_NAME                = "skps"
	STORAGE_DIR_NAME             = "storage"
	REPO_DIR_NAME                = "skia-repo"
	TASKS_DIR_NAME               = "tasks"
	LUA_TASKS_DIR_NAME           = "lua_runs"
	BENCHMARK_TASKS_DIR_NAME     = "benchmark_runs"
	CHROMIUM_PERF_TASKS_DIR_NAME = "chromium_perf_runs"
	FIX_ARCHIVE_TASKS_DIR_NAME   = "fix_archive_runs"

	// Limit the number of times CT tries to get a remote file before giving up.
	MAX_URI_GET_TRIES = 4

	// Activity constants.
	ACTIVITY_CREATING_PAGESETS        = "CREATING_PAGESETS"
	ACTIVITY_CAPTURING_ARCHIVES       = "CAPTURING_ARCHIVES"
	ACTIVITY_CAPTURING_SKPS           = "CAPTURING_SKPS"
	ACTIVITY_RUNNING_LUA_SCRIPTS      = "RUNNING_LUA_SCRIPTS"
	ACTIVITY_RUNNING_CHROMIUM_PERF    = "RUNNING_CHROMIUM_PERF"
	ACTIVITY_RUNNING_SKIA_CORRECTNESS = "RUNNING_SKIA_CORRECTNESS"
	ACTIVITY_FIXING_ARCHIVES          = "FIXING_ARCHIVES"

	// Pageset types supported by CT.
	PAGESET_TYPE_ALL        = "All"
	PAGESET_TYPE_10k        = "10k"
	PAGESET_TYPE_MOBILE_10k = "Mobile10k"
	PAGESET_TYPE_DUMMY_1k   = "Dummy1k" // Used for testing.

	// Names of binaries executed by CT.
	BINARY_CHROME          = "chrome"
	BINARY_RECORD_WPR      = "record_wpr"
	BINARY_RUN_BENCHMARK   = "ct_run_benchmark"
	BINARY_GCLIENT         = "gclient"
	BINARY_MAKE            = "make"
	BINARY_LUA_PICTURES    = "lua_pictures"
	BINARY_ADB             = "adb"
	BINARY_GIT             = "git"
	BINARY_RENDER_PICTURES = "render_pictures"
	BINARY_MAIL            = "mail"
	BINARY_LUA             = "lua"

	// Platforms supported by CT.
	PLATFORM_ANDROID = "Android"
	PLATFORM_LINUX   = "Linux"

	// Benchmarks supported by CT.
	BENCHMARK_DRAW_PROPERTIES   = "draw_properties"
	BENCHMARK_SKPICTURE_PRINTER = "skpicture_printer"
	BENCHMARK_RR                = "rasterize_and_record_micro"
	BENCHMARK_REPAINT           = "repaint"
	BENCHMARK_SMOOTHNESS        = "smoothness"

	// Logserver links. These are only accessible from Google corp.
	MASTER_LOGSERVER_LINK  = "http://uberchromegw.corp.google.com/i/skia-ct-master/"
	WORKERS_LOGSERVER_LINK = "http://uberchromegw.corp.google.com/i/skia-ct-master/all_logs"

	// Default browser args when running benchmarks.
	DEFAULT_BROWSER_ARGS = "--disable-setuid-sandbox --enable-threaded-compositing --enable-impl-side-painting"

	// Timeouts

	PKILL_TIMEOUT = 5 * time.Minute

	// util.SyncDir
	GIT_PULL_TIMEOUT     = 10 * time.Minute
	GCLIENT_SYNC_TIMEOUT = 15 * time.Minute

	// util.BuildSkiaTools
	MAKE_CLEAN_TIMEOUT = 5 * time.Minute
	MAKE_TOOLS_TIMEOUT = 5 * time.Minute

	// util.ResetCheckout
	GIT_RESET_TIMEOUT = 5 * time.Minute
	GIT_CLEAN_TIMEOUT = 5 * time.Minute
	// util.resetChromiumCheckout calls ResetCheckout three times.
	RESET_CHROMIUM_CHECKOUT_TIMEOUT = 3 * (GIT_RESET_TIMEOUT + GIT_CLEAN_TIMEOUT)

	// util.CreateChromiumBuild
	SYNC_SKIA_IN_CHROME_TIMEOUT   = 2 * time.Hour
	GIT_LS_REMOTE_TIMEOUT         = 5 * time.Minute
	GIT_APPLY_TIMEOUT             = 5 * time.Minute
	GOMA_CTL_RESTART_TIMEOUT      = 10 * time.Minute
	GYP_CHROMIUM_TIMEOUT          = 30 * time.Minute
	NINJA_TIMEOUT                 = 2 * time.Hour
	CREATE_CHROMIUM_BUILD_TIMEOUT = SYNC_SKIA_IN_CHROME_TIMEOUT + GIT_LS_REMOTE_TIMEOUT +
		// Three patches are applied when applyPatches is specified.
		3*GIT_APPLY_TIMEOUT +
		// The build steps are repeated twice when applyPatches is specified.
		2*(GOMA_CTL_RESTART_TIMEOUT+GYP_CHROMIUM_TIMEOUT+NINJA_TIMEOUT+
			RESET_CHROMIUM_CHECKOUT_TIMEOUT)

	// util.InstallChromeAPK
	ADB_INSTALL_TIMEOUT = 15 * time.Minute

	// Allow extra time for updating frontend and any other computation not included in the
	// worker timeouts.
	MASTER_SCRIPT_TIMEOUT_PADDING = 30 * time.Minute

	// Build Chromium Task
	GIT_LOG_TIMEOUT                      = 5 * time.Minute
	MASTER_SCRIPT_BUILD_CHROMIUM_TIMEOUT = CREATE_CHROMIUM_BUILD_TIMEOUT + GIT_LOG_TIMEOUT +
		MASTER_SCRIPT_TIMEOUT_PADDING

	// Capture Archives
	// Setting a 5 day timeout since it may take a while to capture 1M archives.
	CAPTURE_ARCHIVES_TIMEOUT               = 5 * 24 * time.Hour
	MASTER_SCRIPT_CAPTURE_ARCHIVES_TIMEOUT = CAPTURE_ARCHIVES_TIMEOUT +
		MASTER_SCRIPT_TIMEOUT_PADDING

	// Capture SKPs
	REMOVE_INVALID_SKPS_TIMEOUT = 3 * time.Hour
	// Setting a 2 day timeout since it may take a while to capture 1M SKPs.
	CAPTURE_SKPS_TIMEOUT               = 2 * 24 * time.Hour
	MASTER_SCRIPT_CAPTURE_SKPS_TIMEOUT = CAPTURE_SKPS_TIMEOUT + MASTER_SCRIPT_TIMEOUT_PADDING

	// Check Workers Health
	ADB_DEVICES_TIMEOUT          = 30 * time.Minute
	ADB_SHELL_UPTIME_TIMEOUT     = 30 * time.Minute
	CHECK_WORKERS_HEALTH_TIMEOUT = ADB_DEVICES_TIMEOUT + ADB_SHELL_UPTIME_TIMEOUT +
		MASTER_SCRIPT_TIMEOUT_PADDING

	// Create Pagesets
	// Setting a 4 hour timeout since it may take a while to upload page sets to
	// Google Storage when doing 10k page sets per worker.
	CREATE_PAGESETS_TIMEOUT               = 4 * time.Hour
	MASTER_SCRIPT_CREATE_PAGESETS_TIMEOUT = CREATE_PAGESETS_TIMEOUT +
		MASTER_SCRIPT_TIMEOUT_PADDING

	// Run Chromium Perf
	ADB_VERSION_TIMEOUT            = 5 * time.Minute
	ADB_ROOT_TIMEOUT               = 5 * time.Minute
	CSV_PIVOT_TABLE_MERGER_TIMEOUT = 10 * time.Minute
	REBOOT_TIMEOUT                 = 5 * time.Minute
	CSV_MERGER_TIMEOUT             = 1 * time.Hour
	CSV_COMPARER_TIMEOUT           = 2 * time.Hour
	// Setting a 1 day timeout since it may take a while run benchmarks with many
	// repeats.
	RUN_CHROMIUM_PERF_TIMEOUT = 1 * 24 * time.Hour
	// csv_merger runs once for nopatch and once for withpatch
	MASTER_SCRIPT_RUN_CHROMIUM_PERF_TIMEOUT = CREATE_CHROMIUM_BUILD_TIMEOUT + REBOOT_TIMEOUT +
		RUN_CHROMIUM_PERF_TIMEOUT + 2*CSV_MERGER_TIMEOUT + CSV_COMPARER_TIMEOUT +
		MASTER_SCRIPT_TIMEOUT_PADDING

	// Run Lua
	LUA_PICTURES_TIMEOUT          = 2 * time.Hour
	RUN_LUA_TIMEOUT               = 2 * time.Hour
	LUA_AGGREGATOR_TIMEOUT        = 1 * time.Hour
	MASTER_SCRIPT_RUN_LUA_TIMEOUT = RUN_LUA_TIMEOUT + LUA_AGGREGATOR_TIMEOUT +
		MASTER_SCRIPT_TIMEOUT_PADDING

	// Fix Archives
	// Setting a 1 day timeout since it may take a while to validate archives.
	FIX_ARCHIVES_TIMEOUT = 1 * 24 * time.Hour

	// Poller
	MAKE_ALL_TIMEOUT = 15 * time.Minute

	WEBHOOK_SALT_MSG = `For prod, set this file to the value of GCE metadata key webhook_request_salt or call webhook.MustInitRequestSaltFromMetadata() if running in GCE. For testing, run 'echo -n "notverysecret" | base64 -w 0 > /b/storage/webhook_salt.data' or call frontend.InitForTesting().`
)

type PagesetTypeInfo struct {
	NumPages                   int
	CSVSource                  string
	UserAgent                  string
	CaptureArchivesTimeoutSecs int
	CreatePagesetsTimeoutSecs  int
	CaptureSKPsTimeoutSecs     int
	RunChromiumPerfTimeoutSecs int
	Description                string
}

var (
	Master       = fmt.Sprintf(WORKER_NAME_TEMPLATE, 101)
	CtUser       = "chrome-bot"
	Slaves       = GetCTWorkersProd()
	GSBucketName = "cluster-telemetry"

	// Email address of cluster telemetry admins. They will be notified everytime
	// a task has started and completed.
	CtAdmins = []string{"rmistry@google.com"}

	// Names of local directories and files.
	StorageDir           = filepath.Join("/", "b", STORAGE_DIR_NAME)
	RepoDir              = filepath.Join("/", "b", REPO_DIR_NAME)
	GomaDir              = filepath.Join("/", "b", "build", "goma")
	ChromiumBuildsDir    = filepath.Join(StorageDir, CHROMIUM_BUILDS_DIR_NAME)
	ChromiumSrcDir       = filepath.Join(StorageDir, "chromium", "src")
	TelemetryBinariesDir = filepath.Join(ChromiumSrcDir, "tools", "perf")
	TelemetrySrcDir      = filepath.Join(ChromiumSrcDir, "tools", "telemetry")
	TaskFileDir          = filepath.Join(StorageDir, "current_task")
	ClientSecretPath     = filepath.Join(StorageDir, "client_secret.json")
	GSTokenPath          = filepath.Join(StorageDir, "google_storage_token.data")
	EmailTokenPath       = filepath.Join(StorageDir, "email.data")
	WebappPasswordPath   = filepath.Join(StorageDir, "webapp.data")
	// Salt used to authenticate webhook requests, base64-encoded. See WEBHOOK_SALT_MSG.
	WebhookRequestSaltPath = filepath.Join(StorageDir, "webhook_salt.data")
	PagesetsDir            = filepath.Join(StorageDir, PAGESETS_DIR_NAME)
	WebArchivesDir         = filepath.Join(StorageDir, WEB_ARCHIVES_DIR_NAME)
	SkpsDir                = filepath.Join(StorageDir, SKPS_DIR_NAME)
	GLogDir                = filepath.Join(StorageDir, "glog")
	ApkName                = "ChromePublic.apk"
	SkiaTreeDir            = filepath.Join(RepoDir, "trunk")
	CtTreeDir              = filepath.Join(RepoDir, "go", "src", "go.skia.org", "infra", "ct")

	// Names of remote directories and files.
	LuaRunsDir          = filepath.Join(TASKS_DIR_NAME, LUA_TASKS_DIR_NAME)
	BenchmarkRunsDir    = filepath.Join(TASKS_DIR_NAME, BENCHMARK_TASKS_DIR_NAME)
	ChromiumPerfRunsDir = filepath.Join(TASKS_DIR_NAME, CHROMIUM_PERF_TASKS_DIR_NAME)
	FixArchivesRunsDir  = filepath.Join(TASKS_DIR_NAME, FIX_ARCHIVE_TASKS_DIR_NAME)

	// Information about the different CT benchmarks.
	BenchmarksToPagesetName = map[string]string{
		BENCHMARK_DRAW_PROPERTIES:   "DrawPropertiesCTPages",
		BENCHMARK_SKPICTURE_PRINTER: "SkpicturePrinter",
		BENCHMARK_RR:                "RasterizeAndRecordMicroCTPages",
		BENCHMARK_REPAINT:           "RepaintCTPages",
		BENCHMARK_SMOOTHNESS:        "SmoothnessCTPages",
	}

	// Information about the different CT pageset types.
	PagesetTypeToInfo = map[string]*PagesetTypeInfo{
		PAGESET_TYPE_ALL: &PagesetTypeInfo{
			NumPages:                   1000000,
			CSVSource:                  "csv/top-1m.csv",
			UserAgent:                  "desktop",
			CreatePagesetsTimeoutSecs:  60,
			CaptureArchivesTimeoutSecs: 300,
			CaptureSKPsTimeoutSecs:     300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 1M (with desktop user-agent)",
		},
		PAGESET_TYPE_10k: &PagesetTypeInfo{
			NumPages:                   10000,
			CSVSource:                  "csv/top-1m.csv",
			UserAgent:                  "desktop",
			CreatePagesetsTimeoutSecs:  60,
			CaptureArchivesTimeoutSecs: 300,
			CaptureSKPsTimeoutSecs:     300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 10K (with desktop user-agent)",
		},
		PAGESET_TYPE_MOBILE_10k: &PagesetTypeInfo{
			NumPages:                   10000,
			CSVSource:                  "csv/android-top-1m.csv",
			UserAgent:                  "mobile",
			CreatePagesetsTimeoutSecs:  60,
			CaptureArchivesTimeoutSecs: 300,
			CaptureSKPsTimeoutSecs:     300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 10K (with mobile user-agent)",
		},
		PAGESET_TYPE_DUMMY_1k: &PagesetTypeInfo{
			NumPages:                   1000,
			CSVSource:                  "csv/android-top-1m.csv",
			UserAgent:                  "mobile",
			CreatePagesetsTimeoutSecs:  60,
			CaptureArchivesTimeoutSecs: 300,
			CaptureSKPsTimeoutSecs:     300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 1K (used for testing, hidden from Runs History by default)",
		},
	}

	// Frontend constants below.
	SupportedBenchmarks = []string{
		BENCHMARK_RR,
		BENCHMARK_REPAINT,
		BENCHMARK_DRAW_PROPERTIES,
	}

	SupportedPlatformsToDesc = map[string]string{
		PLATFORM_LINUX:   "Linux (100 Ubuntu12.04 machines)",
		PLATFORM_ANDROID: "Android (100 N5 devices)",
	}

	SupportedPageSetsToDesc = map[string]string{
		PLATFORM_LINUX:   "Linux (100 Ubuntu12.04 machines)",
		PLATFORM_ANDROID: "Android (100 N5 devices)",
	}
)

func NumWorkers() int {
	return len(Slaves)
}
<commit_msg>[CT] Remove support for draw_properties from CT<commit_after>package util

import (
	"fmt"
	"path/filepath"
	"time"
)

const (
	NUM_WORKERS_PROD     int = 100
	MASTER_NAME              = "build101-m5"
	WORKER_NAME_TEMPLATE     = "build%d-m5"
	GS_HTTP_LINK             = "https://storage.cloud.google.com/"
	LOGS_LINK_PREFIX         = "http://uberchromegw.corp.google.com/i/skia-ct-worker"

	// File names and dir names.
	TIMESTAMP_FILE_NAME          = "TIMESTAMP"
	CHROMIUM_BUILDS_DIR_NAME     = "chromium_builds"
	PAGESETS_DIR_NAME            = "page_sets"
	WEB_ARCHIVES_DIR_NAME        = "webpage_archives"
	SKPS_DIR_NAME                = "skps"
	STORAGE_DIR_NAME             = "storage"
	REPO_DIR_NAME                = "skia-repo"
	TASKS_DIR_NAME               = "tasks"
	LUA_TASKS_DIR_NAME           = "lua_runs"
	BENCHMARK_TASKS_DIR_NAME     = "benchmark_runs"
	CHROMIUM_PERF_TASKS_DIR_NAME = "chromium_perf_runs"
	FIX_ARCHIVE_TASKS_DIR_NAME   = "fix_archive_runs"

	// Limit the number of times CT tries to get a remote file before giving up.
	MAX_URI_GET_TRIES = 4

	// Activity constants.
	ACTIVITY_CREATING_PAGESETS        = "CREATING_PAGESETS"
	ACTIVITY_CAPTURING_ARCHIVES       = "CAPTURING_ARCHIVES"
	ACTIVITY_CAPTURING_SKPS           = "CAPTURING_SKPS"
	ACTIVITY_RUNNING_LUA_SCRIPTS      = "RUNNING_LUA_SCRIPTS"
	ACTIVITY_RUNNING_CHROMIUM_PERF    = "RUNNING_CHROMIUM_PERF"
	ACTIVITY_RUNNING_SKIA_CORRECTNESS = "RUNNING_SKIA_CORRECTNESS"
	ACTIVITY_FIXING_ARCHIVES          = "FIXING_ARCHIVES"

	// Pageset types supported by CT.
	PAGESET_TYPE_ALL        = "All"
	PAGESET_TYPE_10k        = "10k"
	PAGESET_TYPE_MOBILE_10k = "Mobile10k"
	PAGESET_TYPE_DUMMY_1k   = "Dummy1k" // Used for testing.

	// Names of binaries executed by CT.
	BINARY_CHROME          = "chrome"
	BINARY_RECORD_WPR      = "record_wpr"
	BINARY_RUN_BENCHMARK   = "ct_run_benchmark"
	BINARY_GCLIENT         = "gclient"
	BINARY_MAKE            = "make"
	BINARY_LUA_PICTURES    = "lua_pictures"
	BINARY_ADB             = "adb"
	BINARY_GIT             = "git"
	BINARY_RENDER_PICTURES = "render_pictures"
	BINARY_MAIL            = "mail"
	BINARY_LUA             = "lua"

	// Platforms supported by CT.
	PLATFORM_ANDROID = "Android"
	PLATFORM_LINUX   = "Linux"

	// Benchmarks supported by CT.
	BENCHMARK_SKPICTURE_PRINTER = "skpicture_printer"
	BENCHMARK_RR                = "rasterize_and_record_micro"
	BENCHMARK_REPAINT           = "repaint"
	BENCHMARK_SMOOTHNESS        = "smoothness"

	// Logserver links. These are only accessible from Google corp.
	MASTER_LOGSERVER_LINK  = "http://uberchromegw.corp.google.com/i/skia-ct-master/"
	WORKERS_LOGSERVER_LINK = "http://uberchromegw.corp.google.com/i/skia-ct-master/all_logs"

	// Default browser args when running benchmarks.
	DEFAULT_BROWSER_ARGS = "--disable-setuid-sandbox --enable-threaded-compositing --enable-impl-side-painting"

	// Timeouts

	PKILL_TIMEOUT = 5 * time.Minute

	// util.SyncDir
	GIT_PULL_TIMEOUT     = 10 * time.Minute
	GCLIENT_SYNC_TIMEOUT = 15 * time.Minute

	// util.BuildSkiaTools
	MAKE_CLEAN_TIMEOUT = 5 * time.Minute
	MAKE_TOOLS_TIMEOUT = 5 * time.Minute

	// util.ResetCheckout
	GIT_RESET_TIMEOUT = 5 * time.Minute
	GIT_CLEAN_TIMEOUT = 5 * time.Minute
	// util.resetChromiumCheckout calls ResetCheckout three times.
	RESET_CHROMIUM_CHECKOUT_TIMEOUT = 3 * (GIT_RESET_TIMEOUT + GIT_CLEAN_TIMEOUT)

	// util.CreateChromiumBuild
	SYNC_SKIA_IN_CHROME_TIMEOUT   = 2 * time.Hour
	GIT_LS_REMOTE_TIMEOUT         = 5 * time.Minute
	GIT_APPLY_TIMEOUT             = 5 * time.Minute
	GOMA_CTL_RESTART_TIMEOUT      = 10 * time.Minute
	GYP_CHROMIUM_TIMEOUT          = 30 * time.Minute
	NINJA_TIMEOUT                 = 2 * time.Hour
	CREATE_CHROMIUM_BUILD_TIMEOUT = SYNC_SKIA_IN_CHROME_TIMEOUT + GIT_LS_REMOTE_TIMEOUT +
		// Three patches are applied when applyPatches is specified.
		3*GIT_APPLY_TIMEOUT +
		// The build steps are repeated twice when applyPatches is specified.
		2*(GOMA_CTL_RESTART_TIMEOUT+GYP_CHROMIUM_TIMEOUT+NINJA_TIMEOUT+
			RESET_CHROMIUM_CHECKOUT_TIMEOUT)

	// util.InstallChromeAPK
	ADB_INSTALL_TIMEOUT = 15 * time.Minute

	// Allow extra time for updating frontend and any other computation not included in the
	// worker timeouts.
	MASTER_SCRIPT_TIMEOUT_PADDING = 30 * time.Minute

	// Build Chromium Task
	GIT_LOG_TIMEOUT                      = 5 * time.Minute
	MASTER_SCRIPT_BUILD_CHROMIUM_TIMEOUT = CREATE_CHROMIUM_BUILD_TIMEOUT + GIT_LOG_TIMEOUT +
		MASTER_SCRIPT_TIMEOUT_PADDING

	// Capture Archives
	// Setting a 5 day timeout since it may take a while to capture 1M archives.
	CAPTURE_ARCHIVES_TIMEOUT               = 5 * 24 * time.Hour
	MASTER_SCRIPT_CAPTURE_ARCHIVES_TIMEOUT = CAPTURE_ARCHIVES_TIMEOUT +
		MASTER_SCRIPT_TIMEOUT_PADDING

	// Capture SKPs
	REMOVE_INVALID_SKPS_TIMEOUT = 3 * time.Hour
	// Setting a 2 day timeout since it may take a while to capture 1M SKPs.
	CAPTURE_SKPS_TIMEOUT               = 2 * 24 * time.Hour
	MASTER_SCRIPT_CAPTURE_SKPS_TIMEOUT = CAPTURE_SKPS_TIMEOUT + MASTER_SCRIPT_TIMEOUT_PADDING

	// Check Workers Health
	ADB_DEVICES_TIMEOUT          = 30 * time.Minute
	ADB_SHELL_UPTIME_TIMEOUT     = 30 * time.Minute
	CHECK_WORKERS_HEALTH_TIMEOUT = ADB_DEVICES_TIMEOUT + ADB_SHELL_UPTIME_TIMEOUT +
		MASTER_SCRIPT_TIMEOUT_PADDING

	// Create Pagesets
	// Setting a 4 hour timeout since it may take a while to upload page sets to
	// Google Storage when doing 10k page sets per worker.
	CREATE_PAGESETS_TIMEOUT               = 4 * time.Hour
	MASTER_SCRIPT_CREATE_PAGESETS_TIMEOUT = CREATE_PAGESETS_TIMEOUT +
		MASTER_SCRIPT_TIMEOUT_PADDING

	// Run Chromium Perf
	ADB_VERSION_TIMEOUT            = 5 * time.Minute
	ADB_ROOT_TIMEOUT               = 5 * time.Minute
	CSV_PIVOT_TABLE_MERGER_TIMEOUT = 10 * time.Minute
	REBOOT_TIMEOUT                 = 5 * time.Minute
	CSV_MERGER_TIMEOUT             = 1 * time.Hour
	CSV_COMPARER_TIMEOUT           = 2 * time.Hour
	// Setting a 1 day timeout since it may take a while run benchmarks with many
	// repeats.
	RUN_CHROMIUM_PERF_TIMEOUT = 1 * 24 * time.Hour
	// csv_merger runs once for nopatch and once for withpatch
	MASTER_SCRIPT_RUN_CHROMIUM_PERF_TIMEOUT = CREATE_CHROMIUM_BUILD_TIMEOUT + REBOOT_TIMEOUT +
		RUN_CHROMIUM_PERF_TIMEOUT + 2*CSV_MERGER_TIMEOUT + CSV_COMPARER_TIMEOUT +
		MASTER_SCRIPT_TIMEOUT_PADDING

	// Run Lua
	LUA_PICTURES_TIMEOUT          = 2 * time.Hour
	RUN_LUA_TIMEOUT               = 2 * time.Hour
	LUA_AGGREGATOR_TIMEOUT        = 1 * time.Hour
	MASTER_SCRIPT_RUN_LUA_TIMEOUT = RUN_LUA_TIMEOUT + LUA_AGGREGATOR_TIMEOUT +
		MASTER_SCRIPT_TIMEOUT_PADDING

	// Fix Archives
	// Setting a 1 day timeout since it may take a while to validate archives.
	FIX_ARCHIVES_TIMEOUT = 1 * 24 * time.Hour

	// Poller
	MAKE_ALL_TIMEOUT = 15 * time.Minute

	WEBHOOK_SALT_MSG = `For prod, set this file to the value of GCE metadata key webhook_request_salt or call webhook.MustInitRequestSaltFromMetadata() if running in GCE. For testing, run 'echo -n "notverysecret" | base64 -w 0 > /b/storage/webhook_salt.data' or call frontend.InitForTesting().`
)

type PagesetTypeInfo struct {
	NumPages                   int
	CSVSource                  string
	UserAgent                  string
	CaptureArchivesTimeoutSecs int
	CreatePagesetsTimeoutSecs  int
	CaptureSKPsTimeoutSecs     int
	RunChromiumPerfTimeoutSecs int
	Description                string
}

var (
	Master       = fmt.Sprintf(WORKER_NAME_TEMPLATE, 101)
	CtUser       = "chrome-bot"
	Slaves       = GetCTWorkersProd()
	GSBucketName = "cluster-telemetry"

	// Email address of cluster telemetry admins. They will be notified everytime
	// a task has started and completed.
	CtAdmins = []string{"rmistry@google.com"}

	// Names of local directories and files.
	StorageDir           = filepath.Join("/", "b", STORAGE_DIR_NAME)
	RepoDir              = filepath.Join("/", "b", REPO_DIR_NAME)
	GomaDir              = filepath.Join("/", "b", "build", "goma")
	ChromiumBuildsDir    = filepath.Join(StorageDir, CHROMIUM_BUILDS_DIR_NAME)
	ChromiumSrcDir       = filepath.Join(StorageDir, "chromium", "src")
	TelemetryBinariesDir = filepath.Join(ChromiumSrcDir, "tools", "perf")
	TelemetrySrcDir      = filepath.Join(ChromiumSrcDir, "tools", "telemetry")
	TaskFileDir          = filepath.Join(StorageDir, "current_task")
	ClientSecretPath     = filepath.Join(StorageDir, "client_secret.json")
	GSTokenPath          = filepath.Join(StorageDir, "google_storage_token.data")
	EmailTokenPath       = filepath.Join(StorageDir, "email.data")
	WebappPasswordPath   = filepath.Join(StorageDir, "webapp.data")
	// Salt used to authenticate webhook requests, base64-encoded. See WEBHOOK_SALT_MSG.
	WebhookRequestSaltPath = filepath.Join(StorageDir, "webhook_salt.data")
	PagesetsDir            = filepath.Join(StorageDir, PAGESETS_DIR_NAME)
	WebArchivesDir         = filepath.Join(StorageDir, WEB_ARCHIVES_DIR_NAME)
	SkpsDir                = filepath.Join(StorageDir, SKPS_DIR_NAME)
	GLogDir                = filepath.Join(StorageDir, "glog")
	ApkName                = "ChromePublic.apk"
	SkiaTreeDir            = filepath.Join(RepoDir, "trunk")
	CtTreeDir              = filepath.Join(RepoDir, "go", "src", "go.skia.org", "infra", "ct")

	// Names of remote directories and files.
	LuaRunsDir          = filepath.Join(TASKS_DIR_NAME, LUA_TASKS_DIR_NAME)
	BenchmarkRunsDir    = filepath.Join(TASKS_DIR_NAME, BENCHMARK_TASKS_DIR_NAME)
	ChromiumPerfRunsDir = filepath.Join(TASKS_DIR_NAME, CHROMIUM_PERF_TASKS_DIR_NAME)
	FixArchivesRunsDir  = filepath.Join(TASKS_DIR_NAME, FIX_ARCHIVE_TASKS_DIR_NAME)

	// Information about the different CT benchmarks.
	BenchmarksToPagesetName = map[string]string{
		BENCHMARK_SKPICTURE_PRINTER: "SkpicturePrinter",
		BENCHMARK_RR:                "RasterizeAndRecordMicroCTPages",
		BENCHMARK_REPAINT:           "RepaintCTPages",
		BENCHMARK_SMOOTHNESS:        "SmoothnessCTPages",
	}

	// Information about the different CT pageset types.
	PagesetTypeToInfo = map[string]*PagesetTypeInfo{
		PAGESET_TYPE_ALL: &PagesetTypeInfo{
			NumPages:                   1000000,
			CSVSource:                  "csv/top-1m.csv",
			UserAgent:                  "desktop",
			CreatePagesetsTimeoutSecs:  60,
			CaptureArchivesTimeoutSecs: 300,
			CaptureSKPsTimeoutSecs:     300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 1M (with desktop user-agent)",
		},
		PAGESET_TYPE_10k: &PagesetTypeInfo{
			NumPages:                   10000,
			CSVSource:                  "csv/top-1m.csv",
			UserAgent:                  "desktop",
			CreatePagesetsTimeoutSecs:  60,
			CaptureArchivesTimeoutSecs: 300,
			CaptureSKPsTimeoutSecs:     300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 10K (with desktop user-agent)",
		},
		PAGESET_TYPE_MOBILE_10k: &PagesetTypeInfo{
			NumPages:                   10000,
			CSVSource:                  "csv/android-top-1m.csv",
			UserAgent:                  "mobile",
			CreatePagesetsTimeoutSecs:  60,
			CaptureArchivesTimeoutSecs: 300,
			CaptureSKPsTimeoutSecs:     300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 10K (with mobile user-agent)",
		},
		PAGESET_TYPE_DUMMY_1k: &PagesetTypeInfo{
			NumPages:                   1000,
			CSVSource:                  "csv/android-top-1m.csv",
			UserAgent:                  "mobile",
			CreatePagesetsTimeoutSecs:  60,
			CaptureArchivesTimeoutSecs: 300,
			CaptureSKPsTimeoutSecs:     300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 1K (used for testing, hidden from Runs History by default)",
		},
	}

	// Frontend constants below.
	SupportedBenchmarks = []string{
		BENCHMARK_RR,
		BENCHMARK_REPAINT,
	}

	SupportedPlatformsToDesc = map[string]string{
		PLATFORM_LINUX:   "Linux (100 Ubuntu12.04 machines)",
		PLATFORM_ANDROID: "Android (100 N5 devices)",
	}

	SupportedPageSetsToDesc = map[string]string{
		PLATFORM_LINUX:   "Linux (100 Ubuntu12.04 machines)",
		PLATFORM_ANDROID: "Android (100 N5 devices)",
	}
)

func NumWorkers() int {
	return len(Slaves)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	restful "github.com/emicklei/go-restful"
	"github.com/emicklei/go-restful/swagger"
	"github.com/garyburd/redigo/redis"
	"github.com/the42/ogdat/database"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

const AppID = "5bcbfc24-8e7e-4105-99c4-dd47e7e5094a"
const watcherappid = "a6545f8f-e0c9-4917-83c7-3e47bd1e0247"

var logger *log.Logger

type analyser struct {
	dbcon analyserdb
	pool  *redis.Pool
}

func NewAnalyser(dbcon *sql.DB, pool *redis.Pool) *analyser {
	analyser := &analyser{dbcon: analyserdb{DBConn: database.DBConn{Appid: AppID, DBer: dbcon}}, pool: pool}
	return analyser
}

func onlyweb() bool {
	boolval, err := strconv.ParseBool(os.Getenv("ONLYWEB"))
	if err != nil {
		return false
	}
	return boolval
}

func getredisconnect() string {
	const redisurl = "REDISCLOUD_URL"
	const redisdb = "ANALYSER_REDISDB"

	return os.Getenv(redisurl) + "/" + os.Getenv(redisdb)
}

func getheartbeatinterval() int {

	if i, err := strconv.Atoi(os.Getenv("HEARTBEAT_INTERVAL")); err == nil {
		return i
	}
	return 60 // Minutes
}

func heartbeat(interval int) chan bool {
	retchan := make(chan bool)
	go func() {
		for {
			dbconn, err := database.GetDatabaseConnection()
			if err != nil {
				logger.Panicln(err)
			}
			db := &database.DBConn{DBer: dbconn, Appid: AppID}
			if err := db.HeartBeat(); err != nil {
				logger.Panicln(err)
			}
			dbconn.Close()
			logger.Printf("Watchdog beating every %d minute\n", interval)
			retchan <- true
			time.Sleep(time.Duration(interval) * time.Minute)
		}
	}()
	return retchan
}

func main() {
	dbcon, err := database.GetDatabaseConnection()
	if err != nil {
		logger.Panicln(err)
	}
	defer dbcon.Close()
	analyser := NewAnalyser(dbcon, redis.NewPool(func() (redis.Conn, error) { return database.GetRedisConnection(getredisconnect()) }, 10))

	var datachange, urlchange chan []byte
	var heartbeatchannel chan bool

	if !onlyweb() {
		heartbeatchannel = heartbeat(getheartbeatinterval())

		<-heartbeatchannel // Wait for the first heartbeat, so the logging in the database is properly set up
		if err := analyser.populatedatasetinfo(); err != nil {
			logger.Panicln(err)
		}
		datachange = analyser.listenredischannel(watcherappid + ":DataChange")
		urlchange = analyser.listenredischannel(watcherappid + ":UrlChange")
	}

	restful.DefaultResponseMimeType = restful.MIME_JSON
	restful.Add(NewAnalyseOGDATRESTService(analyser))

	config := swagger.Config{
		WebServicesUrl:  "http://localhost:8080",
		ApiPath:         "/apidoc",
		SwaggerPath:     "/doc/v1/",
		SwaggerFilePath: "swagger-ui/dist/",
		WebServices:     restful.RegisteredWebServices()} // you control what services are visible
	swagger.InstallSwaggerService(config)

	go logger.Fatal(http.ListenAndServe(":8080", nil))

	if !onlyweb() {
		for {
			select {
			case <-urlchange:
			case <-datachange:
				if err := analyser.populatedatasetinfo(); err != nil {
					logger.Panicln(err)
				}
			case <-heartbeatchannel:
				logger.Println("Idle")
			}
		}
	}
}

func init() {
	logger = log.New(os.Stderr, filepath.Base(os.Args[0])+": ", log.LstdFlags)
}
<commit_msg>gzip-encode the response<commit_after>package main

import (
	"database/sql"
	restful "github.com/emicklei/go-restful"
	"github.com/emicklei/go-restful/swagger"
	"github.com/garyburd/redigo/redis"
	"github.com/the42/ogdat/database"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

const AppID = "5bcbfc24-8e7e-4105-99c4-dd47e7e5094a"
const watcherappid = "a6545f8f-e0c9-4917-83c7-3e47bd1e0247"

var logger *log.Logger

type analyser struct {
	dbcon analyserdb
	pool  *redis.Pool
}

func NewAnalyser(dbcon *sql.DB, pool *redis.Pool) *analyser {
	analyser := &analyser{dbcon: analyserdb{DBConn: database.DBConn{Appid: AppID, DBer: dbcon}}, pool: pool}
	return analyser
}

func onlyweb() bool {
	boolval, err := strconv.ParseBool(os.Getenv("ONLYWEB"))
	if err != nil {
		return false
	}
	return boolval
}

func getredisconnect() string {
	const redisurl = "REDISCLOUD_URL"
	const redisdb = "ANALYSER_REDISDB"

	return os.Getenv(redisurl) + "/" + os.Getenv(redisdb)
}

func getheartbeatinterval() int {

	if i, err := strconv.Atoi(os.Getenv("HEARTBEAT_INTERVAL")); err == nil {
		return i
	}
	return 60 // Minutes
}

func heartbeat(interval int) chan bool {
	retchan := make(chan bool)
	go func() {
		for {
			dbconn, err := database.GetDatabaseConnection()
			if err != nil {
				logger.Panicln(err)
			}
			db := &database.DBConn{DBer: dbconn, Appid: AppID}
			if err := db.HeartBeat(); err != nil {
				logger.Panicln(err)
			}
			dbconn.Close()
			logger.Printf("Watchdog beating every %d minute\n", interval)
			retchan <- true
			time.Sleep(time.Duration(interval) * time.Minute)
		}
	}()
	return retchan
}

func main() {
	dbcon, err := database.GetDatabaseConnection()
	if err != nil {
		logger.Panicln(err)
	}
	defer dbcon.Close()
	analyser := NewAnalyser(dbcon, redis.NewPool(func() (redis.Conn, error) { return database.GetRedisConnection(getredisconnect()) }, 10))

	var datachange, urlchange chan []byte
	var heartbeatchannel chan bool

	if !onlyweb() {
		heartbeatchannel = heartbeat(getheartbeatinterval())

		<-heartbeatchannel // Wait for the first heartbeat, so the logging in the database is properly set up
		if err := analyser.populatedatasetinfo(); err != nil {
			logger.Panicln(err)
		}
		datachange = analyser.listenredischannel(watcherappid + ":DataChange")
		urlchange = analyser.listenredischannel(watcherappid + ":UrlChange")
	}

	restful.DefaultResponseMimeType = restful.MIME_JSON
	restful.EnableContentEncoding = true
	restful.Add(NewAnalyseOGDATRESTService(analyser))

	config := swagger.Config{
		WebServicesUrl:  "http://localhost:8080",
		ApiPath:         "/apidoc",
		SwaggerPath:     "/doc/v1/",
		SwaggerFilePath: "swagger-ui/dist/",
		WebServices:     restful.RegisteredWebServices()} // you control what services are visible
	swagger.InstallSwaggerService(config)

	go logger.Fatal(http.ListenAndServe(":8080", nil))

	if !onlyweb() {
		for {
			select {
			case <-urlchange:
			case <-datachange:
				if err := analyser.populatedatasetinfo(); err != nil {
					logger.Panicln(err)
				}
			case <-heartbeatchannel:
				logger.Println("Idle")
			}
		}
	}
}

func init() {
	logger = log.New(os.Stderr, filepath.Base(os.Args[0])+": ", log.LstdFlags)
}
<|endoftext|>


<commit_before>package analyzer

import (
	"fmt"
	"time"

	"github.com/m-lab/signal-searcher/sequencer"
)

// An Incident represents a piece of a timeseries that contains a user-visible
// problem.
type Incident struct {
	Start, End    time.Time
	AffectedCount int
}

// URL converts an incident (along with provided Metadata) into a viz URL.
func (i *Incident) URL(m sequencer.Meta) string {
	twelveBeforeStart := i.Start.AddDate(-1, 0, 0)
	return fmt.Sprintf(
		"http://viz.measurementlab.net/location/%s?aggr=month&isps=%s&start=%s&end=%s",
		m.Loc, m.ASN, twelveBeforeStart.Format("2006-01-02"), i.End.Format("2006-01-02"))
}

type arrayIncident struct {
	start, end int
}

func mergeArrayIncidents(a []arrayIncident) (merged []arrayIncident) {
	if len(a) <= 1 {
		return a
	}
	current := a[0]
	for i := 1; i < len(a); i++ {
		if current.end+1 == a[i].end {
			current.end = a[i].end
		} else {
			merged = append(merged, current)
			current = a[i]
		}
	}
	merged = append(merged, current)
	return
}

// FindPerformanceDrops discovers time periods of a year or greater where
// performance showed more than a 30% average drop.
func FindPerformanceDrops(s *sequencer.Sequence) []Incident {
	dates, data := s.SortedSlices()
	var previous, current sequencer.Datum
	for i := 0; i < 12; i++ {
		previous.Download += data[i].Download
	}
	for i := 12; i < 24; i++ {
		current.Download += data[i].Download
	}
	var arrayIncidents []arrayIncident
	for i := 24; i < len(data); i++ {
		// Update the running sums
		previous.Download = previous.Download - data[i-24].Download + data[i-12].Download
		current.Download = current.Download - data[i-12].Download + data[i].Download

		if previous.Download*.7 > current.Download {
			arrayIncidents = append(arrayIncidents, arrayIncident{start: i - 12, end: i})
		}
	}
	arrayIncidents = mergeArrayIncidents(arrayIncidents)
	incidents := []Incident{}
	for _, ai := range arrayIncidents {
		newIncident := Incident{Start: dates[ai.start], End: dates[ai.end]}
		for i := ai.start; i < ai.end; i++ {
			newIncident.AffectedCount += data[i].Count
		}
		incidents = append(incidents, newIncident)
	}
	return incidents
}
<commit_msg>http -> https<commit_after>package analyzer

import (
	"fmt"
	"time"

	"github.com/m-lab/signal-searcher/sequencer"
)

// An Incident represents a piece of a timeseries that contains a user-visible
// problem.
type Incident struct {
	Start, End    time.Time
	AffectedCount int
}

// URL converts an incident (along with provided Metadata) into a viz URL.
func (i *Incident) URL(m sequencer.Meta) string {
	twelveBeforeStart := i.Start.AddDate(-1, 0, 0)
	return fmt.Sprintf(
		"https://viz.measurementlab.net/location/%s?aggr=month&isps=%s&start=%s&end=%s",
		m.Loc, m.ASN, twelveBeforeStart.Format("2006-01-02"), i.End.Format("2006-01-02"))
}

type arrayIncident struct {
	start, end int
}

func mergeArrayIncidents(a []arrayIncident) (merged []arrayIncident) {
	if len(a) <= 1 {
		return a
	}
	current := a[0]
	for i := 1; i < len(a); i++ {
		if current.end+1 == a[i].end {
			current.end = a[i].end
		} else {
			merged = append(merged, current)
			current = a[i]
		}
	}
	merged = append(merged, current)
	return
}

// FindPerformanceDrops discovers time periods of a year or greater where
// performance showed more than a 30% average drop.
func FindPerformanceDrops(s *sequencer.Sequence) []Incident {
	dates, data := s.SortedSlices()
	var previous, current sequencer.Datum
	for i := 0; i < 12; i++ {
		previous.Download += data[i].Download
	}
	for i := 12; i < 24; i++ {
		current.Download += data[i].Download
	}
	var arrayIncidents []arrayIncident
	for i := 24; i < len(data); i++ {
		// Update the running sums
		previous.Download = previous.Download - data[i-24].Download + data[i-12].Download
		current.Download = current.Download - data[i-12].Download + data[i].Download

		if previous.Download*.7 > current.Download {
			arrayIncidents = append(arrayIncidents, arrayIncident{start: i - 12, end: i})
		}
	}
	arrayIncidents = mergeArrayIncidents(arrayIncidents)
	incidents := []Incident{}
	for _, ai := range arrayIncidents {
		newIncident := Incident{Start: dates[ai.start], End: dates[ai.end]}
		for i := ai.start; i < ai.end; i++ {
			newIncident.AffectedCount += data[i].Count
		}
		incidents = append(incidents, newIncident)
	}
	return incidents
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"gotest.tools/assert"

	"knative.dev/client/pkg/util"
)

func SubscriptionCreate(r *KnRunResultCollector, sname string, args ...string) {
	cmd := []string{"subscription", "create", sname}
	cmd = append(cmd, args...)
	out := r.KnTest().Kn().Run(cmd...)
	r.AssertNoError(out)
	assert.Check(r.T(), util.ContainsAllIgnoreCase(out.Stdout, "subscription", sname, "created"))
}

func SubscriptionList(r *KnRunResultCollector, args ...string) string {
	cmd := []string{"subscription", "list"}
	cmd = append(cmd, args...)
	out := r.KnTest().Kn().Run(cmd...)
	r.AssertNoError(out)
	return out.Stdout
}

func SubscriptionDescribe(r *KnRunResultCollector, sname string, args ...string) string {
	cmd := []string{"subscription", "describe", sname}
	cmd = append(cmd, args...)
	out := r.KnTest().Kn().Run(cmd...)
	r.AssertNoError(out)
	return out.Stdout
}

func SubscriptionDelete(r *KnRunResultCollector, sname string) {
	out := r.KnTest().Kn().Run("subscription", "delete", sname)
	r.AssertNoError(out)
	assert.Check(r.T(), util.ContainsAllIgnoreCase(out.Stdout, "subscription", sname, "deleted"))
}

func SubscriptionUpdate(r *KnRunResultCollector, sname string, args ...string) {
	cmd := []string{"subscription", "update", sname}
	cmd = append(cmd, args...)
	out := r.KnTest().Kn().Run(cmd...)
	r.AssertNoError(out)
	assert.Check(r.T(), util.ContainsAllIgnoreCase(out.Stdout, "subscription", sname, "updated"))
}
<commit_msg>fix(e2e): Let the subscription and related resource reconcile (#1044)<commit_after>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"time"

	"gotest.tools/assert"

	"knative.dev/client/pkg/util"
)

func SubscriptionCreate(r *KnRunResultCollector, sname string, args ...string) {
	cmd := []string{"subscription", "create", sname}
	cmd = append(cmd, args...)
	out := r.KnTest().Kn().Run(cmd...)
	r.AssertNoError(out)
	assert.Check(r.T(), util.ContainsAllIgnoreCase(out.Stdout, "subscription", sname, "created"))
	// let the subscription and related resource reconcile
	time.Sleep(time.Second * 5)
}

func SubscriptionList(r *KnRunResultCollector, args ...string) string {
	cmd := []string{"subscription", "list"}
	cmd = append(cmd, args...)
	out := r.KnTest().Kn().Run(cmd...)
	r.AssertNoError(out)
	return out.Stdout
}

func SubscriptionDescribe(r *KnRunResultCollector, sname string, args ...string) string {
	cmd := []string{"subscription", "describe", sname}
	cmd = append(cmd, args...)
	out := r.KnTest().Kn().Run(cmd...)
	r.AssertNoError(out)
	return out.Stdout
}

func SubscriptionDelete(r *KnRunResultCollector, sname string) {
	out := r.KnTest().Kn().Run("subscription", "delete", sname)
	r.AssertNoError(out)
	assert.Check(r.T(), util.ContainsAllIgnoreCase(out.Stdout, "subscription", sname, "deleted"))
}

func SubscriptionUpdate(r *KnRunResultCollector, sname string, args ...string) {
	cmd := []string{"subscription", "update", sname}
	cmd = append(cmd, args...)
	out := r.KnTest().Kn().Run(cmd...)
	r.AssertNoError(out)
	assert.Check(r.T(), util.ContainsAllIgnoreCase(out.Stdout, "subscription", sname, "updated"))
	// let the subscription and related resource reconcile
	time.Sleep(time.Second * 5)
}
<|endoftext|>


<commit_before>package ipc

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"syscall"

	"github.com/docker/docker-registry/storagedriver"
	"github.com/docker/libchan"
	"github.com/docker/libchan/spdy"
)

// StorageDriverExecutablePrefix is the prefix which the IPC storage driver
// loader expects driver executables to begin with. For example, the s3 driver
// should be named "registry-storagedriver-s3".
const StorageDriverExecutablePrefix = "registry-storagedriver-"

// StorageDriverClient is a storagedriver.StorageDriver implementation using a
// managed child process communicating over IPC using libchan with a unix domain
// socket
type StorageDriverClient struct {
	subprocess *exec.Cmd
	exitChan   chan error
	exitErr    error
	stopChan   chan struct{}
	socket     *os.File
	transport  *spdy.Transport
	sender     libchan.Sender
	version    storagedriver.Version
}

// NewDriverClient constructs a new out-of-process storage driver using the
// driver name and configuration parameters
// A user must call Start on this driver client before remote method calls can
// be made
//
// Looks for drivers in the following locations in order:
// - Storage drivers directory (to be determined, yet not implemented)
// - $GOPATH/bin
// - $PATH
func NewDriverClient(name string, parameters map[string]string) (*StorageDriverClient, error) {
	paramsBytes, err := json.Marshal(parameters)
	if err != nil {
		return nil, err
	}

	driverExecName := StorageDriverExecutablePrefix + name
	driverPath, err := exec.LookPath(driverExecName)
	if err != nil {
		return nil, err
	}

	command := exec.Command(driverPath, string(paramsBytes))

	return &StorageDriverClient{
		subprocess: command,
	}, nil
}

// Start starts the designated child process storage driver and binds a socket
// to this process for IPC method calls
func (driver *StorageDriverClient) Start() error {
	driver.exitErr = nil
	driver.exitChan = make(chan error)
	driver.stopChan = make(chan struct{})

	fileDescriptors, err := syscall.Socketpair(syscall.AF_LOCAL, syscall.SOCK_STREAM, 0)
	if err != nil {
		return err
	}

	childSocket := os.NewFile(uintptr(fileDescriptors[0]), "childSocket")
	driver.socket = os.NewFile(uintptr(fileDescriptors[1]), "parentSocket")

	driver.subprocess.Stdout = os.Stdout
	driver.subprocess.Stderr = os.Stderr
	driver.subprocess.ExtraFiles = []*os.File{childSocket}

	if err = driver.subprocess.Start(); err != nil {
		driver.Stop()
		return err
	}

	go driver.handleSubprocessExit()

	if err = childSocket.Close(); err != nil {
		driver.Stop()
		return err
	}

	connection, err := net.FileConn(driver.socket)
	if err != nil {
		driver.Stop()
		return err
	}
	driver.transport, err = spdy.NewClientTransport(connection)
	if err != nil {
		driver.Stop()
		return err
	}
	driver.sender, err = driver.transport.NewSendChannel()
	if err != nil {
		driver.Stop()
		return err
	}

	// Check the driver's version to determine compatibility
	receiver, remoteSender := libchan.Pipe()
	err = driver.sender.Send(&Request{Type: "Version", ResponseChannel: remoteSender})
	if err != nil {
		driver.Stop()
		return err
	}

	var response VersionResponse
	err = receiver.Receive(&response)
	if err != nil {
		driver.Stop()
		return err
	}

	if response.Error != nil {
		return response.Error.Unwrap()
	}

	driver.version = response.Version

	if driver.version.Major() != storagedriver.CurrentVersion.Major() || driver.version.Minor() > storagedriver.CurrentVersion.Minor() {
		return IncompatibleVersionError{driver.version}
	}

	return nil
}

// Stop stops the child process storage driver
// storagedriver.StorageDriver methods called after Stop will fail
func (driver *StorageDriverClient) Stop() error {
	var closeSenderErr, closeTransportErr, closeSocketErr, killErr error

	if driver.sender != nil {
		closeSenderErr = driver.sender.Close()
	}
	if driver.transport != nil {
		closeTransportErr = driver.transport.Close()
	}
	if driver.socket != nil {
		closeSocketErr = driver.socket.Close()
	}
	if driver.subprocess != nil {
		killErr = driver.subprocess.Process.Kill()
	}
	if driver.stopChan != nil {
		driver.stopChan <- struct{}{}
		close(driver.stopChan)
	}

	if closeSenderErr != nil {
		return closeSenderErr
	} else if closeTransportErr != nil {
		return closeTransportErr
	} else if closeSocketErr != nil {
		return closeSocketErr
	}

	return killErr
}

// Implement the storagedriver.StorageDriver interface over IPC

// GetContent retrieves the content stored at "path" as a []byte.
func (driver *StorageDriverClient) GetContent(path string) ([]byte, error) {
	if err := driver.exited(); err != nil {
		return nil, err
	}

	receiver, remoteSender := libchan.Pipe()

	params := map[string]interface{}{"Path": path}
	err := driver.sender.Send(&Request{Type: "GetContent", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return nil, err
	}

	response := new(ReadStreamResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return nil, err
	}

	if response.Error != nil {
		return nil, response.Error.Unwrap()
	}

	defer response.Reader.Close()
	contents, err := ioutil.ReadAll(response.Reader)
	if err != nil {
		return nil, err
	}
	return contents, nil
}

// PutContent stores the []byte content at a location designated by "path".
func (driver *StorageDriverClient) PutContent(path string, contents []byte) error {
	if err := driver.exited(); err != nil {
		return err
	}

	receiver, remoteSender := libchan.Pipe()

	params := map[string]interface{}{"Path": path, "Reader": ioutil.NopCloser(bytes.NewReader(contents))}
	err := driver.sender.Send(&Request{Type: "PutContent", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return err
	}

	response := new(WriteStreamResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return err
	}

	if response.Error != nil {
		return response.Error.Unwrap()
	}

	return nil
}

// ReadStream retrieves an io.ReadCloser for the content stored at "path" with a
// given byte offset.
func (driver *StorageDriverClient) ReadStream(path string, offset uint64) (io.ReadCloser, error) {
	if err := driver.exited(); err != nil {
		return nil, err
	}

	receiver, remoteSender := libchan.Pipe()
	params := map[string]interface{}{"Path": path, "Offset": offset}
	err := driver.sender.Send(&Request{Type: "ReadStream", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return nil, err
	}

	response := new(ReadStreamResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return nil, err
	}

	if response.Error != nil {
		return nil, response.Error.Unwrap()
	}

	return response.Reader, nil
}

// WriteStream stores the contents of the provided io.ReadCloser at a location
// designated by the given path.
func (driver *StorageDriverClient) WriteStream(path string, offset, size uint64, reader io.ReadCloser) error {
	if err := driver.exited(); err != nil {
		return err
	}

	receiver, remoteSender := libchan.Pipe()
	params := map[string]interface{}{"Path": path, "Offset": offset, "Size": size, "Reader": reader}
	err := driver.sender.Send(&Request{Type: "WriteStream", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return err
	}

	response := new(WriteStreamResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return err
	}

	if response.Error != nil {
		return response.Error.Unwrap()
	}

	return nil
}

// CurrentSize retrieves the curernt size in bytes of the object at the given
// path.
func (driver *StorageDriverClient) CurrentSize(path string) (uint64, error) {
	if err := driver.exited(); err != nil {
		return 0, err
	}

	receiver, remoteSender := libchan.Pipe()
	params := map[string]interface{}{"Path": path}
	err := driver.sender.Send(&Request{Type: "CurrentSize", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return 0, err
	}

	response := new(CurrentSizeResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return 0, err
	}

	if response.Error != nil {
		return 0, response.Error.Unwrap()
	}

	return response.Position, nil
}

// List returns a list of the objects that are direct descendants of the given
// path.
func (driver *StorageDriverClient) List(path string) ([]string, error) {
	if err := driver.exited(); err != nil {
		return nil, err
	}

	receiver, remoteSender := libchan.Pipe()
	params := map[string]interface{}{"Path": path}
	err := driver.sender.Send(&Request{Type: "List", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return nil, err
	}

	response := new(ListResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return nil, err
	}

	if response.Error != nil {
		return nil, response.Error.Unwrap()
	}

	return response.Keys, nil
}

// Move moves an object stored at sourcePath to destPath, removing the original
// object.
func (driver *StorageDriverClient) Move(sourcePath string, destPath string) error {
	if err := driver.exited(); err != nil {
		return err
	}

	receiver, remoteSender := libchan.Pipe()
	params := map[string]interface{}{"SourcePath": sourcePath, "DestPath": destPath}
	err := driver.sender.Send(&Request{Type: "Move", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return err
	}

	response := new(MoveResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return err
	}

	if response.Error != nil {
		return response.Error.Unwrap()
	}

	return nil
}

// Delete recursively deletes all objects stored at "path" and its subpaths.
func (driver *StorageDriverClient) Delete(path string) error {
	if err := driver.exited(); err != nil {
		return err
	}

	receiver, remoteSender := libchan.Pipe()
	params := map[string]interface{}{"Path": path}
	err := driver.sender.Send(&Request{Type: "Delete", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return err
	}

	response := new(DeleteResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return err
	}

	if response.Error != nil {
		return response.Error.Unwrap()
	}

	return nil
}

// handleSubprocessExit populates the exit channel until we have explicitly
// stopped the storage driver subprocess
// Requests can select on driver.exitChan and response receiving and not hang if
// the process exits
func (driver *StorageDriverClient) handleSubprocessExit() {
	exitErr := driver.subprocess.Wait()
	if exitErr == nil {
		exitErr = fmt.Errorf("Storage driver subprocess already exited cleanly")
	} else {
		exitErr = fmt.Errorf("Storage driver subprocess exited with error: %s", exitErr)
	}

	driver.exitErr = exitErr

	for {
		select {
		case driver.exitChan <- exitErr:
		case <-driver.stopChan:
			close(driver.exitChan)
			return
		}
	}
}

// receiveResponse populates the response value with the next result from the
// given receiver, or returns an error if receiving failed or the driver has
// stopped
func (driver *StorageDriverClient) receiveResponse(receiver libchan.Receiver, response interface{}) error {
	receiveChan := make(chan error, 1)
	go func(receiver libchan.Receiver, receiveChan chan<- error) {
		defer close(receiveChan)
		receiveChan <- receiver.Receive(response)
	}(receiver, receiveChan)

	var err error
	var ok bool
	select {
	case err = <-receiveChan:
	case err, ok = <-driver.exitChan:
		go func(receiveChan <-chan error) {
			<-receiveChan
		}(receiveChan)
		if !ok {
			err = driver.exitErr
		}
	}

	return err
}

// exited returns an exit error if the driver has exited or nil otherwise
func (driver *StorageDriverClient) exited() error {
	select {
	case err, ok := <-driver.exitChan:
		if !ok {
			return driver.exitErr
		}
		return err
	default:
		return nil
	}
}
<commit_msg>[IPC] Tiny cleaning<commit_after>package ipc

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"syscall"

	"github.com/docker/docker-registry/storagedriver"
	"github.com/docker/libchan"
	"github.com/docker/libchan/spdy"
)

// StorageDriverExecutablePrefix is the prefix which the IPC storage driver
// loader expects driver executables to begin with. For example, the s3 driver
// should be named "registry-storagedriver-s3".
const StorageDriverExecutablePrefix = "registry-storagedriver-"

// StorageDriverClient is a storagedriver.StorageDriver implementation using a
// managed child process communicating over IPC using libchan with a unix domain
// socket
type StorageDriverClient struct {
	subprocess *exec.Cmd
	exitChan   chan error
	exitErr    error
	stopChan   chan struct{}
	socket     *os.File
	transport  *spdy.Transport
	sender     libchan.Sender
	version    storagedriver.Version
}

// NewDriverClient constructs a new out-of-process storage driver using the
// driver name and configuration parameters
// A user must call Start on this driver client before remote method calls can
// be made
//
// Looks for drivers in the following locations in order:
// - Storage drivers directory (to be determined, yet not implemented)
// - $GOPATH/bin
// - $PATH
func NewDriverClient(name string, parameters map[string]string) (*StorageDriverClient, error) {
	paramsBytes, err := json.Marshal(parameters)
	if err != nil {
		return nil, err
	}

	driverExecName := StorageDriverExecutablePrefix + name
	driverPath, err := exec.LookPath(driverExecName)
	if err != nil {
		return nil, err
	}

	command := exec.Command(driverPath, string(paramsBytes))

	return &StorageDriverClient{
		subprocess: command,
	}, nil
}

// Start starts the designated child process storage driver and binds a socket
// to this process for IPC method calls
func (driver *StorageDriverClient) Start() error {
	driver.exitErr = nil
	driver.exitChan = make(chan error)
	driver.stopChan = make(chan struct{})

	fileDescriptors, err := syscall.Socketpair(syscall.AF_LOCAL, syscall.SOCK_STREAM, 0)
	if err != nil {
		return err
	}

	childSocket := os.NewFile(uintptr(fileDescriptors[0]), "childSocket")
	driver.socket = os.NewFile(uintptr(fileDescriptors[1]), "parentSocket")

	driver.subprocess.Stdout = os.Stdout
	driver.subprocess.Stderr = os.Stderr
	driver.subprocess.ExtraFiles = []*os.File{childSocket}

	if err = driver.subprocess.Start(); err != nil {
		driver.Stop()
		return err
	}

	go driver.handleSubprocessExit()

	if err = childSocket.Close(); err != nil {
		driver.Stop()
		return err
	}

	connection, err := net.FileConn(driver.socket)
	if err != nil {
		driver.Stop()
		return err
	}
	driver.transport, err = spdy.NewClientTransport(connection)
	if err != nil {
		driver.Stop()
		return err
	}
	driver.sender, err = driver.transport.NewSendChannel()
	if err != nil {
		driver.Stop()
		return err
	}

	// Check the driver's version to determine compatibility
	receiver, remoteSender := libchan.Pipe()
	err = driver.sender.Send(&Request{Type: "Version", ResponseChannel: remoteSender})
	if err != nil {
		driver.Stop()
		return err
	}

	var response VersionResponse
	err = receiver.Receive(&response)
	if err != nil {
		driver.Stop()
		return err
	}

	if response.Error != nil {
		return response.Error.Unwrap()
	}

	driver.version = response.Version

	if driver.version.Major() != storagedriver.CurrentVersion.Major() || driver.version.Minor() > storagedriver.CurrentVersion.Minor() {
		return IncompatibleVersionError{driver.version}
	}

	return nil
}

// Stop stops the child process storage driver
// storagedriver.StorageDriver methods called after Stop will fail
func (driver *StorageDriverClient) Stop() error {
	var closeSenderErr, closeTransportErr, closeSocketErr, killErr error

	if driver.sender != nil {
		closeSenderErr = driver.sender.Close()
	}
	if driver.transport != nil {
		closeTransportErr = driver.transport.Close()
	}
	if driver.socket != nil {
		closeSocketErr = driver.socket.Close()
	}
	if driver.subprocess != nil {
		killErr = driver.subprocess.Process.Kill()
	}
	if driver.stopChan != nil {
		driver.stopChan <- struct{}{}
		close(driver.stopChan)
	}

	if closeSenderErr != nil {
		return closeSenderErr
	} else if closeTransportErr != nil {
		return closeTransportErr
	} else if closeSocketErr != nil {
		return closeSocketErr
	}

	return killErr
}

// Implement the storagedriver.StorageDriver interface over IPC

// GetContent retrieves the content stored at "path" as a []byte.
func (driver *StorageDriverClient) GetContent(path string) ([]byte, error) {
	if err := driver.exited(); err != nil {
		return nil, err
	}

	receiver, remoteSender := libchan.Pipe()

	params := map[string]interface{}{"Path": path}
	err := driver.sender.Send(&Request{Type: "GetContent", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return nil, err
	}

	response := new(ReadStreamResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return nil, err
	}

	if response.Error != nil {
		return nil, response.Error.Unwrap()
	}

	defer response.Reader.Close()
	contents, err := ioutil.ReadAll(response.Reader)
	if err != nil {
		return nil, err
	}
	return contents, nil
}

// PutContent stores the []byte content at a location designated by "path".
func (driver *StorageDriverClient) PutContent(path string, contents []byte) error {
	if err := driver.exited(); err != nil {
		return err
	}

	receiver, remoteSender := libchan.Pipe()

	params := map[string]interface{}{"Path": path, "Reader": ioutil.NopCloser(bytes.NewReader(contents))}
	err := driver.sender.Send(&Request{Type: "PutContent", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return err
	}

	response := new(WriteStreamResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return err
	}

	if response.Error != nil {
		return response.Error.Unwrap()
	}

	return nil
}

// ReadStream retrieves an io.ReadCloser for the content stored at "path" with a
// given byte offset.
func (driver *StorageDriverClient) ReadStream(path string, offset uint64) (io.ReadCloser, error) {
	if err := driver.exited(); err != nil {
		return nil, err
	}

	receiver, remoteSender := libchan.Pipe()
	params := map[string]interface{}{"Path": path, "Offset": offset}
	err := driver.sender.Send(&Request{Type: "ReadStream", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return nil, err
	}

	response := new(ReadStreamResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return nil, err
	}

	if response.Error != nil {
		return nil, response.Error.Unwrap()
	}

	return response.Reader, nil
}

// WriteStream stores the contents of the provided io.ReadCloser at a location
// designated by the given path.
func (driver *StorageDriverClient) WriteStream(path string, offset, size uint64, reader io.ReadCloser) error {
	if err := driver.exited(); err != nil {
		return err
	}

	receiver, remoteSender := libchan.Pipe()
	params := map[string]interface{}{"Path": path, "Offset": offset, "Size": size, "Reader": reader}
	err := driver.sender.Send(&Request{Type: "WriteStream", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return err
	}

	response := new(WriteStreamResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return err
	}

	if response.Error != nil {
		return response.Error.Unwrap()
	}

	return nil
}

// CurrentSize retrieves the curernt size in bytes of the object at the given
// path.
func (driver *StorageDriverClient) CurrentSize(path string) (uint64, error) {
	if err := driver.exited(); err != nil {
		return 0, err
	}

	receiver, remoteSender := libchan.Pipe()
	params := map[string]interface{}{"Path": path}
	err := driver.sender.Send(&Request{Type: "CurrentSize", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return 0, err
	}

	response := new(CurrentSizeResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return 0, err
	}

	if response.Error != nil {
		return 0, response.Error.Unwrap()
	}

	return response.Position, nil
}

// List returns a list of the objects that are direct descendants of the given
// path.
func (driver *StorageDriverClient) List(path string) ([]string, error) {
	if err := driver.exited(); err != nil {
		return nil, err
	}

	receiver, remoteSender := libchan.Pipe()
	params := map[string]interface{}{"Path": path}
	err := driver.sender.Send(&Request{Type: "List", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return nil, err
	}

	response := new(ListResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return nil, err
	}

	if response.Error != nil {
		return nil, response.Error.Unwrap()
	}

	return response.Keys, nil
}

// Move moves an object stored at sourcePath to destPath, removing the original
// object.
func (driver *StorageDriverClient) Move(sourcePath string, destPath string) error {
	if err := driver.exited(); err != nil {
		return err
	}

	receiver, remoteSender := libchan.Pipe()
	params := map[string]interface{}{"SourcePath": sourcePath, "DestPath": destPath}
	err := driver.sender.Send(&Request{Type: "Move", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return err
	}

	response := new(MoveResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return err
	}

	if response.Error != nil {
		return response.Error.Unwrap()
	}

	return nil
}

// Delete recursively deletes all objects stored at "path" and its subpaths.
func (driver *StorageDriverClient) Delete(path string) error {
	if err := driver.exited(); err != nil {
		return err
	}

	receiver, remoteSender := libchan.Pipe()
	params := map[string]interface{}{"Path": path}
	err := driver.sender.Send(&Request{Type: "Delete", Parameters: params, ResponseChannel: remoteSender})
	if err != nil {
		return err
	}

	response := new(DeleteResponse)
	err = driver.receiveResponse(receiver, response)
	if err != nil {
		return err
	}

	if response.Error != nil {
		return response.Error.Unwrap()
	}

	return nil
}

// handleSubprocessExit populates the exit channel until we have explicitly
// stopped the storage driver subprocess
// Requests can select on driver.exitChan and response receiving and not hang if
// the process exits
func (driver *StorageDriverClient) handleSubprocessExit() {
	exitErr := driver.subprocess.Wait()
	if exitErr == nil {
		exitErr = fmt.Errorf("Storage driver subprocess already exited cleanly")
	} else {
		exitErr = fmt.Errorf("Storage driver subprocess exited with error: %s", exitErr)
	}

	driver.exitErr = exitErr

	for {
		select {
		case driver.exitChan <- exitErr:
		case <-driver.stopChan:
			close(driver.exitChan)
			return
		}
	}
}

// receiveResponse populates the response value with the next result from the
// given receiver, or returns an error if receiving failed or the driver has
// stopped
func (driver *StorageDriverClient) receiveResponse(receiver libchan.Receiver, response interface{}) error {
	receiveChan := make(chan error, 1)
	go func(receiver libchan.Receiver, receiveChan chan<- error) {
		receiveChan <- receiver.Receive(response)
	}(receiver, receiveChan)

	var err error
	var ok bool
	select {
	case err = <-receiveChan:
	case err, ok = <-driver.exitChan:
		if !ok {
			err = driver.exitErr
		}
	}

	return err
}

// exited returns an exit error if the driver has exited or nil otherwise
func (driver *StorageDriverClient) exited() error {
	select {
	case err, ok := <-driver.exitChan:
		if !ok {
			return driver.exitErr
		}
		return err
	default:
		return nil
	}
}
<|endoftext|>