<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"fmt"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/test/e2e/framework"
	imageutils "k8s.io/kubernetes/test/utils/image"

	"github.com/onsi/ginkgo"
)

var _ = ginkgo.Describe("[sig-api-machinery] Secrets", func() {
	f := framework.NewDefaultFramework("secrets")

	/*
		Release : v1.9
		Testname: Secrets, pod environment field
		Description: Create a secret. Create a Pod with Container that declares a environment variable which references the secret created to extract a key value from the secret. Pod MUST have the environment variable that contains proper value for the key to the secret.
	*/
	framework.ConformanceIt("should be consumable from pods in env vars [NodeConformance]", func() {
		name := "secret-test-" + string(uuid.NewUUID())
		secret := secretForTest(f.Namespace.Name, name)

		ginkgo.By(fmt.Sprintf("Creating secret with name %s", secret.Name))
		var err error
		if secret, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(secret); err != nil {
			framework.Failf("unable to create test secret %s: %v", secret.Name, err)
		}

		pod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "pod-secrets-" + string(uuid.NewUUID()),
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:    "secret-env-test",
						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
						Command: []string{"sh", "-c", "env"},
						Env: []v1.EnvVar{
							{
								Name: "SECRET_DATA",
								ValueFrom: &v1.EnvVarSource{
									SecretKeyRef: &v1.SecretKeySelector{
										LocalObjectReference: v1.LocalObjectReference{
											Name: name,
										},
										Key: "data-1",
									},
								},
							},
						},
					},
				},
				RestartPolicy: v1.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume secrets", pod, 0, []string{
			"SECRET_DATA=value-1",
		})
	})

	/*
		Release : v1.9
		Testname: Secrets, pod environment from source
		Description: Create a secret. Create a Pod with Container that declares a environment variable using ‘EnvFrom’ which references the secret created to extract a key value from the secret. Pod MUST have the environment variable that contains proper value for the key to the secret.
	*/
	framework.ConformanceIt("should be consumable via the environment [NodeConformance]", func() {
		name := "secret-test-" + string(uuid.NewUUID())
		secret := newEnvFromSecret(f.Namespace.Name, name)
		ginkgo.By(fmt.Sprintf("creating secret %v/%v", f.Namespace.Name, secret.Name))
		var err error
		if secret, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(secret); err != nil {
			framework.Failf("unable to create test secret %s: %v", secret.Name, err)
		}

		pod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "pod-configmaps-" + string(uuid.NewUUID()),
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:    "env-test",
						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
						Command: []string{"sh", "-c", "env"},
						EnvFrom: []v1.EnvFromSource{
							{
								SecretRef: &v1.SecretEnvSource{LocalObjectReference: v1.LocalObjectReference{Name: name}},
							},
							{
								Prefix:    "p_",
								SecretRef: &v1.SecretEnvSource{LocalObjectReference: v1.LocalObjectReference{Name: name}},
							},
						},
					},
				},
				RestartPolicy: v1.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume secrets", pod, 0, []string{
			"data_1=value-1", "data_2=value-2", "data_3=value-3",
			"p_data_1=value-1", "p_data_2=value-2", "p_data_3=value-3",
		})
	})

	/*
	   Release : v1.15
	   Testname: Secrets, with empty-key
	   Description: Attempt to create a Secret with an empty key. The creation MUST fail.
	*/
	framework.ConformanceIt("should fail to create secret due to empty secret key", func() {
		secret, err := createEmptyKeySecretForTest(f)
		framework.ExpectError(err, "created secret %q with empty key in namespace %q", secret.Name, f.Namespace.Name)
	})
})

func newEnvFromSecret(namespace, name string) *v1.Secret {
	return &v1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: namespace,
			Name:      name,
		},
		Data: map[string][]byte{
			"data_1": []byte("value-1\n"),
			"data_2": []byte("value-2\n"),
			"data_3": []byte("value-3\n"),
		},
	}
}

func createEmptyKeySecretForTest(f *framework.Framework) (*v1.Secret, error) {
	secretName := "secret-emptykey-test-" + string(uuid.NewUUID())
	secret := &v1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: f.Namespace.Name,
			Name:      secretName,
		},
		Data: map[string][]byte{
			"": []byte("value-1\n"),
		},
	}
	ginkgo.By(fmt.Sprintf("Creating projection with secret that has name %s", secret.Name))
	return f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(secret)
}
<commit_msg>Add: test to patch a secret<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"fmt"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/test/e2e/framework"
	imageutils "k8s.io/kubernetes/test/utils/image"

	"github.com/onsi/ginkgo"
	"github.com/onsi/gomega"
	"encoding/base64"
	"k8s.io/apimachinery/pkg/types"
)

var _ = ginkgo.Describe("[sig-api-machinery] Secrets", func() {
	f := framework.NewDefaultFramework("secrets")

	/*
		Release : v1.9
		Testname: Secrets, pod environment field
		Description: Create a secret. Create a Pod with Container that declares a environment variable which references the secret created to extract a key value from the secret. Pod MUST have the environment variable that contains proper value for the key to the secret.
	*/
	framework.ConformanceIt("should be consumable from pods in env vars [NodeConformance]", func() {
		name := "secret-test-" + string(uuid.NewUUID())
		secret := secretForTest(f.Namespace.Name, name)

		ginkgo.By(fmt.Sprintf("Creating secret with name %s", secret.Name))
		var err error
		if secret, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(secret); err != nil {
			framework.Failf("unable to create test secret %s: %v", secret.Name, err)
		}

		pod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "pod-secrets-" + string(uuid.NewUUID()),
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:    "secret-env-test",
						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
						Command: []string{"sh", "-c", "env"},
						Env: []v1.EnvVar{
							{
								Name: "SECRET_DATA",
								ValueFrom: &v1.EnvVarSource{
									SecretKeyRef: &v1.SecretKeySelector{
										LocalObjectReference: v1.LocalObjectReference{
											Name: name,
										},
										Key: "data-1",
									},
								},
							},
						},
					},
				},
				RestartPolicy: v1.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume secrets", pod, 0, []string{
			"SECRET_DATA=value-1",
		})
	})

	/*
		Release : v1.9
		Testname: Secrets, pod environment from source
		Description: Create a secret. Create a Pod with Container that declares a environment variable using ‘EnvFrom’ which references the secret created to extract a key value from the secret. Pod MUST have the environment variable that contains proper value for the key to the secret.
	*/
	framework.ConformanceIt("should be consumable via the environment [NodeConformance]", func() {
		name := "secret-test-" + string(uuid.NewUUID())
		secret := newEnvFromSecret(f.Namespace.Name, name)
		ginkgo.By(fmt.Sprintf("creating secret %v/%v", f.Namespace.Name, secret.Name))
		var err error
		if secret, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(secret); err != nil {
			framework.Failf("unable to create test secret %s: %v", secret.Name, err)
		}

		pod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "pod-configmaps-" + string(uuid.NewUUID()),
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:    "env-test",
						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
						Command: []string{"sh", "-c", "env"},
						EnvFrom: []v1.EnvFromSource{
							{
								SecretRef: &v1.SecretEnvSource{LocalObjectReference: v1.LocalObjectReference{Name: name}},
							},
							{
								Prefix:    "p_",
								SecretRef: &v1.SecretEnvSource{LocalObjectReference: v1.LocalObjectReference{Name: name}},
							},
						},
					},
				},
				RestartPolicy: v1.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume secrets", pod, 0, []string{
			"data_1=value-1", "data_2=value-2", "data_3=value-3",
			"p_data_1=value-1", "p_data_2=value-2", "p_data_3=value-3",
		})
	})

	/*
	   Release : v1.15
	   Testname: Secrets, with empty-key
	   Description: Attempt to create a Secret with an empty key. The creation MUST fail.
	*/
	framework.ConformanceIt("should fail to create secret due to empty secret key", func() {
		secret, err := createEmptyKeySecretForTest(f)
		framework.ExpectError(err, "created secret %q with empty key in namespace %q", secret.Name, f.Namespace.Name)
	})

	ginkgo.It("should patch a secret", func() {
		ginkgo.By("creating a secret")

		secretTestName := "test-secret-" + string(uuid.NewUUID())

		 // create a secret in namespace default
		_, err := f.ClientSet.CoreV1().Secrets("default").Create(&v1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Name: secretTestName,
			},
			Data: map[string][]byte{
				"key": []byte("value"),
			},
			Type: "Opaque",
		})
		framework.ExpectNoError(err, "failed to create secret")

		ginkgo.By("listing secrets in all namespaces to ensure that there are more than zero")
		// list all secrets in namespace default
		secretsList, err := f.ClientSet.CoreV1().Secrets("").List(metav1.ListOptions{})
		framework.ExpectNoError(err, "failed to list secrets")
		gomega.Expect(len(secretsList.Items)).ToNot(gomega.Equal(0), "no secrets found")

		foundCreatedSecret := false
		var secretCreatedName string
		for _, val := range secretsList.Items {
			if val.ObjectMeta.Name == secretTestName && string(val.Data["key"]) == "value" {
				foundCreatedSecret = true
				secretCreatedName = val.ObjectMeta.Name
			}
		}
		gomega.Expect(foundCreatedSecret).To(gomega.BeTrue(), "unable to find secret by its value")

		ginkgo.By("patching the secret")
		// patch the secret
		secretPatchNewData := base64.StdEncoding.EncodeToString([]byte("value1"))
		secretPatch := fmt.Sprintf(`{"metadata":{"labels":{"testsecret":"true"}},"data":{"key":"%v"}}`, secretPatchNewData)
		_, err = f.ClientSet.CoreV1().Secrets("default").Patch(secretCreatedName, types.StrategicMergePatchType, []byte(secretPatch))
		framework.ExpectNoError(err, "failed to patch secret")

		ginkgo.By("deleting the secret using a LabelSelector")
		err = f.ClientSet.CoreV1().Secrets("default").DeleteCollection(&metav1.DeleteOptions{}, metav1.ListOptions{
			LabelSelector: "testsecret=true",
		})
		framework.ExpectNoError(err, "failed to delete patched secret")

		ginkgo.By("listing secrets in all namespaces, searching for label name and value in patch")
		// list all secrets in namespace default
		secretsList, err = f.ClientSet.CoreV1().Secrets("").List(metav1.ListOptions{})
		framework.ExpectNoError(err, "failed to list secrets")

		foundCreatedSecret = false
		for _, val := range secretsList.Items {
			if val.ObjectMeta.Name == secretTestName && string(val.Data["key"]) == "value" {
				foundCreatedSecret = true
				secretCreatedName = val.ObjectMeta.Name
			}
		}
		gomega.Expect(foundCreatedSecret).To(gomega.BeFalse(), "secret was not deleted sucessfully")
	})
})

func newEnvFromSecret(namespace, name string) *v1.Secret {
	return &v1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: namespace,
			Name:      name,
		},
		Data: map[string][]byte{
			"data_1": []byte("value-1\n"),
			"data_2": []byte("value-2\n"),
			"data_3": []byte("value-3\n"),
		},
	}
}

func createEmptyKeySecretForTest(f *framework.Framework) (*v1.Secret, error) {
	secretName := "secret-emptykey-test-" + string(uuid.NewUUID())
	secret := &v1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: f.Namespace.Name,
			Name:      secretName,
		},
		Data: map[string][]byte{
			"": []byte("value-1\n"),
		},
	}
	ginkgo.By(fmt.Sprintf("Creating projection with secret that has name %s", secret.Name))
	return f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(secret)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
	"context"
	"fmt"
	"regexp"
	"sync"
	"time"

	v1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes/scheme"
	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/kubectl/pkg/util/podutils"

	"github.com/onsi/ginkgo"
	"github.com/onsi/gomega"

	// TODO: Remove the following imports (ref: https://github.com/kubernetes/kubernetes/issues/81245)
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
)

const (
	// DefaultPodDeletionTimeout is the default timeout for deleting pod
	DefaultPodDeletionTimeout = 3 * time.Minute

	// the status of container event, copied from k8s.io/kubernetes/pkg/kubelet/events
	killingContainer = "Killing"

	// the status of container event, copied from k8s.io/kubernetes/pkg/kubelet/events
	failedToCreateContainer = "Failed"

	// the status of container event, copied from k8s.io/kubernetes/pkg/kubelet/events
	startedContainer = "Started"

	// it is copied from k8s.io/kubernetes/pkg/kubelet/sysctl
	forbiddenReason = "SysctlForbidden"
)

// ImageWhiteList is the images used in the current test suite. It should be initialized in test suite and
// the images in the white list should be pre-pulled in the test suite.  Currently, this is only used by
// node e2e test.
var ImageWhiteList sets.String

// PodClient is a convenience method for getting a pod client interface in the framework's namespace,
// possibly applying test-suite specific transformations to the pod spec, e.g. for
// node e2e pod scheduling.
func (f *Framework) PodClient() *PodClient {
	return &PodClient{
		f:            f,
		PodInterface: f.ClientSet.CoreV1().Pods(f.Namespace.Name),
	}
}

// PodClientNS is a convenience method for getting a pod client interface in an alternative namespace,
// possibly applying test-suite specific transformations to the pod spec, e.g. for
// node e2e pod scheduling.
func (f *Framework) PodClientNS(namespace string) *PodClient {
	return &PodClient{
		f:            f,
		PodInterface: f.ClientSet.CoreV1().Pods(namespace),
	}
}

// PodClient is a struct for pod client.
type PodClient struct {
	f *Framework
	v1core.PodInterface
}

// Create creates a new pod according to the framework specifications (don't wait for it to start).
func (c *PodClient) Create(pod *v1.Pod) *v1.Pod {
	c.mungeSpec(pod)
	p, err := c.PodInterface.Create(context.TODO(), pod, metav1.CreateOptions{})
	ExpectNoError(err, "Error creating Pod")
	return p
}

// CreateSync creates a new pod according to the framework specifications, and wait for it to start.
func (c *PodClient) CreateSync(pod *v1.Pod) *v1.Pod {
	namespace := c.f.Namespace.Name
	p := c.Create(pod)
	ExpectNoError(e2epod.WaitForPodNameRunningInNamespace(c.f.ClientSet, p.Name, namespace))
	// Get the newest pod after it becomes running, some status may change after pod created, such as pod ip.
	p, err := c.Get(context.TODO(), p.Name, metav1.GetOptions{})
	ExpectNoError(err)
	return p
}

// CreateBatch create a batch of pods. All pods are created before waiting.
func (c *PodClient) CreateBatch(pods []*v1.Pod) []*v1.Pod {
	ps := make([]*v1.Pod, len(pods))
	var wg sync.WaitGroup
	for i, pod := range pods {
		wg.Add(1)
		go func(i int, pod *v1.Pod) {
			defer wg.Done()
			defer ginkgo.GinkgoRecover()
			ps[i] = c.CreateSync(pod)
		}(i, pod)
	}
	wg.Wait()
	return ps
}

// Update updates the pod object. It retries if there is a conflict, throw out error if
// there is any other apierrors. name is the pod name, updateFn is the function updating the
// pod object.
func (c *PodClient) Update(name string, updateFn func(pod *v1.Pod)) {
	ExpectNoError(wait.Poll(time.Millisecond*500, time.Second*30, func() (bool, error) {
		pod, err := c.PodInterface.Get(context.TODO(), name, metav1.GetOptions{})
		if err != nil {
			return false, fmt.Errorf("failed to get pod %q: %v", name, err)
		}
		updateFn(pod)
		_, err = c.PodInterface.Update(context.TODO(), pod, metav1.UpdateOptions{})
		if err == nil {
			Logf("Successfully updated pod %q", name)
			return true, nil
		}
		if apierrors.IsConflict(err) {
			Logf("Conflicting update to pod %q, re-get and re-update: %v", name, err)
			return false, nil
		}
		return false, fmt.Errorf("failed to update pod %q: %v", name, err)
	}))
}

// DeleteSync deletes the pod and wait for the pod to disappear for `timeout`. If the pod doesn't
// disappear before the timeout, it will fail the test.
func (c *PodClient) DeleteSync(name string, options metav1.DeleteOptions, timeout time.Duration) {
	namespace := c.f.Namespace.Name
	err := c.Delete(context.TODO(), name, options)
	if err != nil && !apierrors.IsNotFound(err) {
		Failf("Failed to delete pod %q: %v", name, err)
	}
	gomega.Expect(e2epod.WaitForPodToDisappear(c.f.ClientSet, namespace, name, labels.Everything(),
		2*time.Second, timeout)).To(gomega.Succeed(), "wait for pod %q to disappear", name)
}

// mungeSpec apply test-suite specific transformations to the pod spec.
func (c *PodClient) mungeSpec(pod *v1.Pod) {
	if !TestContext.NodeE2E {
		return
	}

	gomega.Expect(pod.Spec.NodeName).To(gomega.Or(gomega.BeZero(), gomega.Equal(TestContext.NodeName)), "Test misconfigured")
	pod.Spec.NodeName = TestContext.NodeName
	// Node e2e does not support the default DNSClusterFirst policy. Set
	// the policy to DNSDefault, which is configured per node.
	pod.Spec.DNSPolicy = v1.DNSDefault

	// PrepullImages only works for node e2e now. For cluster e2e, image prepull is not enforced,
	// we should not munge ImagePullPolicy for cluster e2e pods.
	if !TestContext.PrepullImages {
		return
	}
	// If prepull is enabled, munge the container spec to make sure the images are not pulled
	// during the test.
	for i := range pod.Spec.Containers {
		c := &pod.Spec.Containers[i]
		if c.ImagePullPolicy == v1.PullAlways {
			// If the image pull policy is PullAlways, the image doesn't need to be in
			// the white list or pre-pulled, because the image is expected to be pulled
			// in the test anyway.
			continue
		}
		// If the image policy is not PullAlways, the image must be in the white list and
		// pre-pulled.
		gomega.Expect(ImageWhiteList.Has(c.Image)).To(gomega.BeTrue(), "Image %q is not in the white list, consider adding it to CommonImageWhiteList in test/e2e/common/util.go or NodeImageWhiteList in test/e2e_node/image_list.go", c.Image)
		// Do not pull images during the tests because the images in white list should have
		// been prepulled.
		c.ImagePullPolicy = v1.PullNever
	}
}

// WaitForSuccess waits for pod to succeed.
// TODO(random-liu): Move pod wait function into this file
func (c *PodClient) WaitForSuccess(name string, timeout time.Duration) {
	f := c.f
	gomega.Expect(e2epod.WaitForPodCondition(f.ClientSet, f.Namespace.Name, name, fmt.Sprintf("%s or %s", v1.PodSucceeded, v1.PodFailed), timeout,
		func(pod *v1.Pod) (bool, error) {
			switch pod.Status.Phase {
			case v1.PodFailed:
				return true, fmt.Errorf("pod %q failed with reason: %q, message: %q", name, pod.Status.Reason, pod.Status.Message)
			case v1.PodSucceeded:
				return true, nil
			default:
				return false, nil
			}
		},
	)).To(gomega.Succeed(), "wait for pod %q to success", name)
}

// WaitForFinish waits for pod to finish running, regardless of success or failure.
func (c *PodClient) WaitForFinish(name string, timeout time.Duration) {
	f := c.f
	gomega.Expect(e2epod.WaitForPodCondition(f.ClientSet, f.Namespace.Name, name, fmt.Sprintf("%s or %s", v1.PodSucceeded, v1.PodFailed), timeout,
		func(pod *v1.Pod) (bool, error) {
			switch pod.Status.Phase {
			case v1.PodFailed:
				return true, nil
			case v1.PodSucceeded:
				return true, nil
			default:
				return false, nil
			}
		},
	)).To(gomega.Succeed(), "wait for pod %q to finish running", name)
}

// WaitForErrorEventOrSuccess waits for pod to succeed or an error event for that pod.
func (c *PodClient) WaitForErrorEventOrSuccess(pod *v1.Pod) (*v1.Event, error) {
	var ev *v1.Event
	err := wait.Poll(Poll, PodStartTimeout, func() (bool, error) {
		evnts, err := c.f.ClientSet.CoreV1().Events(pod.Namespace).Search(scheme.Scheme, pod)
		if err != nil {
			return false, fmt.Errorf("error in listing events: %s", err)
		}
		for _, e := range evnts.Items {
			switch e.Reason {
			case killingContainer, failedToCreateContainer, forbiddenReason:
				ev = &e
				return true, nil
			case startedContainer:
				return true, nil
			default:
				// ignore all other errors
			}
		}
		return false, nil
	})
	return ev, err
}

// MatchContainerOutput gets output of a container and match expected regexp in the output.
func (c *PodClient) MatchContainerOutput(name string, containerName string, expectedRegexp string) error {
	f := c.f
	output, err := e2epod.GetPodLogs(f.ClientSet, f.Namespace.Name, name, containerName)
	if err != nil {
		return fmt.Errorf("failed to get output for container %q of pod %q", containerName, name)
	}
	regex, err := regexp.Compile(expectedRegexp)
	if err != nil {
		return fmt.Errorf("failed to compile regexp %q: %v", expectedRegexp, err)
	}
	if !regex.MatchString(output) {
		return fmt.Errorf("failed to match regexp %q in output %q", expectedRegexp, output)
	}
	return nil
}

// PodIsReady returns true if the specified pod is ready. Otherwise false.
func (c *PodClient) PodIsReady(name string) bool {
	pod, err := c.Get(context.TODO(), name, metav1.GetOptions{})
	ExpectNoError(err)
	return podutils.IsPodReady(pod)
}
<commit_msg>test/e2e/framework/pods: Fix "to success" -> "to succeed"<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
	"context"
	"fmt"
	"regexp"
	"sync"
	"time"

	v1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes/scheme"
	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/kubectl/pkg/util/podutils"

	"github.com/onsi/ginkgo"
	"github.com/onsi/gomega"

	// TODO: Remove the following imports (ref: https://github.com/kubernetes/kubernetes/issues/81245)
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
)

const (
	// DefaultPodDeletionTimeout is the default timeout for deleting pod
	DefaultPodDeletionTimeout = 3 * time.Minute

	// the status of container event, copied from k8s.io/kubernetes/pkg/kubelet/events
	killingContainer = "Killing"

	// the status of container event, copied from k8s.io/kubernetes/pkg/kubelet/events
	failedToCreateContainer = "Failed"

	// the status of container event, copied from k8s.io/kubernetes/pkg/kubelet/events
	startedContainer = "Started"

	// it is copied from k8s.io/kubernetes/pkg/kubelet/sysctl
	forbiddenReason = "SysctlForbidden"
)

// ImageWhiteList is the images used in the current test suite. It should be initialized in test suite and
// the images in the white list should be pre-pulled in the test suite.  Currently, this is only used by
// node e2e test.
var ImageWhiteList sets.String

// PodClient is a convenience method for getting a pod client interface in the framework's namespace,
// possibly applying test-suite specific transformations to the pod spec, e.g. for
// node e2e pod scheduling.
func (f *Framework) PodClient() *PodClient {
	return &PodClient{
		f:            f,
		PodInterface: f.ClientSet.CoreV1().Pods(f.Namespace.Name),
	}
}

// PodClientNS is a convenience method for getting a pod client interface in an alternative namespace,
// possibly applying test-suite specific transformations to the pod spec, e.g. for
// node e2e pod scheduling.
func (f *Framework) PodClientNS(namespace string) *PodClient {
	return &PodClient{
		f:            f,
		PodInterface: f.ClientSet.CoreV1().Pods(namespace),
	}
}

// PodClient is a struct for pod client.
type PodClient struct {
	f *Framework
	v1core.PodInterface
}

// Create creates a new pod according to the framework specifications (don't wait for it to start).
func (c *PodClient) Create(pod *v1.Pod) *v1.Pod {
	c.mungeSpec(pod)
	p, err := c.PodInterface.Create(context.TODO(), pod, metav1.CreateOptions{})
	ExpectNoError(err, "Error creating Pod")
	return p
}

// CreateSync creates a new pod according to the framework specifications, and wait for it to start.
func (c *PodClient) CreateSync(pod *v1.Pod) *v1.Pod {
	namespace := c.f.Namespace.Name
	p := c.Create(pod)
	ExpectNoError(e2epod.WaitForPodNameRunningInNamespace(c.f.ClientSet, p.Name, namespace))
	// Get the newest pod after it becomes running, some status may change after pod created, such as pod ip.
	p, err := c.Get(context.TODO(), p.Name, metav1.GetOptions{})
	ExpectNoError(err)
	return p
}

// CreateBatch create a batch of pods. All pods are created before waiting.
func (c *PodClient) CreateBatch(pods []*v1.Pod) []*v1.Pod {
	ps := make([]*v1.Pod, len(pods))
	var wg sync.WaitGroup
	for i, pod := range pods {
		wg.Add(1)
		go func(i int, pod *v1.Pod) {
			defer wg.Done()
			defer ginkgo.GinkgoRecover()
			ps[i] = c.CreateSync(pod)
		}(i, pod)
	}
	wg.Wait()
	return ps
}

// Update updates the pod object. It retries if there is a conflict, throw out error if
// there is any other apierrors. name is the pod name, updateFn is the function updating the
// pod object.
func (c *PodClient) Update(name string, updateFn func(pod *v1.Pod)) {
	ExpectNoError(wait.Poll(time.Millisecond*500, time.Second*30, func() (bool, error) {
		pod, err := c.PodInterface.Get(context.TODO(), name, metav1.GetOptions{})
		if err != nil {
			return false, fmt.Errorf("failed to get pod %q: %v", name, err)
		}
		updateFn(pod)
		_, err = c.PodInterface.Update(context.TODO(), pod, metav1.UpdateOptions{})
		if err == nil {
			Logf("Successfully updated pod %q", name)
			return true, nil
		}
		if apierrors.IsConflict(err) {
			Logf("Conflicting update to pod %q, re-get and re-update: %v", name, err)
			return false, nil
		}
		return false, fmt.Errorf("failed to update pod %q: %v", name, err)
	}))
}

// DeleteSync deletes the pod and wait for the pod to disappear for `timeout`. If the pod doesn't
// disappear before the timeout, it will fail the test.
func (c *PodClient) DeleteSync(name string, options metav1.DeleteOptions, timeout time.Duration) {
	namespace := c.f.Namespace.Name
	err := c.Delete(context.TODO(), name, options)
	if err != nil && !apierrors.IsNotFound(err) {
		Failf("Failed to delete pod %q: %v", name, err)
	}
	gomega.Expect(e2epod.WaitForPodToDisappear(c.f.ClientSet, namespace, name, labels.Everything(),
		2*time.Second, timeout)).To(gomega.Succeed(), "wait for pod %q to disappear", name)
}

// mungeSpec apply test-suite specific transformations to the pod spec.
func (c *PodClient) mungeSpec(pod *v1.Pod) {
	if !TestContext.NodeE2E {
		return
	}

	gomega.Expect(pod.Spec.NodeName).To(gomega.Or(gomega.BeZero(), gomega.Equal(TestContext.NodeName)), "Test misconfigured")
	pod.Spec.NodeName = TestContext.NodeName
	// Node e2e does not support the default DNSClusterFirst policy. Set
	// the policy to DNSDefault, which is configured per node.
	pod.Spec.DNSPolicy = v1.DNSDefault

	// PrepullImages only works for node e2e now. For cluster e2e, image prepull is not enforced,
	// we should not munge ImagePullPolicy for cluster e2e pods.
	if !TestContext.PrepullImages {
		return
	}
	// If prepull is enabled, munge the container spec to make sure the images are not pulled
	// during the test.
	for i := range pod.Spec.Containers {
		c := &pod.Spec.Containers[i]
		if c.ImagePullPolicy == v1.PullAlways {
			// If the image pull policy is PullAlways, the image doesn't need to be in
			// the white list or pre-pulled, because the image is expected to be pulled
			// in the test anyway.
			continue
		}
		// If the image policy is not PullAlways, the image must be in the white list and
		// pre-pulled.
		gomega.Expect(ImageWhiteList.Has(c.Image)).To(gomega.BeTrue(), "Image %q is not in the white list, consider adding it to CommonImageWhiteList in test/e2e/common/util.go or NodeImageWhiteList in test/e2e_node/image_list.go", c.Image)
		// Do not pull images during the tests because the images in white list should have
		// been prepulled.
		c.ImagePullPolicy = v1.PullNever
	}
}

// WaitForSuccess waits for pod to succeed.
// TODO(random-liu): Move pod wait function into this file
func (c *PodClient) WaitForSuccess(name string, timeout time.Duration) {
	f := c.f
	gomega.Expect(e2epod.WaitForPodCondition(f.ClientSet, f.Namespace.Name, name, fmt.Sprintf("%s or %s", v1.PodSucceeded, v1.PodFailed), timeout,
		func(pod *v1.Pod) (bool, error) {
			switch pod.Status.Phase {
			case v1.PodFailed:
				return true, fmt.Errorf("pod %q failed with reason: %q, message: %q", name, pod.Status.Reason, pod.Status.Message)
			case v1.PodSucceeded:
				return true, nil
			default:
				return false, nil
			}
		},
	)).To(gomega.Succeed(), "wait for pod %q to succeed", name)
}

// WaitForFinish waits for pod to finish running, regardless of success or failure.
func (c *PodClient) WaitForFinish(name string, timeout time.Duration) {
	f := c.f
	gomega.Expect(e2epod.WaitForPodCondition(f.ClientSet, f.Namespace.Name, name, fmt.Sprintf("%s or %s", v1.PodSucceeded, v1.PodFailed), timeout,
		func(pod *v1.Pod) (bool, error) {
			switch pod.Status.Phase {
			case v1.PodFailed:
				return true, nil
			case v1.PodSucceeded:
				return true, nil
			default:
				return false, nil
			}
		},
	)).To(gomega.Succeed(), "wait for pod %q to finish running", name)
}

// WaitForErrorEventOrSuccess waits for pod to succeed or an error event for that pod.
func (c *PodClient) WaitForErrorEventOrSuccess(pod *v1.Pod) (*v1.Event, error) {
	var ev *v1.Event
	err := wait.Poll(Poll, PodStartTimeout, func() (bool, error) {
		evnts, err := c.f.ClientSet.CoreV1().Events(pod.Namespace).Search(scheme.Scheme, pod)
		if err != nil {
			return false, fmt.Errorf("error in listing events: %s", err)
		}
		for _, e := range evnts.Items {
			switch e.Reason {
			case killingContainer, failedToCreateContainer, forbiddenReason:
				ev = &e
				return true, nil
			case startedContainer:
				return true, nil
			default:
				// ignore all other errors
			}
		}
		return false, nil
	})
	return ev, err
}

// MatchContainerOutput gets output of a container and match expected regexp in the output.
func (c *PodClient) MatchContainerOutput(name string, containerName string, expectedRegexp string) error {
	f := c.f
	output, err := e2epod.GetPodLogs(f.ClientSet, f.Namespace.Name, name, containerName)
	if err != nil {
		return fmt.Errorf("failed to get output for container %q of pod %q", containerName, name)
	}
	regex, err := regexp.Compile(expectedRegexp)
	if err != nil {
		return fmt.Errorf("failed to compile regexp %q: %v", expectedRegexp, err)
	}
	if !regex.MatchString(output) {
		return fmt.Errorf("failed to match regexp %q in output %q", expectedRegexp, output)
	}
	return nil
}

// PodIsReady returns true if the specified pod is ready. Otherwise false.
func (c *PodClient) PodIsReady(name string) bool {
	pod, err := c.Get(context.TODO(), name, metav1.GetOptions{})
	ExpectNoError(err)
	return podutils.IsPodReady(pod)
}
<|endoftext|>


<commit_before>package cl11

import (
	"strings"

	clw "github.com/rdwilliamson/clw11"
)

// The OpenCL functions that are submitted to a command-queue are enqueued in
// the order the calls are made but can be configured to execute in-order or
// out-of-order. The properties argument in CreateCommandQueue can be used to
// specify the execution order. In addition, a wait for events or a barrier
// command can be enqueued to the command-queue. The wait for events command
// ensures that previously enqueued commands identified by the list of events to
// wait for have finished before the next batch of commands is executed. The
// barrier command ensures that all previously enqueued commands in a command-
// queue have finished execution before the next batch of commands is executed.
// Similarly, commands to read, write, copy or map memory objects that are
// enqueued after EnqueueNDRangeKernel, EnqueueTask or EnqueueNativeKernel
// commands are not guaranteed to wait for kernels scheduled for execution to
// have completed (if the out of order property is set). To ensure correct
// ordering of commands, the event object returned by EnqueueNDRangeKernel,
// EnqueueTask or EnqueueNativeKernel can be used to enqueue a wait for event or
// a barrier command can be enqueued that must complete before reads or writes
// to the memory object(s) occur.
type CommandQueue struct {
	id clw.CommandQueue

	// The context the command queue was created on.
	Context *Context

	// The device the command queue was created for.
	Device *Device

	// Bit-field list of properties for the command queue.
	Properties CommandQueueProperties

	// Scratch space to avoid allocating memory when converting a wait list.
	eventsScratch []clw.Event
}

type CommandQueueProperties uint

// Bit field.
const (
	QueueOutOfOrderExecution = CommandQueueProperties(clw.QueueOutOfOrderExecModeEnable)
	QueueProfilingEnable     = CommandQueueProperties(clw.QueueProfilingEnable)
)

func (cqp CommandQueueProperties) String() string {
	var propertiesStrings []string
	if cqp&QueueOutOfOrderExecution != 0 {
		propertiesStrings = append(propertiesStrings, "out of order execution")
	}
	if cqp&QueueProfilingEnable != 0 {
		propertiesStrings = append(propertiesStrings, "profiling")
	}
	return "{" + strings.Join(propertiesStrings, ", ") + "}"
}

// Create a command-queue on a specific device.
func (c *Context) CreateCommandQueue(d *Device, cqp CommandQueueProperties) (*CommandQueue, error) {

	commandQueue, err := clw.CreateCommandQueue(c.id, d.id, clw.CommandQueueProperties(cqp))
	if err != nil {
		return nil, err
	}

	return &CommandQueue{id: commandQueue, Context: c, Device: d, Properties: cqp}, nil
}

// Issues all previously queued OpenCL commands in a command-queue to the device
// associated with the command-queue.
//
// Flush only guarantees that all queued commands to command_queue get issued to
// the appropriate device. There is no guarantee that they will be complete
// after clFlush returns. Any blocking commands queued in a command-queue
// perform an implicit flush of the command-queue. To use event objects that
// refer to commands enqueued in a command-queue as event objects to wait on by
// commands enqueued in a different command-queue, the application must call a
// Flush or any blocking commands that perform an implicit flush of the command-
// queue where the commands that refer to these event objects are enqueued.
func (cq *CommandQueue) Flush() error {
	return clw.Flush(cq.id)
}

// Blocks until all previously queued OpenCL commands in a command-queue are
// issued to the associated device and have completed.
//
// Blocks until all previously queued OpenCL commands in command_queue are
// issued to the associated device and have completed. Finish does not return
// until all queued commands in command_queue have been processed and completed.
// Finish is also a synchronization point.
func (cq *CommandQueue) Finish() error {
	return clw.Finish(cq.id)
}

// Increments the command_queue reference count.
func (cq *CommandQueue) Retain() error {
	return clw.RetainCommandQueue(cq.id)
}

// Decrements the command_queue reference count.
func (cq *CommandQueue) Release() error {
	return clw.ReleaseCommandQueue(cq.id)
}

// Enqueues a marker command.
//
// Enqueues a marker command to the command queue. The marker command is not
// completed until all commands enqueued before it have completed. The marker
// command returns an event which can be waited on, i.e. this event can be
// waited on to ensure that all commands which have been queued before the
// market command have been completed. complete.
func (cq *CommandQueue) EnqueueMarker(e *Event) error {

	if e != nil {
		e.Context = cq.Context
		e.CommandType = CommandMarker
		e.CommandQueue = cq
	}

	return clw.EnqueueMarker(cq.id, &e.id)
}

// Enqueues a wait for a specific event or a list of events to complete before
// any future commands queued in the command-queue are executed.
//
// The context associated with events in event_list and command_queue must be
// the same.
func (cq *CommandQueue) EnqueueWaitForEvents(waitList []*Event) error {
	return clw.EnqueueWaitForEvents(cq.id, cq.toEvents(waitList))
}

// A synchronization point that enqueues a barrier operation.
//
// EnqueueBarrier is a synchronization point that ensures that all queued
// commands in command_queue have finished execution before the next batch of
// commands can begin execution.
func (cq *CommandQueue) EnqueueBarrier() error {
	return clw.EnqueueBarrier(cq.id)
}

func (cq *CommandQueue) toEvents(in []*Event) []clw.Event {

	if in == nil {
		return nil
	}

	if len(cq.eventsScratch) < len(in) {
		cq.eventsScratch = make([]clw.Event, len(in))
	}

	for i := range in {
		cq.eventsScratch[i] = in[i].id
	}

	return cq.eventsScratch[:len(in)]
}
<commit_msg>Added command queue reference count.<commit_after>package cl11

import (
	"strings"
	"unsafe"

	clw "github.com/rdwilliamson/clw11"
)

// The OpenCL functions that are submitted to a command-queue are enqueued in
// the order the calls are made but can be configured to execute in-order or
// out-of-order. The properties argument in CreateCommandQueue can be used to
// specify the execution order. In addition, a wait for events or a barrier
// command can be enqueued to the command-queue. The wait for events command
// ensures that previously enqueued commands identified by the list of events to
// wait for have finished before the next batch of commands is executed. The
// barrier command ensures that all previously enqueued commands in a command-
// queue have finished execution before the next batch of commands is executed.
// Similarly, commands to read, write, copy or map memory objects that are
// enqueued after EnqueueNDRangeKernel, EnqueueTask or EnqueueNativeKernel
// commands are not guaranteed to wait for kernels scheduled for execution to
// have completed (if the out of order property is set). To ensure correct
// ordering of commands, the event object returned by EnqueueNDRangeKernel,
// EnqueueTask or EnqueueNativeKernel can be used to enqueue a wait for event or
// a barrier command can be enqueued that must complete before reads or writes
// to the memory object(s) occur.
type CommandQueue struct {
	id clw.CommandQueue

	// The context the command queue was created on.
	Context *Context

	// The device the command queue was created for.
	Device *Device

	// Bit-field list of properties for the command queue.
	Properties CommandQueueProperties

	// Scratch space to avoid allocating memory when converting a wait list.
	eventsScratch []clw.Event
}

type CommandQueueProperties uint

// Bit field.
const (
	QueueOutOfOrderExecution = CommandQueueProperties(clw.QueueOutOfOrderExecModeEnable)
	QueueProfilingEnable     = CommandQueueProperties(clw.QueueProfilingEnable)
)

func (cqp CommandQueueProperties) String() string {
	var propertiesStrings []string
	if cqp&QueueOutOfOrderExecution != 0 {
		propertiesStrings = append(propertiesStrings, "out of order execution")
	}
	if cqp&QueueProfilingEnable != 0 {
		propertiesStrings = append(propertiesStrings, "profiling")
	}
	return "{" + strings.Join(propertiesStrings, ", ") + "}"
}

// Create a command-queue on a specific device.
func (c *Context) CreateCommandQueue(d *Device, cqp CommandQueueProperties) (*CommandQueue, error) {

	commandQueue, err := clw.CreateCommandQueue(c.id, d.id, clw.CommandQueueProperties(cqp))
	if err != nil {
		return nil, err
	}

	return &CommandQueue{id: commandQueue, Context: c, Device: d, Properties: cqp}, nil
}

// Issues all previously queued OpenCL commands in a command-queue to the device
// associated with the command-queue.
//
// Flush only guarantees that all queued commands to command_queue get issued to
// the appropriate device. There is no guarantee that they will be complete
// after clFlush returns. Any blocking commands queued in a command-queue
// perform an implicit flush of the command-queue. To use event objects that
// refer to commands enqueued in a command-queue as event objects to wait on by
// commands enqueued in a different command-queue, the application must call a
// Flush or any blocking commands that perform an implicit flush of the command-
// queue where the commands that refer to these event objects are enqueued.
func (cq *CommandQueue) Flush() error {
	return clw.Flush(cq.id)
}

// Blocks until all previously queued OpenCL commands in a command-queue are
// issued to the associated device and have completed.
//
// Blocks until all previously queued OpenCL commands in command_queue are
// issued to the associated device and have completed. Finish does not return
// until all queued commands in command_queue have been processed and completed.
// Finish is also a synchronization point.
func (cq *CommandQueue) Finish() error {
	return clw.Finish(cq.id)
}

// Increments the command_queue reference count.
func (cq *CommandQueue) Retain() error {
	return clw.RetainCommandQueue(cq.id)
}

// Decrements the command_queue reference count.
func (cq *CommandQueue) Release() error {
	return clw.ReleaseCommandQueue(cq.id)
}

// Return the command queue's reference count.
//
// The reference count returned should be considered immediately stale. It is
// unsuitable for general use in applications. This feature is provided for
// identifying memory leaks.
func (cq *CommandQueue) ReferenceCount() (int, error) {

	var count clw.Uint
	err := clw.GetCommandQueueInfo(cq.id, clw.QueueReferenceCount, clw.Size(unsafe.Sizeof(count)),
		unsafe.Pointer(&count), nil)
	if err != nil {
		return 0, err
	}

	return int(count), nil
}

// Enqueues a marker command.
//
// Enqueues a marker command to the command queue. The marker command is not
// completed until all commands enqueued before it have completed. The marker
// command returns an event which can be waited on, i.e. this event can be
// waited on to ensure that all commands which have been queued before the
// market command have been completed. complete.
func (cq *CommandQueue) EnqueueMarker(e *Event) error {

	if e != nil {
		e.Context = cq.Context
		e.CommandType = CommandMarker
		e.CommandQueue = cq
	}

	return clw.EnqueueMarker(cq.id, &e.id)
}

// Enqueues a wait for a specific event or a list of events to complete before
// any future commands queued in the command-queue are executed.
//
// The context associated with events in event_list and command_queue must be
// the same.
func (cq *CommandQueue) EnqueueWaitForEvents(waitList []*Event) error {
	return clw.EnqueueWaitForEvents(cq.id, cq.toEvents(waitList))
}

// A synchronization point that enqueues a barrier operation.
//
// EnqueueBarrier is a synchronization point that ensures that all queued
// commands in command_queue have finished execution before the next batch of
// commands can begin execution.
func (cq *CommandQueue) EnqueueBarrier() error {
	return clw.EnqueueBarrier(cq.id)
}

func (cq *CommandQueue) toEvents(in []*Event) []clw.Event {

	if in == nil {
		return nil
	}

	if len(cq.eventsScratch) < len(in) {
		cq.eventsScratch = make([]clw.Event, len(in))
	}

	for i := range in {
		cq.eventsScratch[i] = in[i].id
	}

	return cq.eventsScratch[:len(in)]
}
<|endoftext|>


<commit_before>//  Copyright (c) 2012-2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package base

import (
	"fmt"
	"log"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"
)

// 1 enables regular logs, 2 enables warnings, 3+ is nothing but panics.
// Default value is 1.
var logLevel int = 1

// Set of LogTo() key strings that are enabled.
var LogKeys map[string]bool

var logNoTime bool

var logLock sync.RWMutex

var logger *log.Logger

var logFile *os.File

var logStar bool // enabling log key "*" enables all key-based logging

//Attach logger to stderr during load, this may get re-attached once config is loaded
func init() {
	logger = log.New(os.Stderr, "", 0)
	LogKeys = make(map[string]bool)
	logNoTime = false
}

func LogLevel() int {
	return logLevel
}

func SetLogLevel(level int) {
	logLock.Lock()
	defer logLock.Unlock()
	logLevel = level
}

// Disables ANSI color in log output.
func LogNoColor() {
	// this is now the default state; see LogColor() below
}

func LogNoTime() {
	logLock.RLock()
	defer logLock.RUnlock()
	//Disable timestamp for default logger, this may be used by other packages
	log.SetFlags(log.Flags() &^ (log.Ldate | log.Ltime | log.Lmicroseconds))
	logNoTime = true
}

// Parses a comma-separated list of log keys, probably coming from an argv flag.
// The key "bw" is interpreted as a call to LogNoColor, not a key.
func ParseLogFlag(flag string) {
	if flag != "" {
		ParseLogFlags(strings.Split(flag, ","))
	}
}

// Parses an array of log keys, probably coming from a argv flags.
// The key "bw" is interpreted as a call to LogNoColor, not a key.
func ParseLogFlags(flags []string) {
	logLock.Lock()
	for _, key := range flags {
		switch key {
		case "bw":
			LogNoColor()
		case "color":
			LogColor()
		case "notime":
			LogNoTime()
		default:
			LogKeys[key] = true
			if key == "*" {
				logStar = true
			}
			// gocb requires a call into the gocb library to enable logging
			if key == "gocb" {
				EnableGoCBLogging()
			}
			for strings.HasSuffix(key, "+") {
				key = key[0 : len(key)-1]
				LogKeys[key] = true // "foo+" also enables "foo"
			}
		}
	}
	logLock.Unlock()
	Logf("Enabling logging: %s", flags)
}

func GetLogKeys() map[string]bool {
	logLock.RLock()
	defer logLock.RUnlock()
	keys := map[string]bool{}
	for k, v := range LogKeys {
		keys[k] = v
	}
	return keys
}

func UpdateLogKeys(keys map[string]bool, replace bool) {
	logLock.Lock()
	defer logLock.Unlock()
	if replace {
		LogKeys = map[string]bool{}
	}
	for k, v := range keys {
		LogKeys[k] = v
		if k == "*" {
			logStar = v
		}
	}
}

// Returns a string identifying a function on the call stack.
// Use depth=1 for the caller of the function that calls GetCallersName, etc.
func GetCallersName(depth int) string {
	pc, file, line, ok := runtime.Caller(depth + 1)
	if !ok {
		return "???"
	}

	fnname := ""
	if fn := runtime.FuncForPC(pc); fn != nil {
		fnname = fn.Name()
	}

	return fmt.Sprintf("%s() at %s:%d", lastComponent(fnname), lastComponent(file), line)
}

// Logs a message to the console, but only if the corresponding key is true in LogKeys.
func LogTo(key string, format string, args ...interface{}) {
	logLock.RLock()
	defer logLock.RUnlock()
	ok := logLevel <= 1 && (logStar || LogKeys[key])

	if ok {
		printf(fgYellow+key+": "+reset+format, args...)
	}
}

func EnableLogKey(key string) {
	logLock.Lock()
	defer logLock.Unlock()
	LogKeys[key] = true
}

func LogEnabled(key string) bool {
	logLock.RLock()
	defer logLock.RUnlock()
	return logStar || LogKeys[key]
}

// Logs a message to the console.
func Log(message string) {
	logLock.RLock()
	defer logLock.RUnlock()
	ok := logLevel <= 1

	if ok {
		print(message)
	}
}

// Logs a formatted message to the console.
func Logf(format string, args ...interface{}) {
	logLock.RLock()
	defer logLock.RUnlock()
	ok := logLevel <= 1

	if ok {
		printf(format, args...)
	}
}

// If the error is not nil, logs its description and the name of the calling function.
// Returns the input error for easy chaining.
func LogError(err error) error {
	if err != nil {
		logLock.RLock()
		ok := logLevel <= 2
		logLock.RUnlock()

		if ok {
			logWithCaller(fgRed, "ERROR", "%v", err)
		}
	}
	return err
}

// Logs a warning to the console
func Warn(format string, args ...interface{}) {
	logLock.RLock()
	ok := logLevel <= 2
	logLock.RUnlock()

	if ok {
		logWithCaller(fgRed, "WARNING", format, args...)
	}
}

// Logs a highlighted message prefixed with "TEMP". This function is intended for
// temporary logging calls added during development and not to be checked in, hence its
// distinctive name (which is visible and easy to search for before committing.)
func TEMP(format string, args ...interface{}) {
	logWithCaller(fgYellow, "TEMP", format, args...)
}

// Logs a warning to the console, then panics.
func LogPanic(format string, args ...interface{}) {
	logWithCaller(fgRed, "PANIC", format, args...)
	panic(fmt.Sprintf(format, args...))
}

// Logs a warning to the console, then exits the process.
func LogFatal(format string, args ...interface{}) {
	logWithCaller(fgRed, "FATAL", format, args...)
	os.Exit(1)
}

func logWithCaller(color string, prefix string, format string, args ...interface{}) {
	message := fmt.Sprintf(format, args...)
	logLock.RLock()
	defer logLock.RUnlock()
	print(color, prefix, ": ", message, reset,
		dim, " -- ", GetCallersName(2), reset)
}

// Simple wrapper that converts Print to Printf
func print(args ...interface{}) {
	ok := logLevel <= 1

	if ok {
		printf("%s", fmt.Sprint(args...))
	}
}

// Logs a formatted message to the underlying logger
func printf(format string, args ...interface{}) {
	logLock.RLock()
	defer logLock.RUnlock()
	ok := logLevel <= 1

	if ok {
		if !logNoTime {
			timestampedFormat := strings.Join([]string{time.Now().Format(ISO8601Format), format}, " ")
			logger.Printf(timestampedFormat, args...)
		} else {
			logger.Printf(format, args...)
		}
	}
}

func lastComponent(path string) string {
	if index := strings.LastIndex(path, "/"); index >= 0 {
		path = path[index+1:]
	} else if index = strings.LastIndex(path, "\\"); index >= 0 {
		path = path[index+1:]
	}
	return path
}

func UpdateLogger(logFilePath string) {
	//Attempt to open file for write at path provided
	fo, err := os.OpenFile(logFilePath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0664)
	if err != nil {
		LogFatal("unable to open logfile for write: %s", logFilePath)
	}

	//defer write lock to here otherwise LogFatal above will deadlock
	logLock.Lock()

	//We keep a reference to the underlying log File as log.Logger and io.Writer
	//have no close() methods and we want to close old files on log rotation
	oldLogFile := logFile
	logFile = fo
	logger = log.New(fo, "", log.Lmicroseconds)
	logLock.Unlock()

	//re-apply log no time flags on new logger
	if logNoTime {
		LogNoTime()
	}

	//If there is a previously opened log file, explicitly close it
	if oldLogFile != nil {
		err = oldLogFile.Close()
		if err != nil {
			Warn("unable to close old log File after updating logger")
		}
	}
}

// ANSI color control escape sequences.
// Shamelessly copied from https://github.com/sqp/godock/blob/master/libs/log/colors.go
var (
	reset,
	bright,
	dim,
	underscore,
	blink,
	reverse,
	hidden,
	fgBlack,
	fgRed,
	fgGreen,
	fgYellow,
	fgBlue,
	fgMagenta,
	fgCyan,
	fgWhite,
	bgBlack,
	bgRed,
	bgGreen,
	bgYellow,
	bgBlue,
	bgMagenta,
	bgCyan,
	bgWhite string
)

func LogColor() {
	reset = "\x1b[0m"
	bright = "\x1b[1m"
	dim = "\x1b[2m"
	underscore = "\x1b[4m"
	blink = "\x1b[5m"
	reverse = "\x1b[7m"
	hidden = "\x1b[8m"
	fgBlack = "\x1b[30m"
	fgRed = "\x1b[31m"
	fgGreen = "\x1b[32m"
	fgYellow = "\x1b[33m"
	fgBlue = "\x1b[34m"
	fgMagenta = "\x1b[35m"
	fgCyan = "\x1b[36m"
	fgWhite = "\x1b[37m"
	bgBlack = "\x1b[40m"
	bgRed = "\x1b[41m"
	bgGreen = "\x1b[42m"
	bgYellow = "\x1b[43m"
	bgBlue = "\x1b[44m"
	bgMagenta = "\x1b[45m"
	bgCyan = "\x1b[46m"
	bgWhite = "\x1b[47m"
}
<commit_msg>Avoid nested read locks in logging.go<commit_after>//  Copyright (c) 2012-2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package base

import (
	"fmt"
	"log"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"
)

// 1 enables regular logs, 2 enables warnings, 3+ is nothing but panics.
// Default value is 1.
var logLevel int = 1

// Set of LogTo() key strings that are enabled.
var LogKeys map[string]bool

var logNoTime bool

var logLock sync.RWMutex

var logger *log.Logger

var logFile *os.File

var logStar bool // enabling log key "*" enables all key-based logging

//Attach logger to stderr during load, this may get re-attached once config is loaded
func init() {
	logger = log.New(os.Stderr, "", 0)
	LogKeys = make(map[string]bool)
	logNoTime = false
}

func LogLevel() int {
	return logLevel
}

func SetLogLevel(level int) {
	logLock.Lock()
	defer logLock.Unlock()
	logLevel = level
}

// Disables ANSI color in log output.
func LogNoColor() {
	// this is now the default state; see LogColor() below
}

func LogNoTime() {
	logLock.RLock()
	defer logLock.RUnlock()
	//Disable timestamp for default logger, this may be used by other packages
	log.SetFlags(log.Flags() &^ (log.Ldate | log.Ltime | log.Lmicroseconds))
	logNoTime = true
}

// Parses a comma-separated list of log keys, probably coming from an argv flag.
// The key "bw" is interpreted as a call to LogNoColor, not a key.
func ParseLogFlag(flag string) {
	if flag != "" {
		ParseLogFlags(strings.Split(flag, ","))
	}
}

// Parses an array of log keys, probably coming from a argv flags.
// The key "bw" is interpreted as a call to LogNoColor, not a key.
func ParseLogFlags(flags []string) {
	logLock.Lock()
	for _, key := range flags {
		switch key {
		case "bw":
			LogNoColor()
		case "color":
			LogColor()
		case "notime":
			LogNoTime()
		default:
			LogKeys[key] = true
			if key == "*" {
				logStar = true
			}
			// gocb requires a call into the gocb library to enable logging
			if key == "gocb" {
				EnableGoCBLogging()
			}
			for strings.HasSuffix(key, "+") {
				key = key[0 : len(key)-1]
				LogKeys[key] = true // "foo+" also enables "foo"
			}
		}
	}
	logLock.Unlock()
	Logf("Enabling logging: %s", flags)
}

func GetLogKeys() map[string]bool {
	logLock.RLock()
	defer logLock.RUnlock()
	keys := map[string]bool{}
	for k, v := range LogKeys {
		keys[k] = v
	}
	return keys
}

func UpdateLogKeys(keys map[string]bool, replace bool) {
	logLock.Lock()
	defer logLock.Unlock()
	if replace {
		LogKeys = map[string]bool{}
	}
	for k, v := range keys {
		LogKeys[k] = v
		if k == "*" {
			logStar = v
		}
	}
}

// Returns a string identifying a function on the call stack.
// Use depth=1 for the caller of the function that calls GetCallersName, etc.
func GetCallersName(depth int) string {
	pc, file, line, ok := runtime.Caller(depth + 1)
	if !ok {
		return "???"
	}

	fnname := ""
	if fn := runtime.FuncForPC(pc); fn != nil {
		fnname = fn.Name()
	}

	return fmt.Sprintf("%s() at %s:%d", lastComponent(fnname), lastComponent(file), line)
}

// Logs a message to the console, but only if the corresponding key is true in LogKeys.
func LogTo(key string, format string, args ...interface{}) {
	logLock.RLock()
	defer logLock.RUnlock()
	ok := logLevel <= 1 && (logStar || LogKeys[key])

	if ok {
		printf(fgYellow+key+": "+reset+format, args...)
	}
}

func EnableLogKey(key string) {
	logLock.Lock()
	defer logLock.Unlock()
	LogKeys[key] = true
}

func LogEnabled(key string) bool {
	logLock.RLock()
	defer logLock.RUnlock()
	return logStar || LogKeys[key]
}

// Logs a message to the console.
func Log(message string) {
	logLock.RLock()
	defer logLock.RUnlock()
	ok := logLevel <= 1

	if ok {
		print(message)
	}
}

// Logs a formatted message to the console.
func Logf(format string, args ...interface{}) {
	logLock.RLock()
	defer logLock.RUnlock()
	ok := logLevel <= 1

	if ok {
		printf(format, args...)
	}
}

// If the error is not nil, logs its description and the name of the calling function.
// Returns the input error for easy chaining.
func LogError(err error) error {
	if err != nil {
		logLock.RLock()
		ok := logLevel <= 2
		logLock.RUnlock()

		if ok {
			logWithCaller(fgRed, "ERROR", "%v", err)
		}
	}
	return err
}

// Logs a warning to the console
func Warn(format string, args ...interface{}) {
	logLock.RLock()
	ok := logLevel <= 2
	logLock.RUnlock()

	if ok {
		logWithCaller(fgRed, "WARNING", format, args...)
	}
}

// Logs a highlighted message prefixed with "TEMP". This function is intended for
// temporary logging calls added during development and not to be checked in, hence its
// distinctive name (which is visible and easy to search for before committing.)
func TEMP(format string, args ...interface{}) {
	logWithCaller(fgYellow, "TEMP", format, args...)
}

// Logs a warning to the console, then panics.
func LogPanic(format string, args ...interface{}) {
	logWithCaller(fgRed, "PANIC", format, args...)
	panic(fmt.Sprintf(format, args...))
}

// Logs a warning to the console, then exits the process.
func LogFatal(format string, args ...interface{}) {
	logWithCaller(fgRed, "FATAL", format, args...)
	os.Exit(1)
}

func logWithCaller(color string, prefix string, format string, args ...interface{}) {
	message := fmt.Sprintf(format, args...)
	logLock.RLock()
	defer logLock.RUnlock()
	print(color, prefix, ": ", message, reset,
		dim, " -- ", GetCallersName(2), reset)
}

// Simple wrapper that converts Print to Printf.  Assumes caller is holding logLock read lock.
func print(args ...interface{}) {
	ok := logLevel <= 1

	if ok {
		printf("%s", fmt.Sprint(args...))
	}
}

// Logs a formatted message to the underlying logger.  Assumes caller is holding logLock read lock.
func printf(format string, args ...interface{}) {
	ok := logLevel <= 1

	if ok {
		if !logNoTime {
			timestampedFormat := strings.Join([]string{time.Now().Format(ISO8601Format), format}, " ")
			logger.Printf(timestampedFormat, args...)
		} else {
			logger.Printf(format, args...)
		}
	}
}

func lastComponent(path string) string {
	if index := strings.LastIndex(path, "/"); index >= 0 {
		path = path[index+1:]
	} else if index = strings.LastIndex(path, "\\"); index >= 0 {
		path = path[index+1:]
	}
	return path
}

func UpdateLogger(logFilePath string) {
	//Attempt to open file for write at path provided
	fo, err := os.OpenFile(logFilePath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0664)
	if err != nil {
		LogFatal("unable to open logfile for write: %s", logFilePath)
	}

	//defer write lock to here otherwise LogFatal above will deadlock
	logLock.Lock()

	//We keep a reference to the underlying log File as log.Logger and io.Writer
	//have no close() methods and we want to close old files on log rotation
	oldLogFile := logFile
	logFile = fo
	logger = log.New(fo, "", log.Lmicroseconds)
	logLock.Unlock()

	//re-apply log no time flags on new logger
	if logNoTime {
		LogNoTime()
	}

	//If there is a previously opened log file, explicitly close it
	if oldLogFile != nil {
		err = oldLogFile.Close()
		if err != nil {
			Warn("unable to close old log File after updating logger")
		}
	}
}

// ANSI color control escape sequences.
// Shamelessly copied from https://github.com/sqp/godock/blob/master/libs/log/colors.go
var (
	reset,
	bright,
	dim,
	underscore,
	blink,
	reverse,
	hidden,
	fgBlack,
	fgRed,
	fgGreen,
	fgYellow,
	fgBlue,
	fgMagenta,
	fgCyan,
	fgWhite,
	bgBlack,
	bgRed,
	bgGreen,
	bgYellow,
	bgBlue,
	bgMagenta,
	bgCyan,
	bgWhite string
)

func LogColor() {
	reset = "\x1b[0m"
	bright = "\x1b[1m"
	dim = "\x1b[2m"
	underscore = "\x1b[4m"
	blink = "\x1b[5m"
	reverse = "\x1b[7m"
	hidden = "\x1b[8m"
	fgBlack = "\x1b[30m"
	fgRed = "\x1b[31m"
	fgGreen = "\x1b[32m"
	fgYellow = "\x1b[33m"
	fgBlue = "\x1b[34m"
	fgMagenta = "\x1b[35m"
	fgCyan = "\x1b[36m"
	fgWhite = "\x1b[37m"
	bgBlack = "\x1b[40m"
	bgRed = "\x1b[41m"
	bgGreen = "\x1b[42m"
	bgYellow = "\x1b[43m"
	bgBlue = "\x1b[44m"
	bgMagenta = "\x1b[45m"
	bgCyan = "\x1b[46m"
	bgWhite = "\x1b[47m"
}
<|endoftext|>


<commit_before>package runner

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	"github.com/iron-io/worker/common"
	"github.com/iron-io/worker/runner/agent"
	"github.com/iron-io/worker/runner/drivers"
	driverscommon "github.com/iron-io/worker/runner/drivers"
	"github.com/iron-io/worker/runner/drivers/docker"
	"github.com/iron-io/worker/runner/drivers/mock"
)

type Config struct {
	ID      string
	Image   string
	Timeout time.Duration
	AppName string
	Memory  uint64
	Env     map[string]string
	Stdout  io.Writer
	Stderr  io.Writer
}

type Runner struct {
	driver       drivers.Driver
	taskQueue    chan *containerTask
	ml           Logger
	availableMem int64
	usedMem      int64
	usedMemMutex sync.RWMutex
}

var (
	ErrTimeOutNoMemory = errors.New("Task timed out. No available memory.")
	ErrFullQueue       = errors.New("The runner queue is full")

	WaitMemoryTimeout = 10 * time.Second
)

func New(metricLogger Logger) (*Runner, error) {
	// TODO: Is this really required for Titan's driver?
	// Can we remove it?
	env := common.NewEnvironment(func(e *common.Environment) {})

	// TODO: Create a drivers.New(runnerConfig) in Titan
	driver, err := selectDriver("docker", env, &driverscommon.Config{})
	if err != nil {
		return nil, err
	}

	r := &Runner{
		driver:       driver,
		taskQueue:    make(chan *containerTask, 100),
		ml:           metricLogger,
		availableMem: getAvailableMemory(),
		usedMem:      0,
	}

	go r.queueHandler()

	return r, nil
}

// This routine checks for available memory;
// If there's memory then send signal to the task to proceed.
// If there's not available memory to run the task it waits
// If the task waits for more than X seconds it timeouts
func (r *Runner) queueHandler() {
	var task *containerTask
	var waitStart time.Time
	var waitTime time.Duration
	var timedOut bool
	for {
		select {
		case task = <-r.taskQueue:
			waitStart = time.Now()
			timedOut = false
		}

		// Loop waiting for available memory
		canRun := r.checkRequiredMem(task.cfg.Memory)
		for ; !canRun; canRun = r.checkRequiredMem(task.cfg.Memory) {
			waitTime = time.Since(waitStart)
			if waitTime > WaitMemoryTimeout {
				timedOut = true
				break
			}
			time.Sleep(time.Microsecond)
		}

		metricBaseName := fmt.Sprintf("run.%s.", task.cfg.AppName)
		r.ml.LogTime(task.ctx, metricBaseName+"waittime", waitTime)

		if timedOut {
			// Send to a signal to this task saying it cannot run
			r.ml.LogCount(task.ctx, metricBaseName+"timeout", 1)
			task.canRun <- false
			continue
		}

		// Send a signal to this task saying it can run
		task.canRun <- true
	}
}

func (r *Runner) checkRequiredMem(req uint64) bool {
	r.usedMemMutex.RLock()
	defer r.usedMemMutex.RUnlock()
	return r.availableMem-r.usedMem/int64(req)*1024*1024 > 0
}

func (r *Runner) addUsedMem(used int64) {
	r.usedMemMutex.Lock()
	r.usedMem = r.usedMem + used*1024*1024
	if r.usedMem < 0 {
		r.usedMem = 0
	}
	r.usedMemMutex.Unlock()
}

func (r *Runner) checkMemAndUse(req uint64) bool {
	r.usedMemMutex.Lock()
	defer r.usedMemMutex.Unlock()

	used := int64(req) * 1024 * 1024

	if r.availableMem-r.usedMem/used < 0 {
		return false
	}

	r.usedMem += used

	return true
}

func (r *Runner) Run(ctx context.Context, cfg *Config) (drivers.RunResult, error) {
	var err error

	if cfg.Memory == 0 {
		cfg.Memory = 128
	}

	ctask := &containerTask{
		ctx:    ctx,
		cfg:    cfg,
		auth:   &agent.ConfigAuth{},
		canRun: make(chan bool),
	}

	metricBaseName := fmt.Sprintf("run.%s.", cfg.AppName)
	r.ml.LogCount(ctx, metricBaseName+"requests", 1)

	// Check if has enough available memory
	// If available, use it
	if !r.checkMemAndUse(cfg.Memory) {
		// If not, try add task to the queue
		select {
		case r.taskQueue <- ctask:
		default:
			// If queue is full, return error
			r.ml.LogCount(ctx, "queue.full", 1)
			return nil, ErrFullQueue
		}

		// If task was added to the queue, wait for permission
		if ok := <-ctask.canRun; !ok {
			// This task timed out, not available memory
			return nil, ErrTimeOutNoMemory
		}
	} else {
		r.ml.LogTime(ctx, metricBaseName+"waittime", 0)
	}

	closer, err := r.driver.Prepare(ctx, ctask)
	if err != nil {
		return nil, err
	}
	defer closer.Close()

	metricStart := time.Now()

	result, err := r.driver.Run(ctx, ctask)
	if err != nil {
		return nil, err
	}

	r.addUsedMem(-1 * int64(cfg.Memory))

	if result.Status() == "success" {
		r.ml.LogCount(ctx, metricBaseName+"succeeded", 1)
	} else {
		r.ml.LogCount(ctx, metricBaseName+"error", 1)
	}

	metricElapsed := time.Since(metricStart)
	r.ml.LogTime(ctx, metricBaseName+"time", metricElapsed)
	r.ml.LogTime(ctx, "run.exec_time", metricElapsed)

	return result, nil
}

func (r Runner) EnsureImageExists(ctx context.Context, cfg *Config) error {
	ctask := &containerTask{
		cfg:  cfg,
		auth: &agent.ConfigAuth{},
	}

	err := r.driver.EnsureImageExists(ctx, ctask)
	if err != nil {
		return err
	}
	return nil
}

func selectDriver(driver string, env *common.Environment, conf *driverscommon.Config) (drivers.Driver, error) {
	switch driver {
	case "docker":
		docker := docker.NewDocker(env, *conf)
		return docker, nil
	case "mock":
		return mock.New(), nil
	}
	return nil, fmt.Errorf("driver %v not found", driver)
}

func getAvailableMemory() int64 {
	const tooBig = 322122547200 // #300GB or 0, biggest aws instance is 244GB

	var availableMemory uint64
	if os.Getenv("IGNORE_MEMORY") == "1" {
		availableMemory = tooBig
	} else {
		availableMemory, err := checkCgroup()
		if err != nil {
			logrus.WithError(err).Error("Error checking for cgroup memory limits, falling back to host memory available..")
		}
		if availableMemory > tooBig || availableMemory == 0 {
			// Then -m flag probably wasn't set, so use max available on system
			availableMemory, err = checkProc()
			if availableMemory > tooBig || availableMemory == 0 {
				logrus.WithError(err).Fatal("Your Linux version is too old (<3.14) then we can't get the proper information to . You must specify the maximum available memory by passing the -m command with docker run when starting the runner via docker, eg:  `docker run -m 2G ...`")
			}
		}
	}

	return int64(availableMemory)
}

func checkCgroup() (uint64, error) {
	f, err := os.Open("/sys/fs/cgroup/memory/memory.limit_in_bytes")
	if err != nil {
		return 0, err
	}
	defer f.Close()
	b, err := ioutil.ReadAll(f)
	limBytes := string(b)
	limBytes = strings.TrimSpace(limBytes)
	if err != nil {
		return 0, err
	}
	return strconv.ParseUint(limBytes, 10, 64)
}

func checkProc() (uint64, error) {
	f, err := os.Open("/proc/meminfo")
	if err != nil {
		return 0, err
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		b := scanner.Text()
		if !strings.HasPrefix(b, "MemAvailable") {
			continue
		}

		// expect form:
		// MemAvailable: 1234567890 kB
		tri := strings.Fields(b)
		if len(tri) != 3 {
			return 0, fmt.Errorf("MemAvailable line has unexpected format: %v", b)
		}

		c, err := strconv.ParseUint(tri[1], 10, 64)
		if err != nil {
			return 0, fmt.Errorf("Could not parse MemAvailable: %v", b)
		}
		switch tri[2] { // convert units to bytes
		case "kB":
			c *= 1024
		case "MB":
			c *= 1024 * 1024
		default:
			return 0, fmt.Errorf("Unexpected units for MemAvailable in /proc/meminfo, need kB or MB, got: %v", tri[2])
		}
		return c, nil
	}

	return 0, fmt.Errorf("Didn't find MemAvailable in /proc/meminfo, kernel is probably < 3.14")
}
<commit_msg>added wait time total and reserved memory metrics<commit_after>package runner

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	"github.com/iron-io/worker/common"
	"github.com/iron-io/worker/runner/agent"
	"github.com/iron-io/worker/runner/drivers"
	driverscommon "github.com/iron-io/worker/runner/drivers"
	"github.com/iron-io/worker/runner/drivers/docker"
	"github.com/iron-io/worker/runner/drivers/mock"
)

type Config struct {
	ID      string
	Image   string
	Timeout time.Duration
	AppName string
	Memory  uint64
	Env     map[string]string
	Stdout  io.Writer
	Stderr  io.Writer
}

type Runner struct {
	driver       drivers.Driver
	taskQueue    chan *containerTask
	ml           Logger
	availableMem int64
	usedMem      int64
	usedMemMutex sync.RWMutex
}

var (
	ErrTimeOutNoMemory = errors.New("Task timed out. No available memory.")
	ErrFullQueue       = errors.New("The runner queue is full")

	WaitMemoryTimeout = 10 * time.Second
)

func New(metricLogger Logger) (*Runner, error) {
	// TODO: Is this really required for Titan's driver?
	// Can we remove it?
	env := common.NewEnvironment(func(e *common.Environment) {})

	// TODO: Create a drivers.New(runnerConfig) in Titan
	driver, err := selectDriver("docker", env, &driverscommon.Config{})
	if err != nil {
		return nil, err
	}

	r := &Runner{
		driver:       driver,
		taskQueue:    make(chan *containerTask, 100),
		ml:           metricLogger,
		availableMem: getAvailableMemory(),
		usedMem:      0,
	}

	go r.queueHandler()

	return r, nil
}

// This routine checks for available memory;
// If there's memory then send signal to the task to proceed.
// If there's not available memory to run the task it waits
// If the task waits for more than X seconds it timeouts
func (r *Runner) queueHandler() {
	var task *containerTask
	var waitStart time.Time
	var waitTime time.Duration
	var timedOut bool
	for {
		select {
		case task = <-r.taskQueue:
			waitStart = time.Now()
			timedOut = false
		}

		// Loop waiting for available memory
		canRun := r.checkRequiredMem(task.cfg.Memory)
		for ; !canRun; canRun = r.checkRequiredMem(task.cfg.Memory) {
			waitTime = time.Since(waitStart)
			if waitTime > WaitMemoryTimeout {
				timedOut = true
				break
			}
			time.Sleep(time.Microsecond)
		}

		metricBaseName := fmt.Sprintf("run.%s.", task.cfg.AppName)
		r.ml.LogTime(task.ctx, metricBaseName+"wait_time", waitTime)
		r.ml.LogTime(task.ctx, "run.wait_time", waitTime)

		if timedOut {
			// Send to a signal to this task saying it cannot run
			r.ml.LogCount(task.ctx, metricBaseName+"timeout", 1)
			task.canRun <- false
			continue
		}

		// Send a signal to this task saying it can run
		task.canRun <- true
	}
}

func (r *Runner) checkRequiredMem(req uint64) bool {
	r.usedMemMutex.RLock()
	defer r.usedMemMutex.RUnlock()
	return r.availableMem-r.usedMem/int64(req)*1024*1024 > 0
}

func (r *Runner) addUsedMem(used int64) {
	r.usedMemMutex.Lock()
	r.usedMem = r.usedMem + used*1024*1024
	if r.usedMem < 0 {
		r.usedMem = 0
	}
	r.usedMemMutex.Unlock()
}

func (r *Runner) checkMemAndUse(req uint64) bool {
	r.usedMemMutex.Lock()
	defer r.usedMemMutex.Unlock()

	used := int64(req) * 1024 * 1024

	if r.availableMem-r.usedMem/used < 0 {
		return false
	}

	r.usedMem += used

	return true
}

func (r *Runner) Run(ctx context.Context, cfg *Config) (drivers.RunResult, error) {
	var err error

	if cfg.Memory == 0 {
		cfg.Memory = 128
	}

	ctask := &containerTask{
		ctx:    ctx,
		cfg:    cfg,
		auth:   &agent.ConfigAuth{},
		canRun: make(chan bool),
	}

	metricBaseName := fmt.Sprintf("run.%s.", cfg.AppName)
	r.ml.LogCount(ctx, metricBaseName+"requests", 1)

	// Check if has enough available memory
	// If available, use it
	if !r.checkMemAndUse(cfg.Memory) {
		// If not, try add task to the queue
		select {
		case r.taskQueue <- ctask:
		default:
			// If queue is full, return error
			r.ml.LogCount(ctx, "queue.full", 1)
			return nil, ErrFullQueue
		}

		// If task was added to the queue, wait for permission
		if ok := <-ctask.canRun; !ok {
			// This task timed out, not available memory
			return nil, ErrTimeOutNoMemory
		}
	} else {
		r.ml.LogTime(ctx, metricBaseName+"waittime", 0)
	}

	closer, err := r.driver.Prepare(ctx, ctask)
	if err != nil {
		return nil, err
	}
	defer closer.Close()

	metricStart := time.Now()

	result, err := r.driver.Run(ctx, ctask)
	if err != nil {
		return nil, err
	}

	r.addUsedMem(-1 * int64(cfg.Memory))
	r.ml.LogGauge(ctx, metricBaseName+"reserved_memory", int(r.usedMem))

	if result.Status() == "success" {
		r.ml.LogCount(ctx, metricBaseName+"succeeded", 1)
	} else {
		r.ml.LogCount(ctx, metricBaseName+"error", 1)
	}

	metricElapsed := time.Since(metricStart)
	r.ml.LogTime(ctx, metricBaseName+"time", metricElapsed)
	r.ml.LogTime(ctx, "run.exec_time", metricElapsed)

	return result, nil
}

func (r Runner) EnsureImageExists(ctx context.Context, cfg *Config) error {
	ctask := &containerTask{
		cfg:  cfg,
		auth: &agent.ConfigAuth{},
	}

	err := r.driver.EnsureImageExists(ctx, ctask)
	if err != nil {
		return err
	}
	return nil
}

func selectDriver(driver string, env *common.Environment, conf *driverscommon.Config) (drivers.Driver, error) {
	switch driver {
	case "docker":
		docker := docker.NewDocker(env, *conf)
		return docker, nil
	case "mock":
		return mock.New(), nil
	}
	return nil, fmt.Errorf("driver %v not found", driver)
}

func getAvailableMemory() int64 {
	const tooBig = 322122547200 // #300GB or 0, biggest aws instance is 244GB

	var availableMemory uint64
	if os.Getenv("IGNORE_MEMORY") == "1" {
		availableMemory = tooBig
	} else {
		availableMemory, err := checkCgroup()
		if err != nil {
			logrus.WithError(err).Error("Error checking for cgroup memory limits, falling back to host memory available..")
		}
		if availableMemory > tooBig || availableMemory == 0 {
			// Then -m flag probably wasn't set, so use max available on system
			availableMemory, err = checkProc()
			if availableMemory > tooBig || availableMemory == 0 {
				logrus.WithError(err).Fatal("Your Linux version is too old (<3.14) then we can't get the proper information to . You must specify the maximum available memory by passing the -m command with docker run when starting the runner via docker, eg:  `docker run -m 2G ...`")
			}
		}
	}

	return int64(availableMemory)
}

func checkCgroup() (uint64, error) {
	f, err := os.Open("/sys/fs/cgroup/memory/memory.limit_in_bytes")
	if err != nil {
		return 0, err
	}
	defer f.Close()
	b, err := ioutil.ReadAll(f)
	limBytes := string(b)
	limBytes = strings.TrimSpace(limBytes)
	if err != nil {
		return 0, err
	}
	return strconv.ParseUint(limBytes, 10, 64)
}

func checkProc() (uint64, error) {
	f, err := os.Open("/proc/meminfo")
	if err != nil {
		return 0, err
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		b := scanner.Text()
		if !strings.HasPrefix(b, "MemAvailable") {
			continue
		}

		// expect form:
		// MemAvailable: 1234567890 kB
		tri := strings.Fields(b)
		if len(tri) != 3 {
			return 0, fmt.Errorf("MemAvailable line has unexpected format: %v", b)
		}

		c, err := strconv.ParseUint(tri[1], 10, 64)
		if err != nil {
			return 0, fmt.Errorf("Could not parse MemAvailable: %v", b)
		}
		switch tri[2] { // convert units to bytes
		case "kB":
			c *= 1024
		case "MB":
			c *= 1024 * 1024
		default:
			return 0, fmt.Errorf("Unexpected units for MemAvailable in /proc/meminfo, need kB or MB, got: %v", tri[2])
		}
		return c, nil
	}

	return 0, fmt.Errorf("Didn't find MemAvailable in /proc/meminfo, kernel is probably < 3.14")
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"net"
	"net/http"
	"os"

	log "github.com/Sirupsen/logrus"
	ghandlers "github.com/gorilla/handlers"
	"github.com/rs/cors"
	"github.com/urfave/negroni"

	//"github.com/danielkrainas/csense/api/errcode"
	"github.com/danielkrainas/csense/api/server/handlers"
	//"github.com/danielkrainas/csense/api/v1"
	"github.com/danielkrainas/csense/configuration"
	"github.com/danielkrainas/csense/context"
)

type Server struct {
	config *configuration.Config
	app    *handlers.App
	server *http.Server
}

func New(ctx context.Context, config *configuration.Config) (*Server, error) {
	app, err := handlers.NewApp(ctx, config)
	if err != nil {
		return nil, fmt.Errorf("error creating server app: %v", err)
	}

	handler := alive("/", app)
	handler = panicHandler(handler)
	handler = ghandlers.CombinedLoggingHandler(os.Stdout, handler)

	n := negroni.New()

	n.Use(cors.New(cors.Options{
		AllowedOrigins:   config.HTTP.CORS.Origins,
		AllowedMethods:   config.HTTP.CORS.Methods,
		AllowCredentials: true,
		AllowedHeaders:   config.HTTP.CORS.Headers,
		Debug:            config.HTTP.CORS.Debug,
	}))

	n.UseHandler(handler)

	s := &Server{
		app:    app,
		config: config,
		server: &http.Server{
			Addr:    config.HTTP.Addr,
			Handler: n,
		},
	}

	return s, nil
}

func (server *Server) ListenAndServe() error {
	config := server.config
	ln, err := net.Listen("tcp", config.HTTP.Addr)
	if err != nil {
		return err
	}

	context.GetLogger(server.app).Infof("listening on %v", ln.Addr())
	return server.server.Serve(ln)
}

func panicHandler(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if err := recover(); err != nil {
				log.Panicf("%v", err)
			}
		}()

		handler.ServeHTTP(w, r)
	})
}

func alive(path string, handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == path {
			w.Header().Set("Cache-Control", "no-cache")
			w.WriteHeader(http.StatusOK)
			return
		}

		handler.ServeHTTP(w, r)
	})
}
<commit_msg>trim dead code<commit_after>package server

import (
	"fmt"
	"net"
	"net/http"
	"os"

	log "github.com/Sirupsen/logrus"
	ghandlers "github.com/gorilla/handlers"
	"github.com/rs/cors"
	"github.com/urfave/negroni"

	"github.com/danielkrainas/csense/api/server/handlers"
	"github.com/danielkrainas/csense/configuration"
	"github.com/danielkrainas/csense/context"
)

type Server struct {
	config *configuration.Config
	app    *handlers.App
	server *http.Server
}

func New(ctx context.Context, config *configuration.Config) (*Server, error) {
	app, err := handlers.NewApp(ctx, config)
	if err != nil {
		return nil, fmt.Errorf("error creating server app: %v", err)
	}

	handler := alive("/", app)
	handler = panicHandler(handler)
	handler = ghandlers.CombinedLoggingHandler(os.Stdout, handler)

	n := negroni.New()

	n.Use(cors.New(cors.Options{
		AllowedOrigins:   config.HTTP.CORS.Origins,
		AllowedMethods:   config.HTTP.CORS.Methods,
		AllowCredentials: true,
		AllowedHeaders:   config.HTTP.CORS.Headers,
		Debug:            config.HTTP.CORS.Debug,
	}))

	n.UseHandler(handler)

	s := &Server{
		app:    app,
		config: config,
		server: &http.Server{
			Addr:    config.HTTP.Addr,
			Handler: n,
		},
	}

	return s, nil
}

func (server *Server) ListenAndServe() error {
	config := server.config
	ln, err := net.Listen("tcp", config.HTTP.Addr)
	if err != nil {
		return err
	}

	context.GetLogger(server.app).Infof("listening on %v", ln.Addr())
	return server.server.Serve(ln)
}

func panicHandler(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if err := recover(); err != nil {
				log.Panicf("%v", err)
			}
		}()

		handler.ServeHTTP(w, r)
	})
}

func alive(path string, handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == path {
			w.Header().Set("Cache-Control", "no-cache")
			w.WriteHeader(http.StatusOK)
			return
		}

		handler.ServeHTTP(w, r)
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2016 caicloud authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	restful "github.com/emicklei/go-restful"
	swagger "github.com/emicklei/go-restful-swagger12"
	log "github.com/golang/glog"
	"github.com/zoumo/logdog"

	"github.com/caicloud/cyclone/cmd/worker/options"
	"github.com/caicloud/cyclone/pkg/cloud"
	"github.com/caicloud/cyclone/pkg/event"
	"github.com/caicloud/cyclone/pkg/server/router"
	"github.com/caicloud/cyclone/pkg/store"
)

// APIServer ...
type APIServer struct {
	Config        *APIServerOptions
	WorkerOptions *options.WorkerOptions
}

// PrepareRun prepare for apiserver running
func (s *APIServer) PrepareRun() (*PreparedAPIServer, error) {
	// init api doc
	if s.Config.ShowAPIDoc {
		// Open http://localhost:7099/apidocs and enter http://localhost:7099/apidocs.json in the api input field.
		config := swagger.Config{
			WebServices:    restful.DefaultContainer.RegisteredWebServices(), // you control what services are visible.
			WebServicesUrl: fmt.Sprintf(s.Config.CycloneAddrTemplate, s.Config.CyclonePort),
			ApiPath:        "/apidocs.json",

			// Optionally, specify where the UI is located.
			SwaggerPath:     "/apidocs/",
			SwaggerFilePath: "./node_modules/swagger-ui/dist",
		}
		swagger.InstallSwaggerService(config)
	}

	closing := make(chan struct{})

	// init database
	mclosed, err := store.Init(s.Config.MongoDBHost, s.Config.MongoGracePeriod, closing, s.Config.SaltKey)
	if err != nil {
		return nil, err
	}

	go background(closing, mclosed)

	// init event manager
	event.Init(s.WorkerOptions)

	if err = cloud.InitCloud(s.Config.CloudAutoDiscovery); err != nil {
		log.Error(err)
		return nil, err
	}

	return &PreparedAPIServer{s}, nil
}

// PreparedAPIServer is a prepared api server
type PreparedAPIServer struct {
	*APIServer
}

// Run start a api server
func (s *PreparedAPIServer) Run(stopCh <-chan struct{}) error {
	dataStore := store.NewStore()
	defer dataStore.Close()

	// Initialize the V1 API.
	if err := router.InitRouters(dataStore, s.Config.RecordRotationThreshold); err != nil {
		logdog.Fatal(err)
		return err
	}

	// start server
	server := &http.Server{Addr: fmt.Sprintf(":%d", s.Config.CyclonePort), Handler: restful.DefaultContainer}
	logdog.Infof("cyclone server listening on %d", s.Config.CyclonePort)
	logdog.Fatal(server.ListenAndServe())
	// <-stopCh
	return nil
}

// background must be a daemon goroutine for Cyclone server.
// It can catch system signal and send signal to other goroutine before program exits.
func background(closing, mclosed chan struct{}) {
	closed := []chan struct{}{mclosed}
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
	<-sigs
	log.Info("capture system signal, will close \"closing\" channel")
	close(closing)
	for _, c := range closed {
		<-c
	}
	log.Info("exit the process with 0")
	os.Exit(0)
}
<commit_msg>InitRouters must be called before InstallSwaggerService (#538)<commit_after>/*
Copyright 2016 caicloud authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	restful "github.com/emicklei/go-restful"
	swagger "github.com/emicklei/go-restful-swagger12"
	log "github.com/golang/glog"
	"github.com/zoumo/logdog"

	"github.com/caicloud/cyclone/cmd/worker/options"
	"github.com/caicloud/cyclone/pkg/cloud"
	"github.com/caicloud/cyclone/pkg/event"
	"github.com/caicloud/cyclone/pkg/server/router"
	"github.com/caicloud/cyclone/pkg/store"
)

// APIServer ...
type APIServer struct {
	Config        *APIServerOptions
	WorkerOptions *options.WorkerOptions
}

// PrepareRun prepare for apiserver running
func (s *APIServer) PrepareRun() (*PreparedAPIServer, error) {
	closing := make(chan struct{})

	// init database
	mclosed, err := store.Init(s.Config.MongoDBHost, s.Config.MongoGracePeriod, closing, s.Config.SaltKey)
	if err != nil {
		return nil, err
	}

	go background(closing, mclosed)

	// init event manager
	event.Init(s.WorkerOptions)

	if err = cloud.InitCloud(s.Config.CloudAutoDiscovery); err != nil {
		log.Error(err)
		return nil, err
	}

	return &PreparedAPIServer{s}, nil
}

// PreparedAPIServer is a prepared api server
type PreparedAPIServer struct {
	*APIServer
}

// Run start a api server
func (s *PreparedAPIServer) Run(stopCh <-chan struct{}) error {
	dataStore := store.NewStore()
	defer dataStore.Close()

	// Initialize the V1 API.
	if err := router.InitRouters(dataStore, s.Config.RecordRotationThreshold); err != nil {
		logdog.Fatal(err)
		return err
	}

	// init api doc
	if s.Config.ShowAPIDoc {
		// Open http://localhost:7099/apidocs and enter http://localhost:7099/apidocs.json in the api input field.
		config := swagger.Config{
			WebServices:    restful.DefaultContainer.RegisteredWebServices(), // you control what services are visible.
			WebServicesUrl: fmt.Sprintf(s.Config.CycloneAddrTemplate, s.Config.CyclonePort),
			ApiPath:        "/apidocs.json",

			// Optionally, specify where the UI is located.
			SwaggerPath:     "/apidocs/",
			SwaggerFilePath: "./node_modules/swagger-ui/dist",
		}
		swagger.InstallSwaggerService(config)
	}

	// start server
	server := &http.Server{Addr: fmt.Sprintf(":%d", s.Config.CyclonePort), Handler: restful.DefaultContainer}
	logdog.Infof("cyclone server listening on %d", s.Config.CyclonePort)
	logdog.Fatal(server.ListenAndServe())
	// <-stopCh
	return nil
}

// background must be a daemon goroutine for Cyclone server.
// It can catch system signal and send signal to other goroutine before program exits.
func background(closing, mclosed chan struct{}) {
	closed := []chan struct{}{mclosed}
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
	<-sigs
	log.Info("capture system signal, will close \"closing\" channel")
	close(closing)
	for _, c := range closed {
		<-c
	}
	log.Info("exit the process with 0")
	os.Exit(0)
}
<|endoftext|>