<commit_before>/* walter: a deployment pipeline template
 * Copyright (C) 2014 Recruit Technologies Co., Ltd. and contributors
 * (see CONTRIBUTORS.md)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package messengers

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/recruit-tech/walter/log"
)

type Slack struct {
	Channel   string   `config:"channel" json:"channel"`
	UserName  string   `config:"username" json:"username"`
	IconEmoji string   `config:"icon" json:"icon_emoji,omitempty"`
	IncomingUrl string `config:"url" json:"-"` // not map to json
}

// To avoid the infinite recursion
// (see http://stackoverflow.com/questions/23045884/can-i-use-marshaljson-to-add-arbitrary-fields-to-a-json-encoding-in-golang)
type FakeSlack Slack

func (self *Slack) Post(message string) bool {
	params, _ := json.Marshal(struct {
		FakeSlack
		Text string `json:"text"`
	}{
		FakeSlack : FakeSlack(*self),
		Text:  message,
	})

	resp, err := http.PostForm(
		self.IncomingUrl,
		url.Values{"payload": {string(params)}},
	)
	defer resp.Body.Close()

	if err != nil {
		log.Errorf("Failed post message...: %s", message);
		return false
	}

	if body, err := ioutil.ReadAll(resp.Body); err == nil {
		log.Infof("Post result...: %s", body)
		return true
	}
	log.Errorf("Failed to read result...")
	return false
}
<commit_msg>Add sharp to the beggining of channel name when specified channel name does not starts with sharp<commit_after>/* walter: a deployment pipeline template
 * Copyright (C) 2014 Recruit Technologies Co., Ltd. and contributors
 * (see CONTRIBUTORS.md)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package messengers

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/recruit-tech/walter/log"
)

type Slack struct {
	Channel   string   `config:"channel" json:"channel"`
	UserName  string   `config:"username" json:"username"`
	IconEmoji string   `config:"icon" json:"icon_emoji,omitempty"`
	IncomingUrl string `config:"url" json:"-"` // not map to json
}

// To avoid the infinite recursion
// (see http://stackoverflow.com/questions/23045884/can-i-use-marshaljson-to-add-arbitrary-fields-to-a-json-encoding-in-golang)
type FakeSlack Slack

func (self *Slack) Post(message string) bool {
	if self.Channel[0] != '#' {
		log.Infof("Add # to channel name: %s", self.Channel)
		self.Channel = "#" + self.Channel
	}

	params, _ := json.Marshal(struct {
		FakeSlack
		Text string `json:"text"`
	}{
		FakeSlack : FakeSlack(*self),
		Text:  message,
	})

	resp, err := http.PostForm(
		self.IncomingUrl,
		url.Values{"payload": {string(params)}},
	)
	defer resp.Body.Close()

	if err != nil {
		log.Errorf("Failed post message to Slack...: %s", message);
		return false
	}

	if body, err := ioutil.ReadAll(resp.Body); err == nil {
		log.Infof("Slack post result...: %s", body)
		return true
	}
	log.Errorf("Failed to read result from Slack...")
	return false
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package servicecatalog

import (
	"reflect"
	"strconv"
	"time"

	"k8s.io/api/core/v1"
	settings "k8s.io/api/settings/v1alpha1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/watch"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	imageutils "k8s.io/kubernetes/test/utils/image"
)

var _ = SIGDescribe("[Feature:PodPreset] PodPreset", func() {
	f := framework.NewDefaultFramework("podpreset")

	var podClient *framework.PodClient
	BeforeEach(func() {
		// only run on gce for the time being til we find an easier way to update
		// the admission controllers used on the others
		framework.SkipUnlessProviderIs("gce")
		podClient = f.PodClient()
	})

	// Simplest case: all pods succeed promptly
	It("should create a pod preset", func() {
		By("Creating a pod preset")

		pip := &settings.PodPreset{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "hello",
				Namespace: f.Namespace.Name,
			},
			Spec: settings.PodPresetSpec{
				Selector: metav1.LabelSelector{
					MatchExpressions: []metav1.LabelSelectorRequirement{
						{
							Key:      "security",
							Operator: metav1.LabelSelectorOpIn,
							Values:   []string{"S2"},
						},
					},
				},
				Volumes: []v1.Volume{{Name: "vol", VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}},
				VolumeMounts: []v1.VolumeMount{
					{Name: "vol", MountPath: "/foo"},
				},
				Env: []v1.EnvVar{{Name: "abc", Value: "value"}, {Name: "ABC", Value: "value"}},
			},
		}

		_, err := createPodPreset(f.ClientSet, f.Namespace.Name, pip)
		if errors.IsNotFound(err) {
			framework.Skipf("podpresets requires k8s.io/api/settings/v1alpha1 to be enabled")
		}
		Expect(err).NotTo(HaveOccurred())

		By("creating the pod")
		name := "pod-preset-pod"
		value := strconv.Itoa(time.Now().Nanosecond())
		pod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:      name,
				Namespace: f.Namespace.Name,
				Labels: map[string]string{
					"name":     "foo",
					"time":     value,
					"security": "S2",
				},
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:  "nginx",
						Image: imageutils.GetE2EImage(imageutils.Nginx),
					},
				},
				InitContainers: []v1.Container{
					{
						Name:    "init1",
						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
						Command: []string{"/bin/true"},
					},
				},
			},
		}

		By("setting up watch")
		selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": value}))
		options := metav1.ListOptions{LabelSelector: selector.String()}
		pods, err := podClient.List(options)
		Expect(err).NotTo(HaveOccurred(), "failed to query for pod")
		Expect(len(pods.Items)).To(Equal(0))
		options = metav1.ListOptions{
			LabelSelector:   selector.String(),
			ResourceVersion: pods.ListMeta.ResourceVersion,
		}
		w, err := podClient.Watch(options)
		Expect(err).NotTo(HaveOccurred(), "failed to set up watch")

		By("submitting the pod to kubernetes")
		podClient.Create(pod)

		By("verifying the pod is in kubernetes")
		selector = labels.SelectorFromSet(labels.Set(map[string]string{"time": value}))
		options = metav1.ListOptions{LabelSelector: selector.String()}
		pods, err = podClient.List(options)
		Expect(err).NotTo(HaveOccurred(), "failed to query for pod")
		Expect(len(pods.Items)).To(Equal(1))

		By("verifying pod creation was observed")
		select {
		case event, _ := <-w.ResultChan():
			if event.Type != watch.Added {
				framework.Failf("Failed to observe pod creation: %v", event)
			}
		case <-time.After(framework.PodStartTimeout):
			framework.Failf("Timeout while waiting for pod creation")
		}

		// We need to wait for the pod to be running, otherwise the deletion
		// may be carried out immediately rather than gracefully.
		framework.ExpectNoError(f.WaitForPodRunning(pod.Name))

		By("ensuring pod is modified")
		// save the running pod
		pod, err = podClient.Get(pod.Name, metav1.GetOptions{})
		Expect(err).NotTo(HaveOccurred(), "failed to GET scheduled pod")

		// check the annotation is there
		if _, ok := pod.Annotations["podpreset.admission.kubernetes.io/podpreset-hello"]; !ok {
			framework.Failf("Annotation not found in pod annotations: \n%v\n", pod.Annotations)
		}

		// verify the env is the same
		if !reflect.DeepEqual(pip.Spec.Env, pod.Spec.Containers[0].Env) {
			framework.Failf("env of pod container does not match the env of the pip: expected %#v, got: %#v", pip.Spec.Env, pod.Spec.Containers[0].Env)
		}
		if !reflect.DeepEqual(pip.Spec.Env, pod.Spec.InitContainers[0].Env) {
			framework.Failf("env of pod init container does not match the env of the pip: expected %#v, got: %#v", pip.Spec.Env, pod.Spec.InitContainers[0].Env)
		}
	})

	It("should not modify the pod on conflict", func() {
		By("Creating a pod preset")

		pip := &settings.PodPreset{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "hello",
				Namespace: f.Namespace.Name,
			},
			Spec: settings.PodPresetSpec{
				Selector: metav1.LabelSelector{
					MatchExpressions: []metav1.LabelSelectorRequirement{
						{
							Key:      "security",
							Operator: metav1.LabelSelectorOpIn,
							Values:   []string{"S2"},
						},
					},
				},
				Volumes: []v1.Volume{{Name: "vol", VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}},
				VolumeMounts: []v1.VolumeMount{
					{Name: "vol", MountPath: "/foo"},
				},
				Env: []v1.EnvVar{{Name: "abc", Value: "value"}, {Name: "ABC", Value: "value"}},
			},
		}

		_, err := createPodPreset(f.ClientSet, f.Namespace.Name, pip)
		if errors.IsNotFound(err) {
			framework.Skipf("podpresets requires k8s.io/api/settings/v1alpha1 to be enabled")
		}
		Expect(err).NotTo(HaveOccurred())

		By("creating the pod")
		name := "pod-preset-pod"
		value := strconv.Itoa(time.Now().Nanosecond())
		originalPod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:      name,
				Namespace: f.Namespace.Name,
				Labels: map[string]string{
					"name":     "foo",
					"time":     value,
					"security": "S2",
				},
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:  "nginx",
						Image: imageutils.GetE2EImage(imageutils.Nginx),
						Env:   []v1.EnvVar{{Name: "abc", Value: "value2"}, {Name: "ABC", Value: "value"}},
					},
				},
				InitContainers: []v1.Container{
					{
						Name:    "init1",
						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
						Env:     []v1.EnvVar{{Name: "abc", Value: "value2"}, {Name: "ABC", Value: "    value"}},
						Command: []string{"/bin/true"},
					},
				},
			},
		}

		By("setting up watch")
		selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": value}))
		options := metav1.ListOptions{LabelSelector: selector.String()}
		pods, err := podClient.List(options)
		Expect(err).NotTo(HaveOccurred(), "failed to query for pod")
		Expect(len(pods.Items)).To(Equal(0))
		options = metav1.ListOptions{
			LabelSelector:   selector.String(),
			ResourceVersion: pods.ListMeta.ResourceVersion,
		}
		w, err := podClient.Watch(options)
		Expect(err).NotTo(HaveOccurred(), "failed to set up watch")

		By("submitting the pod to kubernetes")
		podClient.Create(originalPod)

		By("verifying the pod is in kubernetes")
		selector = labels.SelectorFromSet(labels.Set(map[string]string{"time": value}))
		options = metav1.ListOptions{LabelSelector: selector.String()}
		pods, err = podClient.List(options)
		Expect(err).NotTo(HaveOccurred(), "failed to query for pod")
		Expect(len(pods.Items)).To(Equal(1))

		By("verifying pod creation was observed")
		select {
		case event, _ := <-w.ResultChan():
			if event.Type != watch.Added {
				framework.Failf("Failed to observe pod creation: %v", event)
			}
		case <-time.After(framework.PodStartTimeout):
			framework.Failf("Timeout while waiting for pod creation")
		}

		// We need to wait for the pod to be running, otherwise the deletion
		// may be carried out immediately rather than gracefully.
		framework.ExpectNoError(f.WaitForPodRunning(originalPod.Name))

		By("ensuring pod is modified")
		// save the running pod
		pod, err := podClient.Get(originalPod.Name, metav1.GetOptions{})
		Expect(err).NotTo(HaveOccurred(), "failed to GET scheduled pod")

		// check the annotation is not there
		if _, ok := pod.Annotations["podpreset.admission.kubernetes.io/podpreset-hello"]; ok {
			framework.Failf("Annotation found in pod annotations and should not be: \n%v\n", pod.Annotations)
		}

		// verify the env is the same
		if !reflect.DeepEqual(originalPod.Spec.Containers[0].Env, pod.Spec.Containers[0].Env) {
			framework.Failf("env of pod container does not match the env of the original pod: expected %#v, got: %#v", originalPod.Spec.Containers[0].Env, pod.Spec.Containers[0].Env)
		}
		if !reflect.DeepEqual(originalPod.Spec.InitContainers[0].Env, pod.Spec.InitContainers[0].Env) {
			framework.Failf("env of pod init container does not match the env of the original pod: expected %#v, g    ot: %#v", originalPod.Spec.InitContainers[0].Env, pod.Spec.InitContainers[0].Env)
		}

	})
})

func getPodPreset(c clientset.Interface, ns, name string) (*settings.PodPreset, error) {
	return c.SettingsV1alpha1().PodPresets(ns).Get(name, metav1.GetOptions{})
}

func createPodPreset(c clientset.Interface, ns string, job *settings.PodPreset) (*settings.PodPreset, error) {
	return c.SettingsV1alpha1().PodPresets(ns).Create(job)
}

func updatePodPreset(c clientset.Interface, ns string, job *settings.PodPreset) (*settings.PodPreset, error) {
	return c.SettingsV1alpha1().PodPresets(ns).Update(job)
}

func deletePodPreset(c clientset.Interface, ns, name string) error {
	return c.SettingsV1alpha1().PodPresets(ns).Delete(name, nil)
}
<commit_msg>Removed erroneous spaces in PodPreset E2E tests<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package servicecatalog

import (
	"reflect"
	"strconv"
	"time"

	"k8s.io/api/core/v1"
	settings "k8s.io/api/settings/v1alpha1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/watch"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	imageutils "k8s.io/kubernetes/test/utils/image"
)

var _ = SIGDescribe("[Feature:PodPreset] PodPreset", func() {
	f := framework.NewDefaultFramework("podpreset")

	var podClient *framework.PodClient
	BeforeEach(func() {
		// only run on gce for the time being til we find an easier way to update
		// the admission controllers used on the others
		framework.SkipUnlessProviderIs("gce")
		podClient = f.PodClient()
	})

	// Simplest case: all pods succeed promptly
	It("should create a pod preset", func() {
		By("Creating a pod preset")

		pip := &settings.PodPreset{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "hello",
				Namespace: f.Namespace.Name,
			},
			Spec: settings.PodPresetSpec{
				Selector: metav1.LabelSelector{
					MatchExpressions: []metav1.LabelSelectorRequirement{
						{
							Key:      "security",
							Operator: metav1.LabelSelectorOpIn,
							Values:   []string{"S2"},
						},
					},
				},
				Volumes: []v1.Volume{{Name: "vol", VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}},
				VolumeMounts: []v1.VolumeMount{
					{Name: "vol", MountPath: "/foo"},
				},
				Env: []v1.EnvVar{{Name: "abc", Value: "value"}, {Name: "ABC", Value: "value"}},
			},
		}

		_, err := createPodPreset(f.ClientSet, f.Namespace.Name, pip)
		if errors.IsNotFound(err) {
			framework.Skipf("podpresets requires k8s.io/api/settings/v1alpha1 to be enabled")
		}
		Expect(err).NotTo(HaveOccurred())

		By("creating the pod")
		name := "pod-preset-pod"
		value := strconv.Itoa(time.Now().Nanosecond())
		pod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:      name,
				Namespace: f.Namespace.Name,
				Labels: map[string]string{
					"name":     "foo",
					"time":     value,
					"security": "S2",
				},
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:  "nginx",
						Image: imageutils.GetE2EImage(imageutils.Nginx),
					},
				},
				InitContainers: []v1.Container{
					{
						Name:    "init1",
						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
						Command: []string{"/bin/true"},
					},
				},
			},
		}

		By("setting up watch")
		selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": value}))
		options := metav1.ListOptions{LabelSelector: selector.String()}
		pods, err := podClient.List(options)
		Expect(err).NotTo(HaveOccurred(), "failed to query for pod")
		Expect(len(pods.Items)).To(Equal(0))
		options = metav1.ListOptions{
			LabelSelector:   selector.String(),
			ResourceVersion: pods.ListMeta.ResourceVersion,
		}
		w, err := podClient.Watch(options)
		Expect(err).NotTo(HaveOccurred(), "failed to set up watch")

		By("submitting the pod to kubernetes")
		podClient.Create(pod)

		By("verifying the pod is in kubernetes")
		selector = labels.SelectorFromSet(labels.Set(map[string]string{"time": value}))
		options = metav1.ListOptions{LabelSelector: selector.String()}
		pods, err = podClient.List(options)
		Expect(err).NotTo(HaveOccurred(), "failed to query for pod")
		Expect(len(pods.Items)).To(Equal(1))

		By("verifying pod creation was observed")
		select {
		case event, _ := <-w.ResultChan():
			if event.Type != watch.Added {
				framework.Failf("Failed to observe pod creation: %v", event)
			}
		case <-time.After(framework.PodStartTimeout):
			framework.Failf("Timeout while waiting for pod creation")
		}

		// We need to wait for the pod to be running, otherwise the deletion
		// may be carried out immediately rather than gracefully.
		framework.ExpectNoError(f.WaitForPodRunning(pod.Name))

		By("ensuring pod is modified")
		// save the running pod
		pod, err = podClient.Get(pod.Name, metav1.GetOptions{})
		Expect(err).NotTo(HaveOccurred(), "failed to GET scheduled pod")

		// check the annotation is there
		if _, ok := pod.Annotations["podpreset.admission.kubernetes.io/podpreset-hello"]; !ok {
			framework.Failf("Annotation not found in pod annotations: \n%v\n", pod.Annotations)
		}

		// verify the env is the same
		if !reflect.DeepEqual(pip.Spec.Env, pod.Spec.Containers[0].Env) {
			framework.Failf("env of pod container does not match the env of the pip: expected %#v, got: %#v", pip.Spec.Env, pod.Spec.Containers[0].Env)
		}
		if !reflect.DeepEqual(pip.Spec.Env, pod.Spec.InitContainers[0].Env) {
			framework.Failf("env of pod init container does not match the env of the pip: expected %#v, got: %#v", pip.Spec.Env, pod.Spec.InitContainers[0].Env)
		}
	})

	It("should not modify the pod on conflict", func() {
		By("Creating a pod preset")

		pip := &settings.PodPreset{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "hello",
				Namespace: f.Namespace.Name,
			},
			Spec: settings.PodPresetSpec{
				Selector: metav1.LabelSelector{
					MatchExpressions: []metav1.LabelSelectorRequirement{
						{
							Key:      "security",
							Operator: metav1.LabelSelectorOpIn,
							Values:   []string{"S2"},
						},
					},
				},
				Volumes: []v1.Volume{{Name: "vol", VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}},
				VolumeMounts: []v1.VolumeMount{
					{Name: "vol", MountPath: "/foo"},
				},
				Env: []v1.EnvVar{{Name: "abc", Value: "value"}, {Name: "ABC", Value: "value"}},
			},
		}

		_, err := createPodPreset(f.ClientSet, f.Namespace.Name, pip)
		if errors.IsNotFound(err) {
			framework.Skipf("podpresets requires k8s.io/api/settings/v1alpha1 to be enabled")
		}
		Expect(err).NotTo(HaveOccurred())

		By("creating the pod")
		name := "pod-preset-pod"
		value := strconv.Itoa(time.Now().Nanosecond())
		originalPod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:      name,
				Namespace: f.Namespace.Name,
				Labels: map[string]string{
					"name":     "foo",
					"time":     value,
					"security": "S2",
				},
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:  "nginx",
						Image: imageutils.GetE2EImage(imageutils.Nginx),
						Env:   []v1.EnvVar{{Name: "abc", Value: "value2"}, {Name: "ABC", Value: "value"}},
					},
				},
				InitContainers: []v1.Container{
					{
						Name:    "init1",
						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
						Env:     []v1.EnvVar{{Name: "abc", Value: "value2"}, {Name: "ABC", Value: "value"}},
						Command: []string{"/bin/true"},
					},
				},
			},
		}

		By("setting up watch")
		selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": value}))
		options := metav1.ListOptions{LabelSelector: selector.String()}
		pods, err := podClient.List(options)
		Expect(err).NotTo(HaveOccurred(), "failed to query for pod")
		Expect(len(pods.Items)).To(Equal(0))
		options = metav1.ListOptions{
			LabelSelector:   selector.String(),
			ResourceVersion: pods.ListMeta.ResourceVersion,
		}
		w, err := podClient.Watch(options)
		Expect(err).NotTo(HaveOccurred(), "failed to set up watch")

		By("submitting the pod to kubernetes")
		podClient.Create(originalPod)

		By("verifying the pod is in kubernetes")
		selector = labels.SelectorFromSet(labels.Set(map[string]string{"time": value}))
		options = metav1.ListOptions{LabelSelector: selector.String()}
		pods, err = podClient.List(options)
		Expect(err).NotTo(HaveOccurred(), "failed to query for pod")
		Expect(len(pods.Items)).To(Equal(1))

		By("verifying pod creation was observed")
		select {
		case event, _ := <-w.ResultChan():
			if event.Type != watch.Added {
				framework.Failf("Failed to observe pod creation: %v", event)
			}
		case <-time.After(framework.PodStartTimeout):
			framework.Failf("Timeout while waiting for pod creation")
		}

		// We need to wait for the pod to be running, otherwise the deletion
		// may be carried out immediately rather than gracefully.
		framework.ExpectNoError(f.WaitForPodRunning(originalPod.Name))

		By("ensuring pod is modified")
		// save the running pod
		pod, err := podClient.Get(originalPod.Name, metav1.GetOptions{})
		Expect(err).NotTo(HaveOccurred(), "failed to GET scheduled pod")

		// check the annotation is not there
		if _, ok := pod.Annotations["podpreset.admission.kubernetes.io/podpreset-hello"]; ok {
			framework.Failf("Annotation found in pod annotations and should not be: \n%v\n", pod.Annotations)
		}

		// verify the env is the same
		if !reflect.DeepEqual(originalPod.Spec.Containers[0].Env, pod.Spec.Containers[0].Env) {
			framework.Failf("env of pod container does not match the env of the original pod: expected %#v, got: %#v", originalPod.Spec.Containers[0].Env, pod.Spec.Containers[0].Env)
		}
		if !reflect.DeepEqual(originalPod.Spec.InitContainers[0].Env, pod.Spec.InitContainers[0].Env) {
			framework.Failf("env of pod init container does not match the env of the original pod: expected %#v, got: %#v", originalPod.Spec.InitContainers[0].Env, pod.Spec.InitContainers[0].Env)
		}

	})
})

func getPodPreset(c clientset.Interface, ns, name string) (*settings.PodPreset, error) {
	return c.SettingsV1alpha1().PodPresets(ns).Get(name, metav1.GetOptions{})
}

func createPodPreset(c clientset.Interface, ns string, job *settings.PodPreset) (*settings.PodPreset, error) {
	return c.SettingsV1alpha1().PodPresets(ns).Create(job)
}

func updatePodPreset(c clientset.Interface, ns string, job *settings.PodPreset) (*settings.PodPreset, error) {
	return c.SettingsV1alpha1().PodPresets(ns).Update(job)
}

func deletePodPreset(c clientset.Interface, ns, name string) error {
	return c.SettingsV1alpha1().PodPresets(ns).Delete(name, nil)
}
<|endoftext|>


<commit_before>package eval

import (
	"reflect"
)

type envSource int
const (
	envUnknown envSource = iota
	envVar
	envFunc
	envConst
)

// A Environment used for evaluation
type Env interface {
	// Return a pointer value to the variable ident if defined in the top scope, or reflect.Value{} otherwise
	Var(ident string) reflect.Value

	// Return the const value ident if defined in the top scope, or reflect.Value{} otherwise
	Const(ident string) reflect.Value

	// Return the func value ident if defined in the top scope, or reflect.Value{} otherwise
	Func(ident string) reflect.Value

	// Return the type ident if defined in the top scope, or reflect.Value{} otherwise
	Type(ident string) reflect.Type

	// Return the environment containing vars, consts, funcs and types of pkg, or nil if not defined.
	// Unlike other lookup methods, packages exist only in the root scope.
	Pkg(pkg string) Env

	// Create a new block scope. Only the behaviour of the returned Env should change
	PushScope() Env

	// Pop the top block scope. Only the behaviour of the returned Env should change
	PopScope() Env

	// Add var ident to the top scope. The value is always a pointer value, and this same value should be
	// returned by Var(ident). It is up to the implementation how to handle duplicate identifiers.
	AddVar(ident string, v reflect.Value)

	// Add const ident to the top scope. It is up to the implementation how to handle duplicate identifiers.
	AddConst(ident string, c reflect.Value)

	// Add func ident to the top scope. It is up to the implementation how to handle duplicate identifiers.
	AddFunc(ident string, f reflect.Value)

	// Add type ident to the top scope. It is up to the implementation how to handle duplicate identifiers.
	AddType(ident string, t reflect.Type)

	// Add pkg to the root scope. It is up to the implementation how to handle duplicate identifiers.
	AddPkg(pkg string, p Env)
}

func MakeSimpleEnv() *SimpleEnv {
	return &SimpleEnv {
		Vars: map[string]reflect.Value{},
		Funcs: map[string]reflect.Value{},
		Consts: map[string]reflect.Value{},
		Types: map[string]reflect.Type{},
		Pkgs: map[string]Env{},
	}
}

type SimpleEnv struct {
	Parent *SimpleEnv
	Vars map[string]reflect.Value
	Funcs map[string]reflect.Value
	Consts map[string]reflect.Value
	Types map[string]reflect.Type
	Pkgs map[string]Env
}

func (env *SimpleEnv) Var(ident string) reflect.Value {
	return env.Vars[ident]
}

func (env *SimpleEnv) Func(ident string) reflect.Value {
	return env.Funcs[ident]
}

func (env *SimpleEnv) Const(ident string) reflect.Value {
	return env.Consts[ident]
}

func (env *SimpleEnv) Type(ident string) reflect.Type {
	return env.Types[ident]
}

func (env *SimpleEnv) Pkg(pkg string) Env {
	for env.Parent != nil {
		env = env.Parent
	}
	return env.Pkgs[pkg]
}

func (env *SimpleEnv) PushScope() Env {
	top := MakeSimpleEnv()
	top.Parent = env
	return top
}

func (env *SimpleEnv) PopScope() Env {
	if env.Parent == nil {
		return nil
	} else {
		return env.Parent
	}
}

func (env *SimpleEnv) AddVar(ident string, v reflect.Value) {
	env.Vars[ident] = v
}

func (env *SimpleEnv) AddFunc(ident string, f reflect.Value) {
	env.Funcs[ident] = f
}

func (env *SimpleEnv) AddConst(ident string, c reflect.Value) {
	env.Consts[ident] = c
}

func (env *SimpleEnv) AddType(ident string, t reflect.Type) {
	env.Types[ident] = t
}

func (env *SimpleEnv) AddPkg(pkg string, p Env) {
	for env.Parent != nil {
		env = env.Parent
	}
	env.Pkgs[pkg] = p
}
<commit_msg>Added Path field to SimpleEnv<commit_after>package eval

import (
	"reflect"
)

type envSource int
const (
	envUnknown envSource = iota
	envVar
	envFunc
	envConst
)

// A Environment used for evaluation
type Env interface {
	// Return a pointer value to the variable ident if defined in the top scope, or reflect.Value{} otherwise
	Var(ident string) reflect.Value

	// Return the const value ident if defined in the top scope, or reflect.Value{} otherwise
	Const(ident string) reflect.Value

	// Return the func value ident if defined in the top scope, or reflect.Value{} otherwise
	Func(ident string) reflect.Value

	// Return the type ident if defined in the top scope, or reflect.Value{} otherwise
	Type(ident string) reflect.Type

	// Return the environment containing vars, consts, funcs and types of pkg, or nil if not defined.
	// Unlike other lookup methods, packages exist only in the root scope.
	Pkg(pkg string) Env

	// Create a new block scope. Only the behaviour of the returned Env should change
	PushScope() Env

	// Pop the top block scope. Only the behaviour of the returned Env should change
	PopScope() Env

	// Add var ident to the top scope. The value is always a pointer value, and this same value should be
	// returned by Var(ident). It is up to the implementation how to handle duplicate identifiers.
	AddVar(ident string, v reflect.Value)

	// Add const ident to the top scope. It is up to the implementation how to handle duplicate identifiers.
	AddConst(ident string, c reflect.Value)

	// Add func ident to the top scope. It is up to the implementation how to handle duplicate identifiers.
	AddFunc(ident string, f reflect.Value)

	// Add type ident to the top scope. It is up to the implementation how to handle duplicate identifiers.
	AddType(ident string, t reflect.Type)

	// Add pkg to the root scope. It is up to the implementation how to handle duplicate identifiers.
	AddPkg(pkg string, p Env)
}

func MakeSimpleEnv() *SimpleEnv {
	return &SimpleEnv {
		Vars: map[string]reflect.Value{},
		Funcs: map[string]reflect.Value{},
		Consts: map[string]reflect.Value{},
		Types: map[string]reflect.Type{},
		Pkgs: map[string]Env{},
	}
}

type SimpleEnv struct {
	// path relative to GOROOT or GOPATH. e.g. github.com/0xfaded/eval
	Path string
	Parent *SimpleEnv
	Vars map[string]reflect.Value
	Funcs map[string]reflect.Value
	Consts map[string]reflect.Value
	Types map[string]reflect.Type
	Pkgs map[string]Env
}

func (env *SimpleEnv) Var(ident string) reflect.Value {
	return env.Vars[ident]
}

func (env *SimpleEnv) Func(ident string) reflect.Value {
	return env.Funcs[ident]
}

func (env *SimpleEnv) Const(ident string) reflect.Value {
	return env.Consts[ident]
}

func (env *SimpleEnv) Type(ident string) reflect.Type {
	return env.Types[ident]
}

func (env *SimpleEnv) Pkg(pkg string) Env {
	for env.Parent != nil {
		env = env.Parent
	}
	return env.Pkgs[pkg]
}

func (env *SimpleEnv) PushScope() Env {
	top := MakeSimpleEnv()
	top.Parent = env
	return top
}

func (env *SimpleEnv) PopScope() Env {
	if env.Parent == nil {
		return nil
	} else {
		return env.Parent
	}
}

func (env *SimpleEnv) AddVar(ident string, v reflect.Value) {
	env.Vars[ident] = v
}

func (env *SimpleEnv) AddFunc(ident string, f reflect.Value) {
	env.Funcs[ident] = f
}

func (env *SimpleEnv) AddConst(ident string, c reflect.Value) {
	env.Consts[ident] = c
}

func (env *SimpleEnv) AddType(ident string, t reflect.Type) {
	env.Types[ident] = t
}

func (env *SimpleEnv) AddPkg(pkg string, p Env) {
	for env.Parent != nil {
		env = env.Parent
	}
	env.Pkgs[pkg] = p
}
<|endoftext|>


<commit_before>package middleware

import (
	"github.com/afex/hystrix-go/hystrix"
	"github.com/gin-gonic/gin"
	"strings"

	"github.com/eirka/eirka-libs/redis"
)

var (
	RedisKeyIndex = make(map[string]RedisKey)
	RedisKeys     = []RedisKey{
		{base: "index", fieldcount: 1, hash: true, expire: false},
		{base: "thread", fieldcount: 2, hash: true, expire: false},
		{base: "tag", fieldcount: 2, hash: true, expire: true},
		{base: "image", fieldcount: 1, hash: true, expire: false},
		{base: "post", fieldcount: 2, hash: true, expire: false},
		{base: "tags", fieldcount: 1, hash: true, expire: false},
		{base: "directory", fieldcount: 1, hash: true, expire: false},
		{base: "new", fieldcount: 1, hash: false, expire: true},
		{base: "popular", fieldcount: 1, hash: false, expire: true},
		{base: "favorited", fieldcount: 1, hash: false, expire: true},
		{base: "tagtypes", fieldcount: 0, hash: false, expire: false},
		{base: "imageboards", fieldcount: 0, hash: false, expire: true},
	}
)

func init() {
	// key index map
	for _, key := range RedisKeys {
		RedisKeyIndex[key.base] = key
	}
}

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {
		var result []byte
		var err error

		// bool for analytics middleware
		c.Set("cached", false)

		// break cache if there is a query
		if c.Request.URL.RawQuery != "" {
			c.Next()
			return
		}

		// redis circuitbreaker
		hystrix.Do("cache", func() (err error) {

			// Trim leading / from path and split
			params := strings.Split(strings.Trim(c.Request.URL.Path, "/"), "/")

			// get the keyname
			key, ok := RedisKeyIndex[params[0]]
			if !ok {
				c.Next()
				return
			}

			// set the key minus the base
			key.SetKey(params[1:]...)

			result, err = key.Get()
			if err == redis.ErrCacheMiss {
				// go to the controller
				c.Next()

				// Check if there was an error from the controller
				_, controllerError := c.Get("controllerError")
				if controllerError {
					c.Abort()
					return
				}

				err = key.Set(c.MustGet("data").([]byte))
				if err != nil {
					c.Error(err)
					c.Abort()
					return
				}

			} else if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

			return

		}, func(err error) error {
			c.Next()
			return nil
		})

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()
		return
	}

}

type RedisKey struct {
	base       string
	fieldcount int
	hash       bool
	expire     bool
	key        string
	hashid     string
}

func (r *RedisKey) SetKey(ids ...string) {

	if r.fieldcount == 0 {
		r.key = r.base
		return
	}

	// create our key
	r.key = strings.Join([]string{r.base, strings.Join(ids[:r.fieldcount], ":")}, ":")

	// get our hash id
	if r.hash {
		r.hashid = strings.Join(ids[r.fieldcount:], "")
	}

	return
}

func (r *RedisKey) Get() (result []byte, err error) {

	if r.hash {
		return redis.RedisCache.HGet(r.key, r.hashid)
	} else {
		return redis.RedisCache.Get(r.key)
	}

	return
}

func (r *RedisKey) Set(data []byte) (err error) {

	if r.hash {
		err = redis.RedisCache.HMSet(r.key, r.hashid, data)
	} else {
		err = redis.RedisCache.Set(r.key, data)
	}

	if r.expire {
		return redis.RedisCache.Expire(r.key, 600)
	}

	return
}
<commit_msg>try hystrix on cache middleware<commit_after>package middleware

import (
	"github.com/afex/hystrix-go/hystrix"
	"github.com/gin-gonic/gin"
	"strings"

	"github.com/eirka/eirka-libs/redis"
)

var (
	RedisKeyIndex = make(map[string]RedisKey)
	RedisKeys     = []RedisKey{
		{base: "index", fieldcount: 1, hash: true, expire: false},
		{base: "thread", fieldcount: 2, hash: true, expire: false},
		{base: "tag", fieldcount: 2, hash: true, expire: true},
		{base: "image", fieldcount: 1, hash: true, expire: false},
		{base: "post", fieldcount: 2, hash: true, expire: false},
		{base: "tags", fieldcount: 1, hash: true, expire: false},
		{base: "directory", fieldcount: 1, hash: true, expire: false},
		{base: "new", fieldcount: 1, hash: false, expire: true},
		{base: "popular", fieldcount: 1, hash: false, expire: true},
		{base: "favorited", fieldcount: 1, hash: false, expire: true},
		{base: "tagtypes", fieldcount: 0, hash: false, expire: false},
		{base: "imageboards", fieldcount: 0, hash: false, expire: true},
	}
)

func init() {
	// key index map
	for _, key := range RedisKeys {
		RedisKeyIndex[key.base] = key
	}
}

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {
		var result []byte

		// bool for analytics middleware
		c.Set("cached", false)

		// break cache if there is a query
		if c.Request.URL.RawQuery != "" {
			c.Next()
			return
		}

		// redis circuitbreaker
		hystrix.Do("cache", func() (err error) {

			// Trim leading / from path and split
			params := strings.Split(strings.Trim(c.Request.URL.Path, "/"), "/")

			// get the keyname
			key, ok := RedisKeyIndex[params[0]]
			if !ok {
				c.Next()
				return
			}

			// set the key minus the base
			key.SetKey(params[1:]...)

			result, err = key.Get()
			if err == redis.ErrCacheMiss {
				// go to the controller
				c.Next()

				// Check if there was an error from the controller
				_, controllerError := c.Get("controllerError")
				if controllerError {
					c.Abort()
					return
				}

				err = key.Set(c.MustGet("data").([]byte))
				if err != nil {
					c.Error(err)
					c.Abort()
					return
				}

			} else if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

			return

		}, func(err error) error {
			c.Next()
			return nil
		})

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()
		return
	}

}

type RedisKey struct {
	base       string
	fieldcount int
	hash       bool
	expire     bool
	key        string
	hashid     string
}

func (r *RedisKey) SetKey(ids ...string) {

	if r.fieldcount == 0 {
		r.key = r.base
		return
	}

	// create our key
	r.key = strings.Join([]string{r.base, strings.Join(ids[:r.fieldcount], ":")}, ":")

	// get our hash id
	if r.hash {
		r.hashid = strings.Join(ids[r.fieldcount:], "")
	}

	return
}

func (r *RedisKey) Get() (result []byte, err error) {

	if r.hash {
		return redis.RedisCache.HGet(r.key, r.hashid)
	} else {
		return redis.RedisCache.Get(r.key)
	}

	return
}

func (r *RedisKey) Set(data []byte) (err error) {

	if r.hash {
		err = redis.RedisCache.HMSet(r.key, r.hashid, data)
	} else {
		err = redis.RedisCache.Set(r.key, data)
	}

	if r.expire {
		return redis.RedisCache.Expire(r.key, 600)
	}

	return
}
<|endoftext|>


<commit_before>package lily

import "log"

func ErrPanic(err error) {
	if err != nil {
		log.Panicln(err)
	}
}

func ErrFatal(err error, msg string) {
	if err != nil {
		log.Fatalln(err)
	}
}
<commit_msg>add detail to errFatal<commit_after>package lily

import (
	"log"
	"runtime"
)

func ErrPanic(err error) {
	if err != nil {
		log.Panicln(err)
	}
}

func ErrFatal(err error, msg ...string) {
	if err != nil {
		_, file, line, _ := runtime.Caller(1)
		log.Fatalf("fatal (%s:%d):\n\t%s\n\t%s\n", file, line, msg, err.Error())
	}
}
<|endoftext|>


<commit_before>package operators

import (
	"context"
	"fmt"
	"regexp"
	"strings"

	g "github.com/onsi/ginkgo"
	"github.com/openshift/origin/pkg/test/ginkgo/result"
	exutil "github.com/openshift/origin/test/extended/util"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kube-openapi/pkg/util/sets"
	e2e "k8s.io/kubernetes/test/e2e/framework"
)

var _ = g.Describe("[sig-arch] Managed cluster", func() {
	oc := exutil.NewCLIWithoutNamespace("operator-resources")

	// Pods that are part of the control plane should set both cpu and memory requests, but require an exception
	// to set limits on memory (CPU limits are generally not allowed). This enforces the rules described in
	// https://github.com/openshift/enhancements/blob/master/CONVENTIONS.md#resources-and-limits.
	//
	// This test enforces all pods in the openshift-*, kube-*, and default namespace have requests set for both
	// CPU and memory, and no limits set. Known bugs will transform this to a flake. Otherwise the test will fail.
	//
	// Release architects can justify an exception with text but must ensure CONVENTIONS.md is updated to document
	// why the exception is granted.
	g.It("should set requests but not limits", func() {
		pods, err := oc.KubeFramework().ClientSet.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
		if err != nil {
			e2e.Failf("unable to list pods: %v", err)
		}

		// pods that have a bug opened, every entry here must have a bug associated
		knownBrokenPods := map[string]string{
			//"<apiVersion>/<kind>/<namespace>/<name>/(initContainer|container)/<container_name>/<violation_type>": "<url to bug>",

			"apps/v1/Deployment/openshift-cluster-csi-drivers/ovirt-csi-driver-controller/container/csi-provisioner/request[cpu]":    "https://bugzilla.redhat.com/show_bug.cgi?id=1940876",
			"apps/v1/Deployment/openshift-cluster-csi-drivers/ovirt-csi-driver-controller/container/csi-provisioner/request[memory]": "https://bugzilla.redhat.com/show_bug.cgi?id=1940876",
		}

		// pods with an exception granted, the value should be the justification and the approver (a release architect)
		exceptionGranted := map[string]string{
			//"<apiVersion>/<kind>/<namespace>/<name>/(initContainer|container)/<container_name>/<violation_type>": "<github handle of approver>: <brief description of the reason for the exception>",

			// CPU limits on these containers may be inappropriate in the future
			"v1/Pod/openshift-etcd/installer-<revision>-<node>/container/installer/limit[cpu]":                          "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-etcd/installer-<revision>-<node>/container/installer/limit[memory]":                       "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-etcd/revision-pruner-<revision>-<node>/container/pruner/limit[cpu]":                       "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-etcd/revision-pruner-<revision>-<node>/container/pruner/limit[memory]":                    "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-apiserver/installer-<revision>-<node>/container/installer/limit[cpu]":                "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-apiserver/installer-<revision>-<node>/container/installer/limit[memory]":             "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-apiserver/revision-pruner-<revision>-<node>/container/pruner/limit[cpu]":             "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-apiserver/revision-pruner-<revision>-<node>/container/pruner/limit[memory]":          "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-controller-manager/installer-<revision>-<node>/container/installer/limit[cpu]":       "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-controller-manager/installer-<revision>-<node>/container/installer/limit[memory]":    "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-controller-manager/revision-pruner-<revision>-<node>/container/pruner/limit[cpu]":    "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-controller-manager/revision-pruner-<revision>-<node>/container/pruner/limit[memory]": "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-scheduler/installer-<revision>-<node>/container/installer/limit[cpu]":                "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-scheduler/installer-<revision>-<node>/container/installer/limit[memory]":             "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-scheduler/revision-pruner-<revision>-<node>/container/pruner/limit[cpu]":             "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-scheduler/revision-pruner-<revision>-<node>/container/pruner/limit[memory]":          "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",

			"apps/v1/Deployment/openshift-monitoring/thanos-querier/container/thanos-query/limit[memory]": "smarterclayton: granted a temporary exception (reasses in 4.10) until Thanos can properly control resource usage from arbitrary queries",
		}

		reNormalizeRunOnceNames := regexp.MustCompile(`^(installer-|revision-pruner-)[\d]+-`)
		reNormalizeRetryNames := regexp.MustCompile(`-retry-[\d]+-`)

		waitingForFix := sets.NewString()
		notAllowed := sets.NewString()
		possibleFuture := sets.NewString()
		for _, pod := range pods.Items {
			// Only pods in the openshift-*, kube-*, and default namespaces are considered
			if !strings.HasPrefix(pod.Namespace, "openshift-") && !strings.HasPrefix(pod.Namespace, "kube-") && pod.Namespace != "default" {
				continue
			}
			// Must-gather runs are excluded from this rule
			if strings.HasPrefix(pod.Namespace, "openshift-must-gather") {
				continue
			}
			// var controlPlaneTarget bool
			// selector := labels.SelectorFromSet(pod.Spec.NodeSelector)
			// if !selector.Empty() && selector.Matches(labels.Set(map[string]string{"node-role.kubernetes.io/master": ""})) {
			// 	controlPlaneTarget = true
			// }

			// Find a unique string that identifies who creates the pod, or the pod itself
			var controller string
			for _, ref := range pod.OwnerReferences {
				if ref.Controller == nil || !*ref.Controller {
					continue
				}
				// simple hack to make the rules cluster better, if we get new hierarchies just add more checks here
				switch ref.Kind {
				case "ReplicaSet":
					if i := strings.LastIndex(ref.Name, "-"); i != -1 {
						name := ref.Name[0:i]
						if deploy, err := oc.KubeFramework().ClientSet.AppsV1().Deployments(pod.Namespace).Get(context.Background(), name, metav1.GetOptions{}); err == nil {
							ref.Name = deploy.Name
							ref.Kind = "Deployment"
							ref.APIVersion = "apps/v1"
						}
					}
				case "Job":
					if pod.Namespace == "openshift-marketplace" {
						ref.Name = "<batch_job>"
					}
				case "Node":
					continue
				}
				controller = fmt.Sprintf("%s/%s/%s/%s", ref.APIVersion, ref.Kind, pod.Namespace, ref.Name)
				break
			}
			if len(controller) == 0 {
				if len(pod.GenerateName) > 0 {
					name := strings.ReplaceAll(pod.GenerateName, pod.Spec.NodeName, "<node>")
					if pod.Spec.RestartPolicy != v1.RestartPolicyAlways {
						name = reNormalizeRunOnceNames.ReplaceAllString(name, "$1<revision>")
					}
					controller = fmt.Sprintf("v1/Pod/%s/%s", pod.Namespace, name)
				} else {
					name := strings.ReplaceAll(pod.Name, pod.Spec.NodeName, "<node>")
					if pod.Spec.RestartPolicy != v1.RestartPolicyAlways {
						name = reNormalizeRunOnceNames.ReplaceAllString(name, "$1<revision>-")
					}
					controller = fmt.Sprintf("v1/Pod/%s/%s", pod.Namespace, name)
				}
			}

			// Remove -retry-#- for, e.g. openshift-etcd/installer-<revision>-retry-1-<node>
			controller = reNormalizeRetryNames.ReplaceAllString(controller, "-")

			// These rules apply to both init and regular containers
			for containerType, containers := range map[string][]v1.Container{
				"initContainer": pod.Spec.InitContainers,
				"container":     pod.Spec.Containers,
			} {
				for _, c := range containers {
					key := fmt.Sprintf("%s/%s/%s", controller, containerType, c.Name)

					// Pods may not set limits
					if len(c.Resources.Limits) > 0 {
						for resource, v := range c.Resources.Limits {
							rule := fmt.Sprintf("%s/%s[%s]", key, "limit", resource)
							if len(exceptionGranted[rule]) == 0 {
								violation := fmt.Sprintf("%s defines a limit on %s of %s which is not allowed", key, resource, v.String())
								if bug, ok := knownBrokenPods[rule]; ok {
									waitingForFix.Insert(fmt.Sprintf("%s (bug %s)", violation, bug))
								} else {
									notAllowed.Insert(fmt.Sprintf("%s (rule: %q)", violation, rule))
								}
							}
						}
					}

					// Pods must have at least CPU and memory requests
					for _, resource := range []string{"cpu", "memory"} {
						v := c.Resources.Requests[v1.ResourceName(resource)]
						if !v.IsZero() {
							continue
						}
						rule := fmt.Sprintf("%s/%s[%s]", key, "request", resource)
						violation := fmt.Sprintf("%s does not have a %s request", key, resource)
						if len(exceptionGranted[rule]) == 0 {
							if bug, ok := knownBrokenPods[rule]; ok {
								waitingForFix.Insert(fmt.Sprintf("%s (bug %s)", violation, bug))
							} else {
								if containerType == "initContainer" {
									possibleFuture.Insert(fmt.Sprintf("%s (candidate rule: %q)", violation, rule))
								} else {
									notAllowed.Insert(fmt.Sprintf("%s (rule: %q)", violation, rule))
								}
							}
						}
					}
				}
			}
		}

		// Some things we may start checking in the future
		if len(possibleFuture) > 0 {
			e2e.Logf("Pods in platform namespaces had resource request/limit that we may enforce in the future:\n\n%s", strings.Join(possibleFuture.List(), "\n"))
		}

		// Users are not allowed to add new violations
		if len(notAllowed) > 0 {
			e2e.Failf("Pods in platform namespaces are not following resource request/limit rules or do not have an exception granted:\n  %s", strings.Join(notAllowed.List(), "\n  "))
		}

		// All known bugs are listed as flakes so we can see them as dashboards
		if len(waitingForFix) > 0 {
			result.Flakef("Pods in platform namespaces had known broken resource request/limit that have not been resolved:\n\n%s", strings.Join(waitingForFix.List(), "\n"))
		}
	})
})
<commit_msg>Fixup request/limits test to accomodate Pods annotated for workload partitioning<commit_after>package operators

import (
	"context"
	"fmt"
	"regexp"
	"strings"

	g "github.com/onsi/ginkgo"
	"github.com/openshift/origin/pkg/test/ginkgo/result"
	exutil "github.com/openshift/origin/test/extended/util"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kube-openapi/pkg/util/sets"
	e2e "k8s.io/kubernetes/test/e2e/framework"
)

var _ = g.Describe("[sig-arch] Managed cluster", func() {
	oc := exutil.NewCLIWithoutNamespace("operator-resources")

	// Pods that are part of the control plane should set both cpu and memory requests, but require an exception
	// to set limits on memory (CPU limits are generally not allowed). This enforces the rules described in
	// https://github.com/openshift/enhancements/blob/master/CONVENTIONS.md#resources-and-limits.
	//
	// This test enforces all pods in the openshift-*, kube-*, and default namespace have requests set for both
	// CPU and memory, and no limits set. Known bugs will transform this to a flake. Otherwise the test will fail.
	//
	// Release architects can justify an exception with text but must ensure CONVENTIONS.md is updated to document
	// why the exception is granted.
	g.It("should set requests but not limits", func() {
		pods, err := oc.KubeFramework().ClientSet.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
		if err != nil {
			e2e.Failf("unable to list pods: %v", err)
		}

		// pods that have a bug opened, every entry here must have a bug associated
		knownBrokenPods := map[string]string{
			//"<apiVersion>/<kind>/<namespace>/<name>/(initContainer|container)/<container_name>/<violation_type>": "<url to bug>",

			"apps/v1/Deployment/openshift-cluster-csi-drivers/ovirt-csi-driver-controller/container/csi-provisioner/request[cpu]":    "https://bugzilla.redhat.com/show_bug.cgi?id=1940876",
			"apps/v1/Deployment/openshift-cluster-csi-drivers/ovirt-csi-driver-controller/container/csi-provisioner/request[memory]": "https://bugzilla.redhat.com/show_bug.cgi?id=1940876",
		}

		// pods with an exception granted, the value should be the justification and the approver (a release architect)
		exceptionGranted := map[string]string{
			//"<apiVersion>/<kind>/<namespace>/<name>/(initContainer|container)/<container_name>/<violation_type>": "<github handle of approver>: <brief description of the reason for the exception>",

			// CPU limits on these containers may be inappropriate in the future
			"v1/Pod/openshift-etcd/installer-<revision>-<node>/container/installer/limit[cpu]":                          "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-etcd/installer-<revision>-<node>/container/installer/limit[memory]":                       "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-etcd/revision-pruner-<revision>-<node>/container/pruner/limit[cpu]":                       "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-etcd/revision-pruner-<revision>-<node>/container/pruner/limit[memory]":                    "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-apiserver/installer-<revision>-<node>/container/installer/limit[cpu]":                "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-apiserver/installer-<revision>-<node>/container/installer/limit[memory]":             "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-apiserver/revision-pruner-<revision>-<node>/container/pruner/limit[cpu]":             "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-apiserver/revision-pruner-<revision>-<node>/container/pruner/limit[memory]":          "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-controller-manager/installer-<revision>-<node>/container/installer/limit[cpu]":       "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-controller-manager/installer-<revision>-<node>/container/installer/limit[memory]":    "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-controller-manager/revision-pruner-<revision>-<node>/container/pruner/limit[cpu]":    "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-controller-manager/revision-pruner-<revision>-<node>/container/pruner/limit[memory]": "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-scheduler/installer-<revision>-<node>/container/installer/limit[cpu]":                "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-scheduler/installer-<revision>-<node>/container/installer/limit[memory]":             "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-scheduler/revision-pruner-<revision>-<node>/container/pruner/limit[cpu]":             "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",
			"v1/Pod/openshift-kube-scheduler/revision-pruner-<revision>-<node>/container/pruner/limit[memory]":          "smarterclayton: run-once pod with very well-known resource usage, does not vary based on workload or cluster size",

			"apps/v1/Deployment/openshift-monitoring/thanos-querier/container/thanos-query/limit[memory]": "smarterclayton: granted a temporary exception (reasses in 4.10) until Thanos can properly control resource usage from arbitrary queries",
		}

		reNormalizeRunOnceNames := regexp.MustCompile(`^(installer-|revision-pruner-)[\d]+-`)
		reNormalizeRetryNames := regexp.MustCompile(`-retry-[\d]+-`)

		waitingForFix := sets.NewString()
		notAllowed := sets.NewString()
		possibleFuture := sets.NewString()
		for _, pod := range pods.Items {
			// Only pods in the openshift-*, kube-*, and default namespaces are considered
			if !strings.HasPrefix(pod.Namespace, "openshift-") && !strings.HasPrefix(pod.Namespace, "kube-") && pod.Namespace != "default" {
				continue
			}
			// Must-gather runs are excluded from this rule
			if strings.HasPrefix(pod.Namespace, "openshift-must-gather") {
				continue
			}
			// var controlPlaneTarget bool
			// selector := labels.SelectorFromSet(pod.Spec.NodeSelector)
			// if !selector.Empty() && selector.Matches(labels.Set(map[string]string{"node-role.kubernetes.io/master": ""})) {
			// 	controlPlaneTarget = true
			// }

			// Find a unique string that identifies who creates the pod, or the pod itself
			var controller string
			for _, ref := range pod.OwnerReferences {
				if ref.Controller == nil || !*ref.Controller {
					continue
				}
				// simple hack to make the rules cluster better, if we get new hierarchies just add more checks here
				switch ref.Kind {
				case "ReplicaSet":
					if i := strings.LastIndex(ref.Name, "-"); i != -1 {
						name := ref.Name[0:i]
						if deploy, err := oc.KubeFramework().ClientSet.AppsV1().Deployments(pod.Namespace).Get(context.Background(), name, metav1.GetOptions{}); err == nil {
							ref.Name = deploy.Name
							ref.Kind = "Deployment"
							ref.APIVersion = "apps/v1"
						}
					}
				case "Job":
					if pod.Namespace == "openshift-marketplace" {
						ref.Name = "<batch_job>"
					}
				case "Node":
					continue
				}
				controller = fmt.Sprintf("%s/%s/%s/%s", ref.APIVersion, ref.Kind, pod.Namespace, ref.Name)
				break
			}
			if len(controller) == 0 {
				if len(pod.GenerateName) > 0 {
					name := strings.ReplaceAll(pod.GenerateName, pod.Spec.NodeName, "<node>")
					if pod.Spec.RestartPolicy != v1.RestartPolicyAlways {
						name = reNormalizeRunOnceNames.ReplaceAllString(name, "$1<revision>")
					}
					controller = fmt.Sprintf("v1/Pod/%s/%s", pod.Namespace, name)
				} else {
					name := strings.ReplaceAll(pod.Name, pod.Spec.NodeName, "<node>")
					if pod.Spec.RestartPolicy != v1.RestartPolicyAlways {
						name = reNormalizeRunOnceNames.ReplaceAllString(name, "$1<revision>-")
					}
					controller = fmt.Sprintf("v1/Pod/%s/%s", pod.Namespace, name)
				}
			}

			// Remove -retry-#- for, e.g. openshift-etcd/installer-<revision>-retry-1-<node>
			controller = reNormalizeRetryNames.ReplaceAllString(controller, "-")

			// These rules apply to both init and regular containers
			for containerType, containers := range map[string][]v1.Container{
				"initContainer": pod.Spec.InitContainers,
				"container":     pod.Spec.Containers,
			} {
				for _, c := range containers {
					key := fmt.Sprintf("%s/%s/%s", controller, containerType, c.Name)

					// Pods may not set limits
					if len(c.Resources.Limits) > 0 {
						for resource, v := range c.Resources.Limits {
							if resource == "management.workload.openshift.io/cores" { // limits are allowed on management.workload.openshift.io/cores resources
								continue
							}
							rule := fmt.Sprintf("%s/%s[%s]", key, "limit", resource)
							if len(exceptionGranted[rule]) == 0 {
								violation := fmt.Sprintf("%s defines a limit on %s of %s which is not allowed", key, resource, v.String())
								if bug, ok := knownBrokenPods[rule]; ok {
									waitingForFix.Insert(fmt.Sprintf("%s (bug %s)", violation, bug))
								} else {
									notAllowed.Insert(fmt.Sprintf("%s (rule: %q)", violation, rule))
								}
							}
						}
					}

					annotationForPreferringManagementCores := "target.workload.openshift.io/management"
					wpPodAnnotation := strings.Replace(pod.Annotations[annotationForPreferringManagementCores], " ", "", -1) // some pods have a space after the : in their annotation definition

					// Pods must have at least CPU and memory requests
					for _, resource := range []string{"cpu", "memory"} {
						if len(wpPodAnnotation) > 0 && resource == "cpu" { // don't check for CPU request if the pod has a WP annotation
							continue
						}
						v := c.Resources.Requests[v1.ResourceName(resource)]
						if !v.IsZero() {
							continue
						}
						rule := fmt.Sprintf("%s/%s[%s]", key, "request", resource)
						violation := fmt.Sprintf("%s does not have a %s request", key, resource)
						if len(exceptionGranted[rule]) == 0 {
							if bug, ok := knownBrokenPods[rule]; ok {
								waitingForFix.Insert(fmt.Sprintf("%s (bug %s)", violation, bug))
							} else {
								if containerType == "initContainer" {
									possibleFuture.Insert(fmt.Sprintf("%s (candidate rule: %q)", violation, rule))
								} else {
									notAllowed.Insert(fmt.Sprintf("%s (rule: %q)", violation, rule))
								}
							}
						}
					}
				}
			}
		}

		// Some things we may start checking in the future
		if len(possibleFuture) > 0 {
			e2e.Logf("Pods in platform namespaces had resource request/limit that we may enforce in the future:\n\n%s", strings.Join(possibleFuture.List(), "\n"))
		}

		// Users are not allowed to add new violations
		if len(notAllowed) > 0 {
			e2e.Failf("Pods in platform namespaces are not following resource request/limit rules or do not have an exception granted:\n  %s", strings.Join(notAllowed.List(), "\n  "))
		}

		// All known bugs are listed as flakes so we can see them as dashboards
		if len(waitingForFix) > 0 {
			result.Flakef("Pods in platform namespaces had known broken resource request/limit that have not been resolved:\n\n%s", strings.Join(waitingForFix.List(), "\n"))
		}
	})
})
<|endoftext|>


<commit_before>/*
Copyright 2018 Google, Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package differs

import (
	"archive/tar"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util"
	"github.com/GoogleContainerTools/container-diff/util"
	dockertar "github.com/containers/image/docker/tarfile"
	godocker "github.com/fsouza/go-dockerclient"
	"github.com/nightlyone/lockfile"
	"github.com/sirupsen/logrus"
)

// daemonMutex is required to protect against other go-routines, as
// nightlyone/lockfile implements a recursive lock, which doesn't protect
// against other go-routines that have the same PID.  Note that the mutex
// *must* always be locked prior to the lockfile, and unlocked after.
var daemonMutex sync.Mutex

type RPMAnalyzer struct {
}

// Name returns the name of the analyzer.
func (a RPMAnalyzer) Name() string {
	return "RPMAnalyzer"
}

// Diff compares the installed rpm packages of image1 and image2.
func (a RPMAnalyzer) Diff(image1, image2 pkgutil.Image) (util.Result, error) {
	diff, err := singleVersionDiff(image1, image2, a)
	return diff, err
}

// Analyze collects information of the installed rpm packages on image.
func (a RPMAnalyzer) Analyze(image pkgutil.Image) (util.Result, error) {
	analysis, err := singleVersionAnalysis(image, a)
	return analysis, err
}

// getPackages returns a map of installed rpm package on image.
func (a RPMAnalyzer) getPackages(image pkgutil.Image) (map[string]util.PackageInfo, error) {
	path := image.FSPath
	packages := make(map[string]util.PackageInfo)
	if _, err := os.Stat(path); err != nil {
		// invalid image directory path
		return packages, err
	}

	// try to find the rpm binary in bin/ or usr/bin/
	rpmBinary := filepath.Join(path, "bin/rpm")
	if _, err := os.Stat(rpmBinary); err != nil {
		rpmBinary = filepath.Join(path, "usr/bin/rpm")
		if _, err = os.Stat(rpmBinary); err != nil {
			logrus.Errorf("Could not detect RPM binary in unpacked image %s", image.Source)
			return packages, nil
		}
	}

	return rpmDataFromContainer(image)
}

// rpmDataFromContainer runs image in a container, queries the data of
// installed rpm packages and returns a map of packages.
func rpmDataFromContainer(image pkgutil.Image) (map[string]util.PackageInfo, error) {
	packages := make(map[string]util.PackageInfo)

	client, err := godocker.NewClientFromEnv()
	if err != nil {
		return packages, err
	}

	// if image is a tar archive, we need to load it
	imageName := image.Source
	if image.IsTar() {
		// NOTE: all work below is only required for multilayer tar
		// archives generated by `docker save`.  Flat file-system
		// archives generated by `docker create` can just be imported
		// without potential name conflicts.  However, those are not
		// supported yet.
		if err := lock(); err != nil {
			return packages, err
		}

		archive, err := generateNewArchive(imageName)
		if err != nil {
			logrus.Errorf(err.Error())
		}
		defer os.Remove(archive)

		imageName, err = loadArchiveToDaemon(archive)
		if err != nil {
			return packages, fmt.Errorf("Error loading archive: %s", err)
		}
		unlock()

		defer client.RemoveImage(imageName)
		defer logrus.Infof("Removing image %s", imageName)
	} else if image.IsCloud() {
		// if it's a remote image we pull it unconditionally and avoid
		// name/tag conflicts by pulling it via its sha256 digest
		var buf bytes.Buffer
		digest, err := image.GetRemoteDigest()
		if err != nil {
			return packages, err
		}
		imageName = image.GetName() + "@" + digest
		logrus.Infof("Pulling remote image %s", imageName)
		pullOpts := godocker.PullImageOptions{
			Repository:   imageName,
			OutputStream: &buf,
		}
		if err := client.PullImage(pullOpts, godocker.AuthConfiguration{}); err != nil {
			return packages, fmt.Errorf("Error pulling remote image %s: %s", imageName, err)
		}
		// log PullImage() output in debug mode
		if logrus.GetLevel() == logrus.DebugLevel {
			// log each line separately for consistency
			for _, out := range strings.Split(buf.String(), "\n") {
				if len(out) > 0 {
					logrus.Debug(out)
				}
			}
		}
		defer client.RemoveImage(imageName)
	}

	contConf := godocker.Config{
		Cmd:   []string{"rpm", "--nodigest", "--nosignature", "-qa", "--qf", "%{NAME}\t%{VERSION}\t%{SIZE}\n"},
		Image: imageName,
	}

	hostConf := godocker.HostConfig{
		AutoRemove: true,
	}

	contOpts := godocker.CreateContainerOptions{Config: &contConf}
	container, err := client.CreateContainer(contOpts)
	if err != nil {
		return packages, err
	}
	logrus.Infof("Created container %s", container.ID)

	removeOpts := godocker.RemoveContainerOptions{
		ID: container.ID,
	}
	defer client.RemoveContainer(removeOpts)

	if err := client.StartContainer(container.ID, &hostConf); err != nil {
		return packages, err
	}

	exitCode, err := client.WaitContainer(container.ID)
	if err != nil {
		return packages, err
	}

	outBuf := new(bytes.Buffer)
	errBuf := new(bytes.Buffer)
	logOpts := godocker.LogsOptions{
		Context:      context.Background(),
		Container:    container.ID,
		Stdout:       true,
		Stderr:       true,
		OutputStream: outBuf,
		ErrorStream:  errBuf,
	}

	if err := client.Logs(logOpts); err != nil {
		return packages, err
	}

	if exitCode != 0 {
		return packages, fmt.Errorf("non-zero exit code %d: %s", exitCode, errBuf.String())
	}

	output := strings.Split(outBuf.String(), "\n")
	return parsePackageData(output)
}

// parsePackageData parses the package data of each line in rpmOutput and
// returns a map of packages.
func parsePackageData(rpmOutput []string) (map[string]util.PackageInfo, error) {
	packages := make(map[string]util.PackageInfo)

	for _, output := range rpmOutput {
		spl := strings.Split(output, "\t")
		if len(spl) != 3 {
			// ignore the empty (last) line
			if output != "" {
				logrus.Errorf("unexpected rpm-query output: '%s'", output)
			}
			continue
		}
		pkg := util.PackageInfo{}

		var err error
		pkg.Size, err = strconv.ParseInt(spl[2], 10, 64)
		if err != nil {
			return packages, fmt.Errorf("error converting package size: %s", spl[2])
		}

		pkg.Version = spl[1]
		packages[spl[0]] = pkg
	}

	return packages, nil
}

// loadArchiveToDaemon loads the image specified by archive to the docker
// Daemon.
func loadArchiveToDaemon(archive string) (string, error) {
	client, err := godocker.NewClientFromEnv()
	if err != nil {
		return "", err
	}

	inBuf, err := os.Open(archive)
	if err != nil {
		return "", err
	}

	outBuf := new(bytes.Buffer)
	loadOpts := godocker.LoadImageOptions{
		InputStream:  inBuf,
		OutputStream: outBuf,
		Context:      context.Background(),
	}
	if err := client.LoadImage(loadOpts); err != nil {
		return "", err
	}

	// output has the form "Loaded image: repository:tag"
	output := outBuf.String()
	spl := strings.Split(output, ": ")
	if len(spl) != 2 {
		return "", fmt.Errorf("unexpected docker load output: %s", output)
	}
	image := strings.TrimSpace(spl[1])
	logrus.Infof("Loaded %s as image %s", archive, image)

	return image, nil
}

// updateRepoTagsFromManifest updates the RepoTags in the manifest.json with a
// unique tag in the `containerdiff` image namespace.  There is a maximum of 10
// tries to generate a unique tag.
func updateRepoTagsFromManifest(r io.Reader) ([]byte, error) {
	f, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}

	var manifestItems []dockertar.ManifestItem
	if err := json.Unmarshal(f, &manifestItems); err != nil {
		return nil, fmt.Errorf("error unmarshalling manifest: %s", err)
	}

	// error out in case the tar ships multiple images
	if len(manifestItems) != 1 {
		return nil, fmt.Errorf("tar archive ships %d images, but only 1 is supported", len(manifestItems))
	}

	client, err := godocker.NewClientFromEnv()
	if err != nil {
		return nil, err
	}

	tag := ""
	rand.Seed(time.Now().UnixNano())
	for i := 0; i < 10; i++ {
		tag = "containerdiff:" + strconv.FormatUint(rand.Uint64(), 10)
		logrus.Debugf("generated new random tag: %s", tag)
		if _, err := client.InspectImage(tag); err != nil {
			break
		}
		tag = ""
	}
	if len(tag) == 0 {
		return nil, fmt.Errorf("failed to generate unique tag: too many tries")
	}

	// Overwrite the `RepoTags` member with a unique tag, which determines
	// the image's tag after being loaded by the Docker daemon.
	// Unfortunately, `docker load` does not generate a new tag in case of
	// a conflict, but it will rename the already present image.  We avoid
	// such conflicts by generating a unique tag in the `containerdiff:`
	// namespace and by updating the manifest.json file in the tar archive.
	manifestItems[0].RepoTags = []string{tag}

	return json.Marshal(manifestItems)
}

// generateNewArchive updates archive's manifest and returns the path of the
// newly generated archive.
func generateNewArchive(archive string) (string, error) {
	// tar reader
	f, err := os.Open(archive)
	if err != nil {
		return "", err
	}
	defer f.Close()
	tr := tar.NewReader(f)

	// tar writer
	tmpfile, err := ioutil.TempFile("", "containerdiff-archive")
	if err != nil {
		return "", err
	}
	defer tmpfile.Close()
	tw := tar.NewWriter(tmpfile)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return "", err
		}

		// update the manifest
		switch t := hdr.Typeflag; t {
		case tar.TypeReg:
			if hdr.Name == "manifest.json" {
				newManifest, err := updateRepoTagsFromManifest(tr)
				if err != nil {
					return "", err
				}
				hdr.Size = int64(len(newManifest))
				if err := tw.WriteHeader(hdr); err != nil {
					return "", err
				}
				if _, err := tw.Write(newManifest); err != nil {
					return "", err
				}
				continue
			}
		}

		if err := tw.WriteHeader(hdr); err != nil {
			return "", err
		}
		if _, err := io.Copy(tw, tr); err != nil {
			return "", err
		}
	}

	return tmpfile.Name(), nil
}

// unlock returns the containerdiff file-system lock.  It is placed in the
// system's temporary directory to make sure it's accessible for all users in
// the system; no root required.
func getLockfile() (lockfile.Lockfile, error) {
	lockPath := filepath.Join(os.TempDir(), ".containerdiff.lock")
	lock, err := lockfile.New(lockPath)
	if err != nil {
		return lock, err
	}
	return lock, nil
}

// lock acquires the containerdiff file-system lock.
func lock() error {
	var err error
	var lock lockfile.Lockfile

	daemonMutex.Lock()
	lock, err = getLockfile()
	if err != nil {
		daemonMutex.Unlock()
		return fmt.Errorf("[lock] cannot init lockfile: %v", err)
	}

	// Try to acquire the lock and in case of a temporary error, sleep for
	// two seconds until the next retry (at most 10 times).  Return fatal
	// errors immediately, as we can't recover.
	for i := 0; i < 10; i++ {
		if err = lock.TryLock(); err != nil {
			switch err.(type) {
			case lockfile.TemporaryError:
				logrus.Debugf("[lock] busy: next retry in two seconds")
				time.Sleep(2 * time.Second)
			default:
				daemonMutex.Unlock()
				return fmt.Errorf("[lock] error acquiring lock: %s", err)
			}
		}
	}
	if err != nil {
		daemonMutex.Unlock()
		return fmt.Errorf("[lock] error acquiring lock: too many tries")
	}

	logrus.Debugf("[lock] lock acquired")
	return nil
}

// unlock releases the containerdiff file-system lock.  Note that errors can be
// ignored as there's no meaningful way to recover.
func unlock() error {
	lock, err := getLockfile()
	if err != nil {
		return fmt.Errorf("[unlock] cannot init lockfile: %v", err)
	}
	err = lock.Unlock()
	if err != nil {
		return fmt.Errorf("[unlock] error releasing lock: %s", err)
	}
	logrus.Debugf("[unlock] lock released")
	daemonMutex.Unlock()
	return nil
}
<commit_msg>rpm differ: use entrypoint for execution<commit_after>/*
Copyright 2018 Google, Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package differs

import (
	"archive/tar"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util"
	"github.com/GoogleContainerTools/container-diff/util"
	dockertar "github.com/containers/image/docker/tarfile"
	godocker "github.com/fsouza/go-dockerclient"
	"github.com/nightlyone/lockfile"
	"github.com/sirupsen/logrus"
)

// daemonMutex is required to protect against other go-routines, as
// nightlyone/lockfile implements a recursive lock, which doesn't protect
// against other go-routines that have the same PID.  Note that the mutex
// *must* always be locked prior to the lockfile, and unlocked after.
var daemonMutex sync.Mutex

type RPMAnalyzer struct {
}

// Name returns the name of the analyzer.
func (a RPMAnalyzer) Name() string {
	return "RPMAnalyzer"
}

// Diff compares the installed rpm packages of image1 and image2.
func (a RPMAnalyzer) Diff(image1, image2 pkgutil.Image) (util.Result, error) {
	diff, err := singleVersionDiff(image1, image2, a)
	return diff, err
}

// Analyze collects information of the installed rpm packages on image.
func (a RPMAnalyzer) Analyze(image pkgutil.Image) (util.Result, error) {
	analysis, err := singleVersionAnalysis(image, a)
	return analysis, err
}

// getPackages returns a map of installed rpm package on image.
func (a RPMAnalyzer) getPackages(image pkgutil.Image) (map[string]util.PackageInfo, error) {
	path := image.FSPath
	packages := make(map[string]util.PackageInfo)
	if _, err := os.Stat(path); err != nil {
		// invalid image directory path
		return packages, err
	}

	// try to find the rpm binary in bin/ or usr/bin/
	rpmBinary := filepath.Join(path, "bin/rpm")
	if _, err := os.Stat(rpmBinary); err != nil {
		rpmBinary = filepath.Join(path, "usr/bin/rpm")
		if _, err = os.Stat(rpmBinary); err != nil {
			logrus.Errorf("Could not detect RPM binary in unpacked image %s", image.Source)
			return packages, nil
		}
	}

	return rpmDataFromContainer(image)
}

// rpmDataFromContainer runs image in a container, queries the data of
// installed rpm packages and returns a map of packages.
func rpmDataFromContainer(image pkgutil.Image) (map[string]util.PackageInfo, error) {
	packages := make(map[string]util.PackageInfo)

	client, err := godocker.NewClientFromEnv()
	if err != nil {
		return packages, err
	}

	// if image is a tar archive, we need to load it
	imageName := image.Source
	if image.IsTar() {
		// NOTE: all work below is only required for multilayer tar
		// archives generated by `docker save`.  Flat file-system
		// archives generated by `docker create` can just be imported
		// without potential name conflicts.  However, those are not
		// supported yet.
		if err := lock(); err != nil {
			return packages, err
		}

		archive, err := generateNewArchive(imageName)
		if err != nil {
			logrus.Errorf(err.Error())
		}
		defer os.Remove(archive)

		imageName, err = loadArchiveToDaemon(archive)
		if err != nil {
			return packages, fmt.Errorf("Error loading archive: %s", err)
		}
		unlock()

		defer client.RemoveImage(imageName)
		defer logrus.Infof("Removing image %s", imageName)
	} else if image.IsCloud() {
		// if it's a remote image we pull it unconditionally and avoid
		// name/tag conflicts by pulling it via its sha256 digest
		var buf bytes.Buffer
		digest, err := image.GetRemoteDigest()
		if err != nil {
			return packages, err
		}
		imageName = image.GetName() + "@" + digest
		logrus.Infof("Pulling remote image %s", imageName)
		pullOpts := godocker.PullImageOptions{
			Repository:   imageName,
			OutputStream: &buf,
		}
		if err := client.PullImage(pullOpts, godocker.AuthConfiguration{}); err != nil {
			return packages, fmt.Errorf("Error pulling remote image %s: %s", imageName, err)
		}
		// log PullImage() output in debug mode
		if logrus.GetLevel() == logrus.DebugLevel {
			// log each line separately for consistency
			for _, out := range strings.Split(buf.String(), "\n") {
				if len(out) > 0 {
					logrus.Debug(out)
				}
			}
		}
		defer client.RemoveImage(imageName)
	}

	contConf := godocker.Config{
		Entrypoint: []string{"rpm", "--nodigest", "--nosignature", "-qa", "--qf", "%{NAME}\t%{VERSION}\t%{SIZE}\n"},
		Image:      imageName,
	}

	hostConf := godocker.HostConfig{
		AutoRemove: true,
	}

	contOpts := godocker.CreateContainerOptions{Config: &contConf}
	container, err := client.CreateContainer(contOpts)
	if err != nil {
		return packages, err
	}
	logrus.Infof("Created container %s", container.ID)

	removeOpts := godocker.RemoveContainerOptions{
		ID: container.ID,
	}
	defer client.RemoveContainer(removeOpts)

	if err := client.StartContainer(container.ID, &hostConf); err != nil {
		return packages, err
	}

	exitCode, err := client.WaitContainer(container.ID)
	if err != nil {
		return packages, err
	}

	outBuf := new(bytes.Buffer)
	errBuf := new(bytes.Buffer)
	logOpts := godocker.LogsOptions{
		Context:      context.Background(),
		Container:    container.ID,
		Stdout:       true,
		Stderr:       true,
		OutputStream: outBuf,
		ErrorStream:  errBuf,
	}

	if err := client.Logs(logOpts); err != nil {
		return packages, err
	}

	if exitCode != 0 {
		return packages, fmt.Errorf("non-zero exit code %d: %s", exitCode, errBuf.String())
	}

	output := strings.Split(outBuf.String(), "\n")
	return parsePackageData(output)
}

// parsePackageData parses the package data of each line in rpmOutput and
// returns a map of packages.
func parsePackageData(rpmOutput []string) (map[string]util.PackageInfo, error) {
	packages := make(map[string]util.PackageInfo)

	for _, output := range rpmOutput {
		spl := strings.Split(output, "\t")
		if len(spl) != 3 {
			// ignore the empty (last) line
			if output != "" {
				logrus.Errorf("unexpected rpm-query output: '%s'", output)
			}
			continue
		}
		pkg := util.PackageInfo{}

		var err error
		pkg.Size, err = strconv.ParseInt(spl[2], 10, 64)
		if err != nil {
			return packages, fmt.Errorf("error converting package size: %s", spl[2])
		}

		pkg.Version = spl[1]
		packages[spl[0]] = pkg
	}

	return packages, nil
}

// loadArchiveToDaemon loads the image specified by archive to the docker
// Daemon.
func loadArchiveToDaemon(archive string) (string, error) {
	client, err := godocker.NewClientFromEnv()
	if err != nil {
		return "", err
	}

	inBuf, err := os.Open(archive)
	if err != nil {
		return "", err
	}

	outBuf := new(bytes.Buffer)
	loadOpts := godocker.LoadImageOptions{
		InputStream:  inBuf,
		OutputStream: outBuf,
		Context:      context.Background(),
	}
	if err := client.LoadImage(loadOpts); err != nil {
		return "", err
	}

	// output has the form "Loaded image: repository:tag"
	output := outBuf.String()
	spl := strings.Split(output, ": ")
	if len(spl) != 2 {
		return "", fmt.Errorf("unexpected docker load output: %s", output)
	}
	image := strings.TrimSpace(spl[1])
	logrus.Infof("Loaded %s as image %s", archive, image)

	return image, nil
}

// updateRepoTagsFromManifest updates the RepoTags in the manifest.json with a
// unique tag in the `containerdiff` image namespace.  There is a maximum of 10
// tries to generate a unique tag.
func updateRepoTagsFromManifest(r io.Reader) ([]byte, error) {
	f, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}

	var manifestItems []dockertar.ManifestItem
	if err := json.Unmarshal(f, &manifestItems); err != nil {
		return nil, fmt.Errorf("error unmarshalling manifest: %s", err)
	}

	// error out in case the tar ships multiple images
	if len(manifestItems) != 1 {
		return nil, fmt.Errorf("tar archive ships %d images, but only 1 is supported", len(manifestItems))
	}

	client, err := godocker.NewClientFromEnv()
	if err != nil {
		return nil, err
	}

	tag := ""
	rand.Seed(time.Now().UnixNano())
	for i := 0; i < 10; i++ {
		tag = "containerdiff:" + strconv.FormatUint(rand.Uint64(), 10)
		logrus.Debugf("generated new random tag: %s", tag)
		if _, err := client.InspectImage(tag); err != nil {
			break
		}
		tag = ""
	}
	if len(tag) == 0 {
		return nil, fmt.Errorf("failed to generate unique tag: too many tries")
	}

	// Overwrite the `RepoTags` member with a unique tag, which determines
	// the image's tag after being loaded by the Docker daemon.
	// Unfortunately, `docker load` does not generate a new tag in case of
	// a conflict, but it will rename the already present image.  We avoid
	// such conflicts by generating a unique tag in the `containerdiff:`
	// namespace and by updating the manifest.json file in the tar archive.
	manifestItems[0].RepoTags = []string{tag}

	return json.Marshal(manifestItems)
}

// generateNewArchive updates archive's manifest and returns the path of the
// newly generated archive.
func generateNewArchive(archive string) (string, error) {
	// tar reader
	f, err := os.Open(archive)
	if err != nil {
		return "", err
	}
	defer f.Close()
	tr := tar.NewReader(f)

	// tar writer
	tmpfile, err := ioutil.TempFile("", "containerdiff-archive")
	if err != nil {
		return "", err
	}
	defer tmpfile.Close()
	tw := tar.NewWriter(tmpfile)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return "", err
		}

		// update the manifest
		switch t := hdr.Typeflag; t {
		case tar.TypeReg:
			if hdr.Name == "manifest.json" {
				newManifest, err := updateRepoTagsFromManifest(tr)
				if err != nil {
					return "", err
				}
				hdr.Size = int64(len(newManifest))
				if err := tw.WriteHeader(hdr); err != nil {
					return "", err
				}
				if _, err := tw.Write(newManifest); err != nil {
					return "", err
				}
				continue
			}
		}

		if err := tw.WriteHeader(hdr); err != nil {
			return "", err
		}
		if _, err := io.Copy(tw, tr); err != nil {
			return "", err
		}
	}

	return tmpfile.Name(), nil
}

// unlock returns the containerdiff file-system lock.  It is placed in the
// system's temporary directory to make sure it's accessible for all users in
// the system; no root required.
func getLockfile() (lockfile.Lockfile, error) {
	lockPath := filepath.Join(os.TempDir(), ".containerdiff.lock")
	lock, err := lockfile.New(lockPath)
	if err != nil {
		return lock, err
	}
	return lock, nil
}

// lock acquires the containerdiff file-system lock.
func lock() error {
	var err error
	var lock lockfile.Lockfile

	daemonMutex.Lock()
	lock, err = getLockfile()
	if err != nil {
		daemonMutex.Unlock()
		return fmt.Errorf("[lock] cannot init lockfile: %v", err)
	}

	// Try to acquire the lock and in case of a temporary error, sleep for
	// two seconds until the next retry (at most 10 times).  Return fatal
	// errors immediately, as we can't recover.
	for i := 0; i < 10; i++ {
		if err = lock.TryLock(); err != nil {
			switch err.(type) {
			case lockfile.TemporaryError:
				logrus.Debugf("[lock] busy: next retry in two seconds")
				time.Sleep(2 * time.Second)
			default:
				daemonMutex.Unlock()
				return fmt.Errorf("[lock] error acquiring lock: %s", err)
			}
		}
	}
	if err != nil {
		daemonMutex.Unlock()
		return fmt.Errorf("[lock] error acquiring lock: too many tries")
	}

	logrus.Debugf("[lock] lock acquired")
	return nil
}

// unlock releases the containerdiff file-system lock.  Note that errors can be
// ignored as there's no meaningful way to recover.
func unlock() error {
	lock, err := getLockfile()
	if err != nil {
		return fmt.Errorf("[unlock] cannot init lockfile: %v", err)
	}
	err = lock.Unlock()
	if err != nil {
		return fmt.Errorf("[unlock] error releasing lock: %s", err)
	}
	logrus.Debugf("[unlock] lock released")
	daemonMutex.Unlock()
	return nil
}
<|endoftext|>


<commit_before>

package main

import (
    "fmt"
    "math/big"
)

func factoral(n uint64) (r *big.Int) {

    one, bn := big.NewInt(1), new(big.Int).SetUint64(n)

    r = big.NewInt(1)
    if bn.Cmp(one) <= 0 {
        return
    }
    for i := big.NewInt(2); i.Cmp(bn) <= 0; i.Add(i, one) {
        r.Mul(r, i)
    }
    return
}

func add(number *big.Int) *big.Int {
    ten := big.NewInt(10)
    sum := big.NewInt(0)
    mod := big.NewInt(0)
    for ten.Cmp(number)<0 {
      sum.Add(sum, mod.Mod(number,ten))
      number.Div(number,ten)
    }
    sum.Add(sum,number)
  return sum
}
func main() {

	fmt.Printf("The Sum of the digits from the factorial value of 100 is: ")
    fmt.Println(add(factoral(100)))

}<commit_msg>ex4 - sum of digits from factorial of 100<commit_after>//Kimberly Burke
//G00269948
//https://stackoverflow.com/questions/46395819/get-sum-of-bigint-number-golang

package main

import (
    "fmt"
    "math/big"
)

//factorial function to get factorial value of a num
func factoral(n uint64) (r *big.Int) {

    one, bn := big.NewInt(1), new(big.Int).SetUint64(n)

    r = big.NewInt(1)
    if bn.Cmp(one) <= 0 {
        return
    }
    for i := big.NewInt(2); i.Cmp(bn) <= 0; i.Add(i, one) {
        r.Mul(r, i)
    }
    return
}

//gets sum of the digits from the factorial value
func add(number *big.Int) *big.Int {
    ten := big.NewInt(10)
    sum := big.NewInt(0)
    mod := big.NewInt(0)
    for ten.Cmp(number)<0 {
      sum.Add(sum, mod.Mod(number,ten))
      number.Div(number,ten)
    }
    sum.Add(sum,number)
  return sum
}
func main() {
	//prints results
	fmt.Printf("The Sum of the digits from the factorial value of 100 is: ")
    fmt.Println(add(factoral(100)))

}<|endoftext|>


<commit_before>package msgpack

import (
	"bytes"
	"fmt"
	"reflect"
	"sync"

	"github.com/vmihailenco/msgpack/codes"
)

var extTypes = make(map[int8]reflect.Type)

var bufferPool = &sync.Pool{
	New: func() interface{} {
		return new(bytes.Buffer)
	},
}

// RegisterExt records a type, identified by a value for that type,
// under the provided id. That id will identify the concrete type of a value
// sent or received as an interface variable. Only types that will be
// transferred as implementations of interface values need to be registered.
// Expecting to be used only during initialization, it panics if the mapping
// between types and ids is not a bijection.
func RegisterExt(id int8, value interface{}) {
	typ := reflect.TypeOf(value)
	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
	}
	ptr := reflect.PtrTo(typ)

	if _, ok := extTypes[id]; ok {
		panic(fmt.Errorf("msgpack: ext with id=%d is already registered", id))
	}

	registerExt(id, ptr, getEncoder(ptr), nil)
	registerExt(id, typ, getEncoder(typ), getDecoder(typ))
}

func registerExt(id int8, typ reflect.Type, enc encoderFunc, dec decoderFunc) {
	if dec != nil {
		extTypes[id] = typ
	}
	if enc != nil {
		typEncMap[typ] = makeExtEncoder(id, enc)
	}
	if dec != nil {
		typDecMap[typ] = dec
	}
}

func (e *Encoder) EncodeExtHeader(typeId byte, length int) error {
	if err := e.encodeExtLen(length); err != nil {
		return err
	}
	if err := e.w.WriteByte(typeId); err != nil {
		return err
	}
	return nil
}

func makeExtEncoder(typeId int8, enc encoderFunc) encoderFunc {
	return func(e *Encoder, v reflect.Value) error {
		buf := bufferPool.Get().(*bytes.Buffer)
		defer bufferPool.Put(buf)
		buf.Reset()

		oldw := e.w
		e.w = buf
		err := enc(e, v)
		e.w = oldw

		if err != nil {
			return err
		}

		if err := e.EncodeExtHeader(byte(typeId), buf.Len()); err != nil {
			return err
		}
		return e.write(buf.Bytes())
	}
}

func (e *Encoder) encodeExtLen(l int) error {
	switch l {
	case 1:
		return e.writeCode(codes.FixExt1)
	case 2:
		return e.writeCode(codes.FixExt2)
	case 4:
		return e.writeCode(codes.FixExt4)
	case 8:
		return e.writeCode(codes.FixExt8)
	case 16:
		return e.writeCode(codes.FixExt16)
	}
	if l < 256 {
		return e.write1(codes.Ext8, uint64(l))
	}
	if l < 65536 {
		return e.write2(codes.Ext16, uint64(l))
	}
	return e.write4(codes.Ext32, uint32(l))
}

func (d *Decoder) parseExtLen(c codes.Code) (int, error) {
	switch c {
	case codes.FixExt1:
		return 1, nil
	case codes.FixExt2:
		return 2, nil
	case codes.FixExt4:
		return 4, nil
	case codes.FixExt8:
		return 8, nil
	case codes.FixExt16:
		return 16, nil
	case codes.Ext8:
		n, err := d.uint8()
		return int(n), err
	case codes.Ext16:
		n, err := d.uint16()
		return int(n), err
	case codes.Ext32:
		n, err := d.uint32()
		return int(n), err
	default:
		return 0, fmt.Errorf("msgpack: invalid code=%x decoding ext length", c)
	}
}

func (d *Decoder) decodeExtHeader(c codes.Code) (typeId int8, length int, err error) {
	length, err = d.parseExtLen(c)
	if err != nil {
		return
	}

	c, err = d.readCode()
	if err != nil {
		return
	}

	typeId = int8(c)
	return
}

func (d *Decoder) DecodeExtHeader() (typeId int8, length int, err error) {
	c, err := d.readCode()
	if err != nil {
		return
	}
	return d.decodeExtHeader(c)
}

func (d *Decoder) extInterface(c codes.Code) (interface{}, error) {
	extId, extLen, err := d.decodeExtHeader(c)
	if err != nil {
		return nil, err
	}

	typ, ok := extTypes[extId]
	if !ok {
		return nil, fmt.Errorf("msgpack: unregistered ext id=%d", extId)
	}

	v := reflect.New(typ)

	d.extLen = extLen
	err = d.DecodeValue(v.Elem())
	d.extLen = 0
	if err != nil {
		return nil, err
	}

	return v.Interface(), nil
}

func (d *Decoder) skipExt(c codes.Code) error {
	n, err := d.parseExtLen(c)
	if err != nil {
		return err
	}
	return d.skipN(n + 1)
}

func (d *Decoder) skipExtHeader(c codes.Code) error {
	// Read ext type.
	_, err := d.readCode()
	if err != nil {
		return err
	}
	// Read ext body len.
	for i := 0; i < extHeaderLen(c); i++ {
		_, err := d.readCode()
		if err != nil {
			return err
		}
	}
	return nil
}

func extHeaderLen(c codes.Code) int {
	switch c {
	case codes.Ext8:
		return 1
	case codes.Ext16:
		return 2
	case codes.Ext32:
		return 4
	}
	return 0
}
<commit_msg>Code cleanup<commit_after>package msgpack

import (
	"bytes"
	"fmt"
	"reflect"
	"sync"

	"github.com/vmihailenco/msgpack/codes"
)

var extTypes = make(map[int8]reflect.Type)

var bufferPool = &sync.Pool{
	New: func() interface{} {
		return new(bytes.Buffer)
	},
}

// RegisterExt records a type, identified by a value for that type,
// under the provided id. That id will identify the concrete type of a value
// sent or received as an interface variable. Only types that will be
// transferred as implementations of interface values need to be registered.
// Expecting to be used only during initialization, it panics if the mapping
// between types and ids is not a bijection.
func RegisterExt(id int8, value interface{}) {
	typ := reflect.TypeOf(value)
	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
	}
	ptr := reflect.PtrTo(typ)

	if _, ok := extTypes[id]; ok {
		panic(fmt.Errorf("msgpack: ext with id=%d is already registered", id))
	}

	registerExt(id, ptr, getEncoder(ptr), nil)
	registerExt(id, typ, getEncoder(typ), getDecoder(typ))
}

func registerExt(id int8, typ reflect.Type, enc encoderFunc, dec decoderFunc) {
	if dec != nil {
		extTypes[id] = typ
	}
	if enc != nil {
		typEncMap[typ] = makeExtEncoder(id, enc)
	}
	if dec != nil {
		typDecMap[typ] = dec
	}
}

func (e *Encoder) EncodeExtHeader(typeId int8, length int) error {
	if err := e.encodeExtLen(length); err != nil {
		return err
	}
	if err := e.w.WriteByte(byte(typeId)); err != nil {
		return err
	}
	return nil
}

func makeExtEncoder(typeId int8, enc encoderFunc) encoderFunc {
	return func(e *Encoder, v reflect.Value) error {
		buf := bufferPool.Get().(*bytes.Buffer)
		defer bufferPool.Put(buf)
		buf.Reset()

		oldw := e.w
		e.w = buf
		err := enc(e, v)
		e.w = oldw

		if err != nil {
			return err
		}

		if err := e.EncodeExtHeader(typeId, buf.Len()); err != nil {
			return err
		}
		return e.write(buf.Bytes())
	}
}

func (e *Encoder) encodeExtLen(l int) error {
	switch l {
	case 1:
		return e.writeCode(codes.FixExt1)
	case 2:
		return e.writeCode(codes.FixExt2)
	case 4:
		return e.writeCode(codes.FixExt4)
	case 8:
		return e.writeCode(codes.FixExt8)
	case 16:
		return e.writeCode(codes.FixExt16)
	}
	if l < 256 {
		return e.write1(codes.Ext8, uint64(l))
	}
	if l < 65536 {
		return e.write2(codes.Ext16, uint64(l))
	}
	return e.write4(codes.Ext32, uint32(l))
}

func (d *Decoder) parseExtLen(c codes.Code) (int, error) {
	switch c {
	case codes.FixExt1:
		return 1, nil
	case codes.FixExt2:
		return 2, nil
	case codes.FixExt4:
		return 4, nil
	case codes.FixExt8:
		return 8, nil
	case codes.FixExt16:
		return 16, nil
	case codes.Ext8:
		n, err := d.uint8()
		return int(n), err
	case codes.Ext16:
		n, err := d.uint16()
		return int(n), err
	case codes.Ext32:
		n, err := d.uint32()
		return int(n), err
	default:
		return 0, fmt.Errorf("msgpack: invalid code=%x decoding ext length", c)
	}
}

func (d *Decoder) decodeExtHeader(c codes.Code) (int8, int, error) {
	length, err := d.parseExtLen(c)
	if err != nil {
		return 0, 0, err
	}

	typeId, err := d.readCode()
	if err != nil {
		return 0, 0, err
	}

	return int8(typeId), length, nil
}

func (d *Decoder) DecodeExtHeader() (typeId int8, length int, err error) {
	c, err := d.readCode()
	if err != nil {
		return
	}
	return d.decodeExtHeader(c)
}

func (d *Decoder) extInterface(c codes.Code) (interface{}, error) {
	extId, extLen, err := d.decodeExtHeader(c)
	if err != nil {
		return nil, err
	}

	typ, ok := extTypes[extId]
	if !ok {
		return nil, fmt.Errorf("msgpack: unregistered ext id=%d", extId)
	}

	v := reflect.New(typ)

	d.extLen = extLen
	err = d.DecodeValue(v.Elem())
	d.extLen = 0
	if err != nil {
		return nil, err
	}

	return v.Interface(), nil
}

func (d *Decoder) skipExt(c codes.Code) error {
	n, err := d.parseExtLen(c)
	if err != nil {
		return err
	}
	return d.skipN(n + 1)
}

func (d *Decoder) skipExtHeader(c codes.Code) error {
	// Read ext type.
	_, err := d.readCode()
	if err != nil {
		return err
	}
	// Read ext body len.
	for i := 0; i < extHeaderLen(c); i++ {
		_, err := d.readCode()
		if err != nil {
			return err
		}
	}
	return nil
}

func extHeaderLen(c codes.Code) int {
	switch c {
	case codes.Ext8:
		return 1
	case codes.Ext16:
		return 2
	case codes.Ext32:
		return 4
	}
	return 0
}
<|endoftext|>