<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	release_1_4 "k8s.io/client-go/1.4/kubernetes"
	apiapi "k8s.io/client-go/1.4/pkg/api"
	"k8s.io/client-go/1.4/pkg/api/unversioned"
	api "k8s.io/client-go/1.4/pkg/api/v1"
	extensions "k8s.io/client-go/1.4/pkg/apis/extensions/v1beta1"
	policy "k8s.io/client-go/1.4/pkg/apis/policy/v1alpha1"
	"k8s.io/client-go/1.4/pkg/util/intstr"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/test/e2e/framework"
)

// timeout is used for most polling/waiting activities
const timeout = 60 * time.Second

// schedulingTimeout is longer specifically because sometimes we need to wait
// awhile to guarantee that we've been patient waiting for something ordinary
// to happen: a pod to get scheduled and move into Ready
const schedulingTimeout = 10 * time.Minute

var _ = framework.KubeDescribe("DisruptionController", func() {
	f := framework.NewDefaultFramework("disruption")
	var ns string
	var cs *release_1_4.Clientset

	BeforeEach(func() {
		// skip on GKE since alpha features are disabled
		framework.SkipIfProviderIs("gke")

		cs = f.StagingClient
		ns = f.Namespace.Name
	})

	It("should create a PodDisruptionBudget", func() {
		createPodDisruptionBudgetOrDie(cs, ns, intstr.FromString("1%"))
	})

	It("should update PodDisruptionBudget status", func() {
		createPodDisruptionBudgetOrDie(cs, ns, intstr.FromInt(2))

		createPodsOrDie(cs, ns, 2)

		// Since disruptionAllowed starts out false, if we see it ever become true,
		// that means the controller is working.
		err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
			pdb, err := cs.Policy().PodDisruptionBudgets(ns).Get("foo")
			if err != nil {
				return false, err
			}
			return pdb.Status.PodDisruptionAllowed, nil
		})
		Expect(err).NotTo(HaveOccurred())

	})

	evictionCases := []struct {
		description    string
		minAvailable   intstr.IntOrString
		podCount       int
		replicaSetSize int32
		shouldDeny     bool
		exclusive      bool
	}{
		{
			description:  "no PDB",
			minAvailable: intstr.FromString(""),
			podCount:     1,
			shouldDeny:   false,
		}, {
			description:  "too few pods, absolute",
			minAvailable: intstr.FromInt(2),
			podCount:     1,
			shouldDeny:   true,
		}, {
			description:  "enough pods, absolute",
			minAvailable: intstr.FromInt(2),
			podCount:     2,
			shouldDeny:   false,
		}, {
			description:    "enough pods, replicaSet, percentage",
			minAvailable:   intstr.FromString("100%"),
			replicaSetSize: 10,
			exclusive:      false,
			shouldDeny:     false,
		}, {
			description:    "too few pods, replicaSet, percentage",
			minAvailable:   intstr.FromString("100%"),
			replicaSetSize: 10,
			exclusive:      true,
			shouldDeny:     true,
		},
	}
	for _, c := range evictionCases {
		expectation := "should allow an eviction"
		if c.shouldDeny {
			expectation = "should not allow an eviction"
		}
		It(fmt.Sprintf("evictions: %s => %s", c.description, expectation), func() {
			createPodsOrDie(cs, ns, c.podCount)
			if c.replicaSetSize > 0 {
				createReplicaSetOrDie(cs, ns, c.replicaSetSize, c.exclusive)
			}

			if c.minAvailable.String() != "" {
				createPodDisruptionBudgetOrDie(cs, ns, c.minAvailable)
			}

			// Locate a running pod.
			var pod api.Pod
			err := wait.PollImmediate(framework.Poll, schedulingTimeout, func() (bool, error) {
				podList, err := cs.Pods(ns).List(apiapi.ListOptions{})
				if err != nil {
					return false, err
				}

				for i := range podList.Items {
					if podList.Items[i].Status.Phase == api.PodRunning {
						pod = podList.Items[i]
						return true, nil
					}
				}

				return false, nil
			})
			Expect(err).NotTo(HaveOccurred())

			e := &policy.Eviction{
				ObjectMeta: api.ObjectMeta{
					Name:      pod.Name,
					Namespace: ns,
				},
			}

			if c.shouldDeny {
				// Since disruptionAllowed starts out false, wait at least 60s hoping that
				// this gives the controller enough time to have truly set the status.
				time.Sleep(timeout)

				err = cs.Pods(ns).Evict(e)
				Expect(err).Should(MatchError("Cannot evict pod as it would violate the pod's disruption budget."))
			} else {
				// Since disruptionAllowed starts out false, if an eviction is ever allowed,
				// that means the controller is working.
				err = wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
					err = cs.Pods(ns).Evict(e)
					if err != nil {
						return false, nil
					} else {
						return true, nil
					}
				})
				Expect(err).NotTo(HaveOccurred())
			}
		})
	}

})

func createPodDisruptionBudgetOrDie(cs *release_1_4.Clientset, ns string, minAvailable intstr.IntOrString) {
	pdb := policy.PodDisruptionBudget{
		ObjectMeta: api.ObjectMeta{
			Name:      "foo",
			Namespace: ns,
		},
		Spec: policy.PodDisruptionBudgetSpec{
			Selector:     &unversioned.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}},
			MinAvailable: minAvailable,
		},
	}
	_, err := cs.Policy().PodDisruptionBudgets(ns).Create(&pdb)
	Expect(err).NotTo(HaveOccurred())
}

func createPodsOrDie(cs *release_1_4.Clientset, ns string, n int) {
	for i := 0; i < n; i++ {
		pod := &api.Pod{
			ObjectMeta: api.ObjectMeta{
				Name:      fmt.Sprintf("pod-%d", i),
				Namespace: ns,
				Labels:    map[string]string{"foo": "bar"},
			},
			Spec: api.PodSpec{
				Containers: []api.Container{
					{
						Name:  "busybox",
						Image: "gcr.io/google_containers/echoserver:1.4",
					},
				},
				RestartPolicy: api.RestartPolicyAlways,
			},
		}

		_, err := cs.Pods(ns).Create(pod)
		framework.ExpectNoError(err, "Creating pod %q in namespace %q", pod.Name, ns)
	}
}

func createReplicaSetOrDie(cs *release_1_4.Clientset, ns string, size int32, exclusive bool) {
	container := api.Container{
		Name:  "busybox",
		Image: "gcr.io/google_containers/echoserver:1.4",
	}
	if exclusive {
		container.Ports = []api.ContainerPort{
			{HostPort: 5555, ContainerPort: 5555},
		}
	}

	rs := &extensions.ReplicaSet{
		ObjectMeta: api.ObjectMeta{
			Name:      "rs",
			Namespace: ns,
		},
		Spec: extensions.ReplicaSetSpec{
			Replicas: &size,
			Selector: &extensions.LabelSelector{
				MatchLabels: map[string]string{"foo": "bar"},
			},
			Template: api.PodTemplateSpec{
				ObjectMeta: api.ObjectMeta{
					Labels: map[string]string{"foo": "bar"},
				},
				Spec: api.PodSpec{
					Containers: []api.Container{container},
				},
			},
		},
	}

	_, err := cs.Extensions().ReplicaSets(ns).Create(rs)
	framework.ExpectNoError(err, "Creating replica set %q in namespace %q", rs.Name, ns)
}
<commit_msg>Stop referring to loop variable inside closure.<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	release_1_4 "k8s.io/client-go/1.4/kubernetes"
	apiapi "k8s.io/client-go/1.4/pkg/api"
	"k8s.io/client-go/1.4/pkg/api/unversioned"
	api "k8s.io/client-go/1.4/pkg/api/v1"
	extensions "k8s.io/client-go/1.4/pkg/apis/extensions/v1beta1"
	policy "k8s.io/client-go/1.4/pkg/apis/policy/v1alpha1"
	"k8s.io/client-go/1.4/pkg/util/intstr"
	"k8s.io/kubernetes/pkg/util/wait"
	"k8s.io/kubernetes/test/e2e/framework"
)

// timeout is used for most polling/waiting activities
const timeout = 60 * time.Second

// schedulingTimeout is longer specifically because sometimes we need to wait
// awhile to guarantee that we've been patient waiting for something ordinary
// to happen: a pod to get scheduled and move into Ready
const schedulingTimeout = 10 * time.Minute

var _ = framework.KubeDescribe("DisruptionController", func() {
	f := framework.NewDefaultFramework("disruption")
	var ns string
	var cs *release_1_4.Clientset

	BeforeEach(func() {
		// skip on GKE since alpha features are disabled
		framework.SkipIfProviderIs("gke")

		cs = f.StagingClient
		ns = f.Namespace.Name
	})

	It("should create a PodDisruptionBudget", func() {
		createPodDisruptionBudgetOrDie(cs, ns, intstr.FromString("1%"))
	})

	It("should update PodDisruptionBudget status", func() {
		createPodDisruptionBudgetOrDie(cs, ns, intstr.FromInt(2))

		createPodsOrDie(cs, ns, 2)

		// Since disruptionAllowed starts out false, if we see it ever become true,
		// that means the controller is working.
		err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
			pdb, err := cs.Policy().PodDisruptionBudgets(ns).Get("foo")
			if err != nil {
				return false, err
			}
			return pdb.Status.PodDisruptionAllowed, nil
		})
		Expect(err).NotTo(HaveOccurred())

	})

	evictionCases := []struct {
		description    string
		minAvailable   intstr.IntOrString
		podCount       int
		replicaSetSize int32
		shouldDeny     bool
		exclusive      bool
	}{
		{
			description:  "no PDB",
			minAvailable: intstr.FromString(""),
			podCount:     1,
			shouldDeny:   false,
		}, {
			description:  "too few pods, absolute",
			minAvailable: intstr.FromInt(2),
			podCount:     1,
			shouldDeny:   true,
		}, {
			description:  "enough pods, absolute",
			minAvailable: intstr.FromInt(2),
			podCount:     2,
			shouldDeny:   false,
		}, {
			description:    "enough pods, replicaSet, percentage",
			minAvailable:   intstr.FromString("100%"),
			replicaSetSize: 10,
			exclusive:      false,
			shouldDeny:     false,
		}, {
			description:    "too few pods, replicaSet, percentage",
			minAvailable:   intstr.FromString("100%"),
			replicaSetSize: 10,
			exclusive:      true,
			shouldDeny:     true,
		},
	}
	for i := range evictionCases {
		c := evictionCases[i]
		expectation := "should allow an eviction"
		if c.shouldDeny {
			expectation = "should not allow an eviction"
		}
		It(fmt.Sprintf("evictions: %s => %s", c.description, expectation), func() {
			createPodsOrDie(cs, ns, c.podCount)
			if c.replicaSetSize > 0 {
				createReplicaSetOrDie(cs, ns, c.replicaSetSize, c.exclusive)
			}

			if c.minAvailable.String() != "" {
				createPodDisruptionBudgetOrDie(cs, ns, c.minAvailable)
			}

			// Locate a running pod.
			var pod api.Pod
			err := wait.PollImmediate(framework.Poll, schedulingTimeout, func() (bool, error) {
				podList, err := cs.Pods(ns).List(apiapi.ListOptions{})
				if err != nil {
					return false, err
				}

				for i := range podList.Items {
					if podList.Items[i].Status.Phase == api.PodRunning {
						pod = podList.Items[i]
						return true, nil
					}
				}

				return false, nil
			})
			Expect(err).NotTo(HaveOccurred())

			e := &policy.Eviction{
				ObjectMeta: api.ObjectMeta{
					Name:      pod.Name,
					Namespace: ns,
				},
			}

			if c.shouldDeny {
				// Since disruptionAllowed starts out false, wait at least 60s hoping that
				// this gives the controller enough time to have truly set the status.
				time.Sleep(timeout)

				err = cs.Pods(ns).Evict(e)
				Expect(err).Should(MatchError("Cannot evict pod as it would violate the pod's disruption budget."))
			} else {
				// Since disruptionAllowed starts out false, if an eviction is ever allowed,
				// that means the controller is working.
				err = wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
					err = cs.Pods(ns).Evict(e)
					if err != nil {
						return false, nil
					} else {
						return true, nil
					}
				})
				Expect(err).NotTo(HaveOccurred())
			}
		})
	}

})

func createPodDisruptionBudgetOrDie(cs *release_1_4.Clientset, ns string, minAvailable intstr.IntOrString) {
	pdb := policy.PodDisruptionBudget{
		ObjectMeta: api.ObjectMeta{
			Name:      "foo",
			Namespace: ns,
		},
		Spec: policy.PodDisruptionBudgetSpec{
			Selector:     &unversioned.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}},
			MinAvailable: minAvailable,
		},
	}
	_, err := cs.Policy().PodDisruptionBudgets(ns).Create(&pdb)
	Expect(err).NotTo(HaveOccurred())
}

func createPodsOrDie(cs *release_1_4.Clientset, ns string, n int) {
	for i := 0; i < n; i++ {
		pod := &api.Pod{
			ObjectMeta: api.ObjectMeta{
				Name:      fmt.Sprintf("pod-%d", i),
				Namespace: ns,
				Labels:    map[string]string{"foo": "bar"},
			},
			Spec: api.PodSpec{
				Containers: []api.Container{
					{
						Name:  "busybox",
						Image: "gcr.io/google_containers/echoserver:1.4",
					},
				},
				RestartPolicy: api.RestartPolicyAlways,
			},
		}

		_, err := cs.Pods(ns).Create(pod)
		framework.ExpectNoError(err, "Creating pod %q in namespace %q", pod.Name, ns)
	}
}

func createReplicaSetOrDie(cs *release_1_4.Clientset, ns string, size int32, exclusive bool) {
	container := api.Container{
		Name:  "busybox",
		Image: "gcr.io/google_containers/echoserver:1.4",
	}
	if exclusive {
		container.Ports = []api.ContainerPort{
			{HostPort: 5555, ContainerPort: 5555},
		}
	}

	rs := &extensions.ReplicaSet{
		ObjectMeta: api.ObjectMeta{
			Name:      "rs",
			Namespace: ns,
		},
		Spec: extensions.ReplicaSetSpec{
			Replicas: &size,
			Selector: &extensions.LabelSelector{
				MatchLabels: map[string]string{"foo": "bar"},
			},
			Template: api.PodTemplateSpec{
				ObjectMeta: api.ObjectMeta{
					Labels: map[string]string{"foo": "bar"},
				},
				Spec: api.PodSpec{
					Containers: []api.Container{container},
				},
			},
		},
	}

	_, err := cs.Extensions().ReplicaSets(ns).Create(rs)
	framework.ExpectNoError(err, "Creating replica set %q in namespace %q", rs.Name, ns)
}
<|endoftext|>


<commit_before>package notifier

import (
	"flag"
	"os"
	"time"

	"github.com/appscode/envconfig"
	"github.com/appscode/go-notify"
	"github.com/appscode/go-notify/unified"
	"github.com/appscode/go/flags"
	"github.com/appscode/log"
	logs "github.com/appscode/log/golog"
	tapi "github.com/appscode/searchlight/api"
	"github.com/appscode/searchlight/pkg/icinga"
	"github.com/appscode/searchlight/pkg/util"
	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
)

type Request struct {
	HostName  string
	AlertName string
	Type      string
	State     string
	Output    string
	// The time object is used in icinga to send request. This
	// indicates detection time from icinga.
	Time    int64
	Author  string
	Comment string
}

type Secret struct {
	Namespace string `json:"namespace"`
	Token     string `json:"token"`
}

func getLoader(client clientset.Interface, alert tapi.Alert) (envconfig.LoaderFunc, error) {
	cfg, err := client.CoreV1().
		Secrets(util.OperatorNamespace()).
		Get(alert.GetNotifierSecretName(), metav1.GetOptions{})
	if err != nil {
		return nil, err
	}

	return func(key string) (value string, found bool) {
		var bytes []byte
		bytes, found = cfg.Data[key]
		value = string(bytes)
		return
	}, nil
}

func sendNotification(req *Request) {
	client, err := util.NewClient()
	if err != nil {
		log.Fatalln(err)
	}

	host, err := icinga.ParseHost(req.HostName)
	if err != nil {
		log.Fatalln(err)
	}

	alert, err := host.GetAlert(client.ExtClient, req.AlertName)
	if err != nil {
		log.Fatalln(err)
	}

	loader, err := getLoader(client.Client, alert)
	if err != nil {
		log.Fatalln(err)
	}

	receivers := alert.GetReceivers()

	for _, receiver := range receivers {
		if receiver.State != req.State || len(receiver.To) == 0 {
			continue
		}
		notifyVia, err := unified.LoadVia(receiver.Notifier, loader)
		if err != nil {
			log.Errorln(err)
			continue
		}

		switch n := notifyVia.(type) {
		case notify.ByEmail:
			var body string
			body, err = RenderMail(alert, req)
			if err != nil {
				log.Errorf("Failed to render email. Reason: %s", err)
				break
			}
			err = n.To(receiver.To[0], receiver.To[1:]...).
				WithSubject(RenderSubject(alert, req)).
				WithBody(body).
				WithNoTracking().
				SendHtml()
		case notify.BySMS:
			err = n.To(receiver.To[0], receiver.To[1:]...).
				WithBody(RenderSMS(alert, req)).
				Send()
		case notify.ByChat:
			err = n.To(receiver.To[0], receiver.To[1:]...).
				WithBody(RenderSMS(alert, req)).
				Send()
		case notify.ByPush:
			err = n.To(receiver.To[0:]...).
				WithBody(RenderSMS(alert, req)).
				Send()
		}

		if err != nil {
			log.Errorln(err)
		} else {
			log.Infof("Notification sent using %s", receiver.Notifier)
		}
	}
}

func NewCmd() *cobra.Command {
	var req Request
	var eventTime string

	c := &cobra.Command{
		Use:   "notifier",
		Short: "AppsCode Icinga2 Notifier",
		Run: func(cmd *cobra.Command, args []string) {
			flags.EnsureRequiredFlags(cmd, "alert", "host", "type", "state", "output", "time")
			t, err := time.Parse("2006-01-02 15:04:05 +0000", eventTime)
			if err != nil {
				log.Errorln(err)
				os.Exit(1)

			}
			req.Time = t.Unix()
			sendNotification(&req)
		},
	}

	c.Flags().StringVarP(&req.HostName, "host", "H", "", "Icinga host name")
	c.Flags().StringVarP(&req.AlertName, "alert", "A", "", "Kubernetes alert object name")
	c.Flags().StringVar(&req.Type, "type", "", "Notification type (PROBLEM | ACKNOWLEDGEMENT | RECOVERY)")
	c.Flags().StringVar(&req.State, "state", "", "Service state (OK | WARNING | CRITICAL)")
	c.Flags().StringVar(&req.Output, "output", "", "Service output")
	c.Flags().StringVar(&eventTime, "time", "", "Event time")
	c.Flags().StringVarP(&req.Author, "author", "a", "", "Event author name")
	c.Flags().StringVarP(&req.Comment, "comment", "c", "", "Event comment")

	c.Flags().AddGoFlagSet(flag.CommandLine)
	logs.InitLogs()

	return c
}
<commit_msg>Find notificaqtion secret from alert namespace (#207)<commit_after>package notifier

import (
	"flag"
	"os"
	"time"

	"github.com/appscode/envconfig"
	"github.com/appscode/go-notify"
	"github.com/appscode/go-notify/unified"
	"github.com/appscode/go/flags"
	"github.com/appscode/log"
	logs "github.com/appscode/log/golog"
	tapi "github.com/appscode/searchlight/api"
	"github.com/appscode/searchlight/pkg/icinga"
	"github.com/appscode/searchlight/pkg/util"
	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
)

type Request struct {
	HostName  string
	AlertName string
	Type      string
	State     string
	Output    string
	// The time object is used in icinga to send request. This
	// indicates detection time from icinga.
	Time    int64
	Author  string
	Comment string
}

type Secret struct {
	Namespace string `json:"namespace"`
	Token     string `json:"token"`
}

func getLoader(client clientset.Interface, alert tapi.Alert) (envconfig.LoaderFunc, error) {
	cfg, err := client.CoreV1().Secrets(alert.GetNamespace()).Get(alert.GetNotifierSecretName(), metav1.GetOptions{})
	if err != nil {
		return nil, err
	}

	return func(key string) (value string, found bool) {
		var bytes []byte
		bytes, found = cfg.Data[key]
		value = string(bytes)
		return
	}, nil
}

func sendNotification(req *Request) {
	client, err := util.NewClient()
	if err != nil {
		log.Fatalln(err)
	}

	host, err := icinga.ParseHost(req.HostName)
	if err != nil {
		log.Fatalln(err)
	}

	alert, err := host.GetAlert(client.ExtClient, req.AlertName)
	if err != nil {
		log.Fatalln(err)
	}

	loader, err := getLoader(client.Client, alert)
	if err != nil {
		log.Fatalln(err)
	}

	receivers := alert.GetReceivers()

	for _, receiver := range receivers {
		if receiver.State != req.State || len(receiver.To) == 0 {
			continue
		}
		notifyVia, err := unified.LoadVia(receiver.Notifier, loader)
		if err != nil {
			log.Errorln(err)
			continue
		}

		switch n := notifyVia.(type) {
		case notify.ByEmail:
			var body string
			body, err = RenderMail(alert, req)
			if err != nil {
				log.Errorf("Failed to render email. Reason: %s", err)
				break
			}
			err = n.To(receiver.To[0], receiver.To[1:]...).
				WithSubject(RenderSubject(alert, req)).
				WithBody(body).
				WithNoTracking().
				SendHtml()
		case notify.BySMS:
			err = n.To(receiver.To[0], receiver.To[1:]...).
				WithBody(RenderSMS(alert, req)).
				Send()
		case notify.ByChat:
			err = n.To(receiver.To[0], receiver.To[1:]...).
				WithBody(RenderSMS(alert, req)).
				Send()
		case notify.ByPush:
			err = n.To(receiver.To[0:]...).
				WithBody(RenderSMS(alert, req)).
				Send()
		}

		if err != nil {
			log.Errorln(err)
		} else {
			log.Infof("Notification sent using %s", receiver.Notifier)
		}
	}
}

func NewCmd() *cobra.Command {
	var req Request
	var eventTime string

	c := &cobra.Command{
		Use:   "notifier",
		Short: "AppsCode Icinga2 Notifier",
		Run: func(cmd *cobra.Command, args []string) {
			flags.EnsureRequiredFlags(cmd, "alert", "host", "type", "state", "output", "time")
			t, err := time.Parse("2006-01-02 15:04:05 +0000", eventTime)
			if err != nil {
				log.Errorln(err)
				os.Exit(1)

			}
			req.Time = t.Unix()
			sendNotification(&req)
		},
	}

	c.Flags().StringVarP(&req.HostName, "host", "H", "", "Icinga host name")
	c.Flags().StringVarP(&req.AlertName, "alert", "A", "", "Kubernetes alert object name")
	c.Flags().StringVar(&req.Type, "type", "", "Notification type (PROBLEM | ACKNOWLEDGEMENT | RECOVERY)")
	c.Flags().StringVar(&req.State, "state", "", "Service state (OK | WARNING | CRITICAL)")
	c.Flags().StringVar(&req.Output, "output", "", "Service output")
	c.Flags().StringVar(&eventTime, "time", "", "Event time")
	c.Flags().StringVarP(&req.Author, "author", "a", "", "Event author name")
	c.Flags().StringVarP(&req.Comment, "comment", "c", "", "Event comment")

	c.Flags().AddGoFlagSet(flag.CommandLine)
	logs.InitLogs()

	return c
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Garbage collection benchmark: parse Go packages repeatedly.

package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"os"
	"path"
	"runtime"
	"strings"
	"time"
)

func isGoFile(dir *os.FileInfo) bool {
	return dir.IsRegular() &&
		!strings.HasPrefix(dir.Name, ".") && // ignore .files
		path.Ext(dir.Name) == ".go"
}

func isPkgFile(dir *os.FileInfo) bool {
	return isGoFile(dir) &&
		!strings.HasSuffix(dir.Name, "_test.go") // ignore test files
}

func pkgName(filename string) string {
	file, err := parser.ParseFile(filename, nil, nil, parser.PackageClauseOnly)
	if err != nil || file == nil {
		return ""
	}
	return file.Name.Name()
}

func parseDir(dirpath string) map[string]*ast.Package {
	// the package name is the directory name within its parent
	// (use dirname instead of path because dirname is clean; i.e. has no trailing '/')
	_, pkgname := path.Split(dirpath)

	// filter function to select the desired .go files
	filter := func(d *os.FileInfo) bool {
		if isPkgFile(d) {
			// Some directories contain main packages: Only accept
			// files that belong to the expected package so that
			// parser.ParsePackage doesn't return "multiple packages
			// found" errors.
			// Additionally, accept the special package name
			// fakePkgName if we are looking at cmd documentation.
			name := pkgName(dirpath + "/" + d.Name)
			return name == pkgname
		}
		return false
	}

	// get package AST
	pkgs, err := parser.ParseDir(dirpath, filter, parser.ParseComments)
	if err != nil {
		println("parse", dirpath, err.String())
		panic("fail")
	}
	return pkgs
}

func main() {
	st := &runtime.MemStats
	n := flag.Int("n", 4, "iterations")
	p := flag.Int("p", len(packages), "# of packages to keep in memory")
	flag.BoolVar(&st.DebugGC, "d", st.DebugGC, "print GC debugging info (pause times)")
	flag.Parse()

	var t0 int64
	pkgroot := runtime.GOROOT() + "/src/pkg/"
	for pass := 0; pass < 2; pass++ {
		// Once the heap is grown to full size, reset counters.
		// This hides the start-up pauses, which are much smaller
		// than the normal pauses and would otherwise make
		// the average look much better than it actually is.
		st.NumGC = 0
		st.PauseNs = 0
		t0 = time.Nanoseconds()

		for i := 0; i < *n; i++ {
			parsed := make([]map[string]*ast.Package, *p)
			for j := range parsed {
				parsed[j] = parseDir(pkgroot + packages[j%len(packages)])
			}
		}
		runtime.GC()
	}
	t1 := time.Nanoseconds()

	fmt.Printf("Alloc=%d/%d Heap=%d Mallocs=%d PauseTime=%.3f/%d = %.3f\n",
		st.Alloc, st.TotalAlloc,
		st.Sys,
		st.Mallocs, float64(st.PauseNs)/1e9,
		st.NumGC, float64(st.PauseNs)/1e9/float64(st.NumGC))

	fmt.Printf("%10s %10s %10s\n", "size", "#alloc", "#free")
	for _, s := range st.BySize {
		fmt.Printf("%10d %10d %10d\n", s.Size, s.Mallocs, s.Frees)
	}

	// Standard gotest benchmark output, collected by build dashboard.
	fmt.Printf("garbage.BenchmarkParser %d %d ns/op\n", *n, (t1-t0)/int64(*n))
	fmt.Printf("garbage.BenchmarkParserPause %d %d ns/op\n", st.NumGC, int64(st.PauseNs)/int64(st.NumGC))
}

var packages = []string{
	"archive/tar",
	"asn1",
	"big",
	"bufio",
	"bytes",
	"cmath",
	"compress/flate",
	"compress/gzip",
	"compress/zlib",
	"container/heap",
	"container/list",
	"container/ring",
	"container/vector",
	"crypto/aes",
	"crypto/block",
	"crypto/blowfish",
	"crypto/hmac",
	"crypto/md4",
	"crypto/md5",
	"crypto/rand",
	"crypto/rc4",
	"crypto/rsa",
	"crypto/sha1",
	"crypto/sha256",
	"crypto/sha512",
	"crypto/subtle",
	"crypto/tls",
	"crypto/x509",
	"crypto/xtea",
	"debug/dwarf",
	"debug/macho",
	"debug/elf",
	"debug/gosym",
	"debug/proc",
	"ebnf",
	"encoding/ascii85",
	"encoding/base64",
	"encoding/binary",
	"encoding/git85",
	"encoding/hex",
	"encoding/pem",
	"exec",
	"exp/datafmt",
	"exp/draw",
	"exp/eval",
	"exp/iterable",
	"expvar",
	"flag",
	"fmt",
	"go/ast",
	"go/doc",
	"go/parser",
	"go/printer",
	"go/scanner",
	"go/token",
	"gob",
	"hash",
	"hash/adler32",
	"hash/crc32",
	"hash/crc64",
	"http",
	"image",
	"image/jpeg",
	"image/png",
	"io",
	"io/ioutil",
	"json",
	"log",
	"math",
	"mime",
	"net",
	"nntp",
	"os",
	"os/signal",
	"patch",
	"path",
	"rand",
	"reflect",
	"regexp",
	"rpc",
	"runtime",
	"scanner",
	"sort",
	"strconv",
	"strings",
	"sync",
	"syscall",
	"syslog",
	"tabwriter",
	"template",
	"testing",
	"testing/iotest",
	"testing/quick",
	"testing/script",
	"time",
	"unicode",
	"utf8",
	"utf16",
	"websocket",
	"xml",
}
<commit_msg>test/garbage/parser: sync with recent parser changes<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Garbage collection benchmark: parse Go packages repeatedly.

package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"os"
	"path"
	"runtime"
	"strings"
	"time"
)

func isGoFile(dir *os.FileInfo) bool {
	return dir.IsRegular() &&
		!strings.HasPrefix(dir.Name, ".") && // ignore .files
		path.Ext(dir.Name) == ".go"
}

func isPkgFile(dir *os.FileInfo) bool {
	return isGoFile(dir) &&
		!strings.HasSuffix(dir.Name, "_test.go") // ignore test files
}

func pkgName(filename string) string {
	file, err := parser.ParseFile(filename, nil, parser.PackageClauseOnly)
	if err != nil || file == nil {
		return ""
	}
	return file.Name.Name
}

func parseDir(dirpath string) map[string]*ast.Package {
	// the package name is the directory name within its parent
	// (use dirname instead of path because dirname is clean; i.e. has no trailing '/')
	_, pkgname := path.Split(dirpath)

	// filter function to select the desired .go files
	filter := func(d *os.FileInfo) bool {
		if isPkgFile(d) {
			// Some directories contain main packages: Only accept
			// files that belong to the expected package so that
			// parser.ParsePackage doesn't return "multiple packages
			// found" errors.
			// Additionally, accept the special package name
			// fakePkgName if we are looking at cmd documentation.
			name := pkgName(dirpath + "/" + d.Name)
			return name == pkgname
		}
		return false
	}

	// get package AST
	pkgs, err := parser.ParseDir(dirpath, filter, parser.ParseComments)
	if err != nil {
		println("parse", dirpath, err.String())
		panic("fail")
	}
	return pkgs
}

func main() {
	st := &runtime.MemStats
	n := flag.Int("n", 4, "iterations")
	p := flag.Int("p", len(packages), "# of packages to keep in memory")
	flag.BoolVar(&st.DebugGC, "d", st.DebugGC, "print GC debugging info (pause times)")
	flag.Parse()

	var t0 int64
	pkgroot := runtime.GOROOT() + "/src/pkg/"
	for pass := 0; pass < 2; pass++ {
		// Once the heap is grown to full size, reset counters.
		// This hides the start-up pauses, which are much smaller
		// than the normal pauses and would otherwise make
		// the average look much better than it actually is.
		st.NumGC = 0
		st.PauseNs = 0
		t0 = time.Nanoseconds()

		for i := 0; i < *n; i++ {
			parsed := make([]map[string]*ast.Package, *p)
			for j := range parsed {
				parsed[j] = parseDir(pkgroot + packages[j%len(packages)])
			}
		}
		runtime.GC()
	}
	t1 := time.Nanoseconds()

	fmt.Printf("Alloc=%d/%d Heap=%d Mallocs=%d PauseTime=%.3f/%d = %.3f\n",
		st.Alloc, st.TotalAlloc,
		st.Sys,
		st.Mallocs, float64(st.PauseNs)/1e9,
		st.NumGC, float64(st.PauseNs)/1e9/float64(st.NumGC))

	fmt.Printf("%10s %10s %10s\n", "size", "#alloc", "#free")
	for _, s := range st.BySize {
		fmt.Printf("%10d %10d %10d\n", s.Size, s.Mallocs, s.Frees)
	}

	// Standard gotest benchmark output, collected by build dashboard.
	fmt.Printf("garbage.BenchmarkParser %d %d ns/op\n", *n, (t1-t0)/int64(*n))
	fmt.Printf("garbage.BenchmarkParserPause %d %d ns/op\n", st.NumGC, int64(st.PauseNs)/int64(st.NumGC))
}

var packages = []string{
	"archive/tar",
	"asn1",
	"big",
	"bufio",
	"bytes",
	"cmath",
	"compress/flate",
	"compress/gzip",
	"compress/zlib",
	"container/heap",
	"container/list",
	"container/ring",
	"container/vector",
	"crypto/aes",
	"crypto/block",
	"crypto/blowfish",
	"crypto/hmac",
	"crypto/md4",
	"crypto/md5",
	"crypto/rand",
	"crypto/rc4",
	"crypto/rsa",
	"crypto/sha1",
	"crypto/sha256",
	"crypto/sha512",
	"crypto/subtle",
	"crypto/tls",
	"crypto/x509",
	"crypto/xtea",
	"debug/dwarf",
	"debug/macho",
	"debug/elf",
	"debug/gosym",
	"debug/proc",
	"ebnf",
	"encoding/ascii85",
	"encoding/base64",
	"encoding/binary",
	"encoding/git85",
	"encoding/hex",
	"encoding/pem",
	"exec",
	"exp/datafmt",
	"exp/draw",
	"exp/eval",
	"exp/iterable",
	"expvar",
	"flag",
	"fmt",
	"go/ast",
	"go/doc",
	"go/parser",
	"go/printer",
	"go/scanner",
	"go/token",
	"gob",
	"hash",
	"hash/adler32",
	"hash/crc32",
	"hash/crc64",
	"http",
	"image",
	"image/jpeg",
	"image/png",
	"io",
	"io/ioutil",
	"json",
	"log",
	"math",
	"mime",
	"net",
	"nntp",
	"os",
	"os/signal",
	"patch",
	"path",
	"rand",
	"reflect",
	"regexp",
	"rpc",
	"runtime",
	"scanner",
	"sort",
	"strconv",
	"strings",
	"sync",
	"syscall",
	"syslog",
	"tabwriter",
	"template",
	"testing",
	"testing/iotest",
	"testing/quick",
	"testing/script",
	"time",
	"unicode",
	"utf8",
	"utf16",
	"websocket",
	"xml",
}
<|endoftext|>


<commit_before>package feedloggr2

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	"html/template"

	rss "github.com/jteeuwen/go-pkg-rss"
)

// TODO: change to env var/flag instead
const TIME_BETWEEN_FEEDS = 4 // In seconds

func UpdateFeeds(c *Config) error {
	db, e := OpenSqliteDB(c.Database)
	if e != nil {
		return e
	}

	if c.Verbose {
		fmt.Println("Updating feeds...")
	}

	var feeds []*Feed
	for _, f := range c.Feeds {
		r := rss.NewWithHandlers(5, false, db, db)
		e = r.Fetch(f.Url, nil)
		if e != nil {
			fmt.Printf("Error connecting to %s: %s\n", f.Title, e)
			continue
		}

		items, e := db.GetItems(f.Url)
		if e != nil {
			fmt.Printf("Error updating %s: %s\n", f.Title, e)
			continue
		}

		if len(items) < 1 {
			// Ignore any feeds with no new items
			continue
		}

		if c.Verbose {
			fmt.Printf("Got %d new items from: %s\n", len(items), f.Title)
		}

		feeds = append(feeds, &Feed{
			Title: f.Title,
			Url:   f.Url,
			Items: items,
		})

		// Slow down the amount of requests, to ensure we won't get spam blocked.
		time.Sleep(time.Duration(TIME_BETWEEN_FEEDS) * time.Second)
	}

	if c.Verbose {
		fmt.Println("Generating page...")
	}
	funcmap := template.FuncMap{
		"date_link": func(h int, t time.Time) string {
			d := t.Add(time.Hour * time.Duration(h)).Format("2006-01-02")
			return fmt.Sprintf("%s.html", d)
		},
		"format": func(t time.Time) string {
			return t.Format("2006-01-02")
		},
	}
	t := template.Must(template.New("TemplateName").Funcs(funcmap).Parse(HTML_BODY))
	d := struct {
		Date  time.Time
		Feeds []*Feed
	}{
		Date:  Now(),
		Feeds: feeds,
	}
	file := fmt.Sprintf("%s.html", Now().Format("2006-01-02"))
	path := filepath.Join(c.OutputPath, file)
	f, e := os.Create(path)
	if e != nil {
		panic(e) // TODO
	}
	e = t.Execute(f, d)
	if e != nil {
		panic(e) // TODO
	}

	if c.Verbose {
		fmt.Println("Updating symlink...")
	}
	path = filepath.Join(c.OutputPath, "index.html")
	e = os.Remove(path)
	if e != nil {
		perr, ok := e.(*os.PathError)
		// Ignore any "no such file" errors
		// It works correctly, but "|| perr.Err == "is logically wrong. Bug?
		if ok == false || perr.Err == fmt.Errorf("no such file or directory") {
			panic(e) // TODO
		}
	}
	e = os.Symlink(file, path)
	if e != nil {
		panic(e) // TODO
	}

	if c.Verbose {
		fmt.Println("Done.")
	}
	return nil
}
<commit_msg>Set custom user agent, thanks to https://github.com/jteeuwen for merging my patch.<commit_after>package feedloggr2

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	"html/template"

	rss "github.com/jteeuwen/go-pkg-rss"
)

// TODO: change to env var/flag instead
const TIME_BETWEEN_FEEDS = 2 // In seconds

// TODO: set another global version const and use it here
const USER_AGENT = "feedloggr2/0.1"

func UpdateFeeds(c *Config) error {
	db, e := OpenSqliteDB(c.Database)
	if e != nil {
		return e
	}

	if c.Verbose {
		fmt.Println("Updating feeds...")
	}

	var feeds []*Feed
	for _, f := range c.Feeds {
		r := rss.NewWithHandlers(5, false, db, db)
		r.SetUserAgent(USER_AGENT)
		e = r.Fetch(f.Url, nil)
		if e != nil {
			fmt.Printf("Error connecting to %s: %s\n", f.Title, e)
			continue
		}

		items, e := db.GetItems(f.Url)
		if e != nil {
			fmt.Printf("Error updating %s: %s\n", f.Title, e)
			continue
		}

		if len(items) < 1 {
			// Ignore any feeds with no new items
			continue
		}

		if c.Verbose {
			fmt.Printf("Got %d new items from: %s\n", len(items), f.Title)
		}

		feeds = append(feeds, &Feed{
			Title: f.Title,
			Url:   f.Url,
			Items: items,
		})

		// Slow down the amount of requests, to ensure we won't get spam blocked.
		time.Sleep(time.Duration(TIME_BETWEEN_FEEDS) * time.Second)
	}

	if c.Verbose {
		fmt.Println("Generating page...")
	}
	funcmap := template.FuncMap{
		"date_link": func(h int, t time.Time) string {
			d := t.Add(time.Hour * time.Duration(h)).Format("2006-01-02")
			return fmt.Sprintf("%s.html", d)
		},
		"format": func(t time.Time) string {
			return t.Format("2006-01-02")
		},
	}
	t := template.Must(template.New("TemplateName").Funcs(funcmap).Parse(HTML_BODY))
	d := struct {
		Date  time.Time
		Feeds []*Feed
	}{
		Date:  Now(),
		Feeds: feeds,
	}
	file := fmt.Sprintf("%s.html", Now().Format("2006-01-02"))
	path := filepath.Join(c.OutputPath, file)
	f, e := os.Create(path)
	if e != nil {
		panic(e) // TODO
	}
	e = t.Execute(f, d)
	if e != nil {
		panic(e) // TODO
	}

	if c.Verbose {
		fmt.Println("Updating symlink...")
	}
	path = filepath.Join(c.OutputPath, "index.html")
	e = os.Remove(path)
	if e != nil {
		perr, ok := e.(*os.PathError)
		// Ignore any "no such file" errors
		// It works correctly, but "|| perr.Err == "is logically wrong. Bug?
		if ok == false || perr.Err == fmt.Errorf("no such file or directory") {
			panic(e) // TODO
		}
	}
	e = os.Symlink(file, path)
	if e != nil {
		panic(e) // TODO
	}

	if c.Verbose {
		fmt.Println("Done.")
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package zap

import (
	"runtime"
	"strings"
	"sync"

	"go.uber.org/zap/internal/bufferpool"
)

var (
	_stacktraceIgnorePrefixes = []string{
		"runtime.goexit",
		"runtime.main",
	}
	_stacktracePool = sync.Pool{
		New: func() interface{} {
			return newProgramCounters(64)
		},
	}
)

func takeStacktrace() string {
	buffer := bufferpool.Get()
	defer bufferpool.Put(buffer)
	programCounters := _stacktracePool.Get().(*programCounters)
	defer _stacktracePool.Put(programCounters)

	for {
		// Skip the call to runtime.Counters and takeStacktrace so that the
		// program counters start at the caller of takeStacktrace.
		n := runtime.Callers(2, programCounters.pcs)
		if n < cap(programCounters.pcs) {
			programCounters.pcs = programCounters.pcs[:n]
			break
		}
		// Don't put the too-short counter slice back into the pool; this lets
		// the pool adjust if we consistently take deep stacktraces.
		programCounters = newProgramCounters(len(programCounters.pcs) * 2)
	}

	i := 0
	for _, programCounter := range programCounters.pcs {
		f := runtime.FuncForPC(programCounter)
		name := f.Name()
		if shouldIgnoreStacktraceName(name) {
			continue
		}
		if i != 0 {
			buffer.AppendByte('\n')
		}
		i++
		file, line := f.FileLine(programCounter - 1)
		buffer.AppendString(name)
		buffer.AppendByte('\n')
		buffer.AppendByte('\t')
		buffer.AppendString(file)
		buffer.AppendByte(':')
		buffer.AppendInt(int64(line))
	}

	return buffer.String()
}

func shouldIgnoreStacktraceName(name string) bool {
	for _, prefix := range _stacktraceIgnorePrefixes {
		if strings.HasPrefix(name, prefix) {
			return true
		}
	}
	return false
}

type programCounters struct {
	pcs []uintptr
}

func newProgramCounters(size int) *programCounters {
	return &programCounters{make([]uintptr, size)}
}
<commit_msg>Use runtime.CallersFrames (#365)<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package zap

import (
	"runtime"
	"strings"
	"sync"

	"go.uber.org/zap/internal/bufferpool"
)

var (
	_stacktraceIgnorePrefixes = []string{
		"runtime.goexit",
		"runtime.main",
	}
	_stacktracePool = sync.Pool{
		New: func() interface{} {
			return newProgramCounters(64)
		},
	}
)

func takeStacktrace() string {
	buffer := bufferpool.Get()
	defer bufferpool.Put(buffer)
	programCounters := _stacktracePool.Get().(*programCounters)
	defer _stacktracePool.Put(programCounters)

	for {
		// Skip the call to runtime.Counters and takeStacktrace so that the
		// program counters start at the caller of takeStacktrace.
		n := runtime.Callers(2, programCounters.pcs)
		if n < cap(programCounters.pcs) {
			programCounters.pcs = programCounters.pcs[:n]
			break
		}
		// Don't put the too-short counter slice back into the pool; this lets
		// the pool adjust if we consistently take deep stacktraces.
		programCounters = newProgramCounters(len(programCounters.pcs) * 2)
	}

	i := 0
	frames := runtime.CallersFrames(programCounters.pcs)
	for frame, more := frames.Next(); more; frame, more = frames.Next() {
		if shouldIgnoreStacktraceFunction(frame.Function) {
			continue
		}
		if i != 0 {
			buffer.AppendByte('\n')
		}
		i++
		buffer.AppendString(frame.Function)
		buffer.AppendByte('\n')
		buffer.AppendByte('\t')
		buffer.AppendString(frame.File)
		buffer.AppendByte(':')
		buffer.AppendInt(int64(frame.Line))
	}

	return buffer.String()
}

func shouldIgnoreStacktraceFunction(function string) bool {
	for _, prefix := range _stacktraceIgnorePrefixes {
		if strings.HasPrefix(function, prefix) {
			return true
		}
	}
	return false
}

type programCounters struct {
	pcs []uintptr
}

func newProgramCounters(size int) *programCounters {
	return &programCounters{make([]uintptr, size)}
}
<|endoftext|>


<commit_before>package supervisor

import (
	"context"
	"fmt"
	"log"
	"sync"
	"time"
)

type service struct {
	svc     Service
	svctype ServiceType
}

type processFailure func()

// AlwaysRestart adjusts the supervisor to never halt in face of failures.
const AlwaysRestart = -1

// ServiceType defines the restart strategy for a service.
type ServiceType int

const (
	// Permanent services are always restarted
	Permanent ServiceType = iota
	// Transient services are restarted only when panic.
	Transient
	// Temporary services are never restarted.
	Temporary
)

// Service is the public interface expected by a Supervisor.
//
// This will be internally named after the result of fmt.Stringer, if available.
// Otherwise it will going to use an internal representation for the service
// name.
type Service interface {
	// Serve is called by a Supervisor to start the service. It expects the
	// service to honor the passed context and its lifetime. Observe
	// <-ctx.Done() and ctx.Err(). If the service is stopped by anything
	// but the Supervisor, it will get started again. Be careful with shared
	// state among restarts.
	Serve(ctx context.Context)
}

// Supervisor is the basic datastructure responsible for offering a supervisor
// tree. It implements Service, therefore it can be nested if necessary. When
// passing the Supervisor around, remind to do it as reference (&supervisor).
type Supervisor struct {
	// Name for this supervisor tree, used for logging.
	Name string

	// MaxRestarts is the number of maximum restarts given MaxTime. If more
	// than MaxRestarts occur in the last MaxTime, then the supervisor
	// stops all services and halts. Set this to AlwaysRestart to prevent
	// supervisor halt.
	MaxRestarts int

	// MaxTime is the time period on which the internal restart count will
	// be reset.
	MaxTime time.Duration

	// Log is a replaceable function used for overall logging.
	// Default: log.Printf.
	Log func(interface{})

	// indicates that supervisor is ready for use.
	prepared sync.Once

	// signals that a new service has just been added, so the started
	// supervisor picks it up.
	added chan struct{}

	// indicates that supervisor has running services.
	running         sync.Mutex
	runningServices sync.WaitGroup

	mu           sync.Mutex
	svcorder     []string                      // order in which services must be started
	services     map[string]service            // added services
	cancelations map[string]context.CancelFunc // each service cancelation
	terminations map[string]context.CancelFunc // each service termination call
	lastRestart  time.Time
	restarts     int
}

func (s *Supervisor) prepare() {
	s.prepared.Do(s.reset)
}

func (s *Supervisor) reset() {
	s.mu.Lock()
	if s.Name == "" {
		s.Name = "supervisor"
	}
	if s.MaxRestarts == 0 {
		s.MaxRestarts = 5
	}
	if s.MaxTime == 0 {
		s.MaxTime = 15 * time.Second
	}
	if s.Log == nil {
		s.Log = func(msg interface{}) {
			log.Printf("%s: %v", s.Name, msg)
		}
	}

	s.added = make(chan struct{})
	s.cancelations = make(map[string]context.CancelFunc)
	s.services = make(map[string]service)
	s.terminations = make(map[string]context.CancelFunc)
	s.mu.Unlock()
}

func (s *Supervisor) shouldRestart() bool {
	if s.MaxRestarts == AlwaysRestart {
		return true
	}

	s.mu.Lock()
	defer s.mu.Unlock()
	if time.Since(s.lastRestart) > s.MaxTime {
		s.restarts = 0
	}
	s.lastRestart = time.Now()
	s.restarts++
	return s.restarts < s.MaxRestarts
}

// Cancelations return a list of services names and their cancelation calls.
// These calls be used to force a service restart.
func (s *Supervisor) Cancelations() map[string]context.CancelFunc {
	svclist := make(map[string]context.CancelFunc)
	s.mu.Lock()
	for k, v := range s.cancelations {
		svclist[k] = v
	}
	s.mu.Unlock()
	return svclist
}

// Add inserts into the Supervisor tree a new permanent service. If the
// Supervisor is already started, it will start it automatically.
func (s *Supervisor) Add(service Service) {
	s.AddService(service, Permanent)
}

// AddFunc inserts into the Supervisor tree a new permanent anonymous service.
// If the Supervisor is already started, it will start it automatically.
func (s *Supervisor) AddFunc(f func(context.Context)) {
	s.AddService(newAnonymousService(f), Permanent)
}

// AddService inserts into the Supervisor tree a new service of ServiceType. If
// the Supervisor is already started, it will start it automatically. If the
// same service is added more than once, it will reset its backoff mechanism and
// force a service restart.
func (s *Supervisor) AddService(svc Service, svctype ServiceType) {
	s.prepare()

	name := fmt.Sprintf("%s", svc)
	s.mu.Lock()
	s.services[name] = service{
		svc:     svc,
		svctype: svctype,
	}
	s.svcorder = append(s.svcorder, name)
	s.mu.Unlock()

	go func() {
		s.added <- struct{}{}
	}()
}

// Remove stops the service in the Supervisor tree and remove from it.
func (s *Supervisor) Remove(name string) {
	s.prepare()

	s.mu.Lock()
	defer s.mu.Unlock()
	if _, ok := s.services[name]; !ok {
		return
	}

	delete(s.services, name)

	for i, n := range s.svcorder {
		if name == n {
			s.svcorder = append(s.svcorder[:i], s.svcorder[i+1:]...)
			break
		}
	}

	if c, ok := s.terminations[name]; ok {
		delete(s.terminations, name)
		c()
	}

	if _, ok := s.cancelations[name]; ok {
		delete(s.cancelations, name)
	}
}

// Serve starts the Supervisor tree. It can be started only once at a time. If
// stopped (canceled), it can be restarted. In case of concurrent calls, it will
// hang until the current call is completed.
func (s *Supervisor) Serve(ctx context.Context) {
	s.prepare()
	restartCtx, cancel := context.WithCancel(ctx)
	processFailure := func() {
		restart := s.shouldRestart()
		if !restart {
			cancel()
		}
	}
	serve(s, restartCtx, processFailure)
}

// Services return a list of services
func (s *Supervisor) Services() map[string]Service {
	svclist := make(map[string]Service)
	s.mu.Lock()
	for k, v := range s.services {
		svclist[k] = v.svc
	}
	s.mu.Unlock()
	return svclist
}

func (s *Supervisor) String() string {
	return s.Name
}
<commit_msg>Fix english mistake<commit_after>package supervisor

import (
	"context"
	"fmt"
	"log"
	"sync"
	"time"
)

type service struct {
	svc     Service
	svctype ServiceType
}

type processFailure func()

// AlwaysRestart adjusts the supervisor to never halt in face of failures.
const AlwaysRestart = -1

// ServiceType defines the restart strategy for a service.
type ServiceType int

const (
	// Permanent services are always restarted
	Permanent ServiceType = iota
	// Transient services are restarted only when panic.
	Transient
	// Temporary services are never restarted.
	Temporary
)

// Service is the public interface expected by a Supervisor.
//
// This will be internally named after the result of fmt.Stringer, if available.
// Otherwise it is going to use an internal representation for the service
// name.
type Service interface {
	// Serve is called by a Supervisor to start the service. It expects the
	// service to honor the passed context and its lifetime. Observe
	// <-ctx.Done() and ctx.Err(). If the service is stopped by anything
	// but the Supervisor, it will get started again. Be careful with shared
	// state among restarts.
	Serve(ctx context.Context)
}

// Supervisor is the basic datastructure responsible for offering a supervisor
// tree. It implements Service, therefore it can be nested if necessary. When
// passing the Supervisor around, remind to do it as reference (&supervisor).
type Supervisor struct {
	// Name for this supervisor tree, used for logging.
	Name string

	// MaxRestarts is the number of maximum restarts given MaxTime. If more
	// than MaxRestarts occur in the last MaxTime, then the supervisor
	// stops all services and halts. Set this to AlwaysRestart to prevent
	// supervisor halt.
	MaxRestarts int

	// MaxTime is the time period on which the internal restart count will
	// be reset.
	MaxTime time.Duration

	// Log is a replaceable function used for overall logging.
	// Default: log.Printf.
	Log func(interface{})

	// indicates that supervisor is ready for use.
	prepared sync.Once

	// signals that a new service has just been added, so the started
	// supervisor picks it up.
	added chan struct{}

	// indicates that supervisor has running services.
	running         sync.Mutex
	runningServices sync.WaitGroup

	mu           sync.Mutex
	svcorder     []string                      // order in which services must be started
	services     map[string]service            // added services
	cancelations map[string]context.CancelFunc // each service cancelation
	terminations map[string]context.CancelFunc // each service termination call
	lastRestart  time.Time
	restarts     int
}

func (s *Supervisor) prepare() {
	s.prepared.Do(s.reset)
}

func (s *Supervisor) reset() {
	s.mu.Lock()
	if s.Name == "" {
		s.Name = "supervisor"
	}
	if s.MaxRestarts == 0 {
		s.MaxRestarts = 5
	}
	if s.MaxTime == 0 {
		s.MaxTime = 15 * time.Second
	}
	if s.Log == nil {
		s.Log = func(msg interface{}) {
			log.Printf("%s: %v", s.Name, msg)
		}
	}

	s.added = make(chan struct{})
	s.cancelations = make(map[string]context.CancelFunc)
	s.services = make(map[string]service)
	s.terminations = make(map[string]context.CancelFunc)
	s.mu.Unlock()
}

func (s *Supervisor) shouldRestart() bool {
	if s.MaxRestarts == AlwaysRestart {
		return true
	}

	s.mu.Lock()
	defer s.mu.Unlock()
	if time.Since(s.lastRestart) > s.MaxTime {
		s.restarts = 0
	}
	s.lastRestart = time.Now()
	s.restarts++
	return s.restarts < s.MaxRestarts
}

// Cancelations return a list of services names and their cancelation calls.
// These calls be used to force a service restart.
func (s *Supervisor) Cancelations() map[string]context.CancelFunc {
	svclist := make(map[string]context.CancelFunc)
	s.mu.Lock()
	for k, v := range s.cancelations {
		svclist[k] = v
	}
	s.mu.Unlock()
	return svclist
}

// Add inserts into the Supervisor tree a new permanent service. If the
// Supervisor is already started, it will start it automatically.
func (s *Supervisor) Add(service Service) {
	s.AddService(service, Permanent)
}

// AddFunc inserts into the Supervisor tree a new permanent anonymous service.
// If the Supervisor is already started, it will start it automatically.
func (s *Supervisor) AddFunc(f func(context.Context)) {
	s.AddService(newAnonymousService(f), Permanent)
}

// AddService inserts into the Supervisor tree a new service of ServiceType. If
// the Supervisor is already started, it will start it automatically. If the
// same service is added more than once, it will reset its backoff mechanism and
// force a service restart.
func (s *Supervisor) AddService(svc Service, svctype ServiceType) {
	s.prepare()

	name := fmt.Sprintf("%s", svc)
	s.mu.Lock()
	s.services[name] = service{
		svc:     svc,
		svctype: svctype,
	}
	s.svcorder = append(s.svcorder, name)
	s.mu.Unlock()

	go func() {
		s.added <- struct{}{}
	}()
}

// Remove stops the service in the Supervisor tree and remove from it.
func (s *Supervisor) Remove(name string) {
	s.prepare()

	s.mu.Lock()
	defer s.mu.Unlock()
	if _, ok := s.services[name]; !ok {
		return
	}

	delete(s.services, name)

	for i, n := range s.svcorder {
		if name == n {
			s.svcorder = append(s.svcorder[:i], s.svcorder[i+1:]...)
			break
		}
	}

	if c, ok := s.terminations[name]; ok {
		delete(s.terminations, name)
		c()
	}

	if _, ok := s.cancelations[name]; ok {
		delete(s.cancelations, name)
	}
}

// Serve starts the Supervisor tree. It can be started only once at a time. If
// stopped (canceled), it can be restarted. In case of concurrent calls, it will
// hang until the current call is completed.
func (s *Supervisor) Serve(ctx context.Context) {
	s.prepare()
	restartCtx, cancel := context.WithCancel(ctx)
	processFailure := func() {
		restart := s.shouldRestart()
		if !restart {
			cancel()
		}
	}
	serve(s, restartCtx, processFailure)
}

// Services return a list of services
func (s *Supervisor) Services() map[string]Service {
	svclist := make(map[string]Service)
	s.mu.Lock()
	for k, v := range s.services {
		svclist[k] = v.svc
	}
	s.mu.Unlock()
	return svclist
}

func (s *Supervisor) String() string {
	return s.Name
}
<|endoftext|>


<commit_before>package copier

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/juju/errors"
)

type (
	// A Supervisor is batch copier handler.
	Supervisor struct {
		context.Context
		base          string
		logger        *Logger
		summary       map[string]int
		FilePaths     []string
		Destination   string
		ExecTimeout   time.Duration
		Progress      chan ProgressInfo
		Speed         float64
		Retries       int
		RetryInterval time.Duration
	}

	// ProgressInfo handle data for building a progress bar.
	ProgressInfo struct {
		Name        string
		Size        int64
		ProxyReader *ProxyReader
		Status      string
	}
)

// NewSupervisor returns a new Supervisor.
func NewSupervisor(paths []string, destination string) (*Supervisor, error) {
	base, err := GetBaseDirectory(paths)
	ctx := context.Background()
	return &Supervisor{
		Context: ctx,
		base:    base,
		logger:  NewLoggerWithContext(ctx),
		summary: map[string]int{
			StatusAlreadyExist: 0,
			StatusOverwritten:  0,
			StatusCopied:       0,
			StatusFailed:       0,
		},
		FilePaths:     paths,
		Destination:   destination,
		ExecTimeout:   10 * time.Minute,
		Progress:      make(chan ProgressInfo),
		Speed:         float64(512 * KBps),
		Retries:       5,
		RetryInterval: 2 * time.Second,
	}, err
}

// Logger returns the Supervisor logger.
func (s *Supervisor) Logger() *Logger {
	return s.logger
}

// Summary returns the summary of an execution.
func (s *Supervisor) Summary() map[string]int {
	return s.summary
}

// Execute runs the Supervisor copy batch.
func (s *Supervisor) Execute() error {
	fmt.Println("Start copy")
	fmt.Println("  - Speed:", int64(s.Speed), "Bps")
	fmt.Println("  - Timeout:", s.ExecTimeout)
	fmt.Println("  - Retries:", s.Retries)
	fmt.Println("  - Retry Interval:", s.RetryInterval)

	for _, from := range s.FilePaths {
		to := strings.Replace(from, s.base, s.Destination, 1)

		if err := s.execute(from, to, s.Retries, nil); err != nil {
			// Ignore returned error; go to next file.
			s.logger.C <- LogEntry{
				Severity:    ERROR,
				Status:      StatusFailed,
				FilepathSrc: from,
				FilepathDst: err.Error(),
			}
		}
	}
	return nil
}

func (s *Supervisor) execute(from, to string, retries int, err2 error) error {
	if retries == 0 {
		return err2 // keep last error
	}

	errc := make(chan error)
	var err error

	ctx, cancel := context.WithTimeout(s.Context, s.ExecTimeout)
	defer cancel()

	cp := NewExecWithContext(ctx, from, to, s.Speed)
	cp.Speed = s.Speed
	go func() {
		if err = cp.Execute(); err != nil {
			errc <- errors.Annotate(err, "Exec#Execute")
		}
		errc <- nil
	}()

	ready := false
	terminated := false
	for {
		select {
		case <-cp.Ready:
			s.Progress <- ProgressInfo{
				Name:        cp.Name(),
				Size:        cp.Size(),
				ProxyReader: cp.Reader(),
				Status:      cp.Status(),
			}
			ready = true
		case <-ctx.Done():
			if ctx.Err() != nil {
				time.Sleep(s.RetryInterval)
				err = s.execute(from, to, retries-1, ctx.Err())
			}

			terminated = true
		case err = <-errc:
			if err != nil {
				time.Sleep(s.RetryInterval)
				err = s.execute(from, to, retries-1, err)
			}

			terminated = true
		}

		if terminated {
			if !ready {
				s.Progress <- ProgressInfo{
					Name:        cp.Name(),
					Size:        cp.Size(),
					ProxyReader: cp.Reader(),
					Status:      StatusFailed,
				}
			}

			if err = s.logsOrReturnError(cp, err); err != nil {
				return err
			}
			break
		}
	}
	return nil
}

func (s *Supervisor) logsOrReturnError(cp *Exec, err error) error {
	// Succeed
	if err == nil {
		s.summary[cp.Status()]++
		s.logger.C <- LogEntry{
			Severity:    INFO,
			Status:      cp.Status(),
			FilepathSrc: cp.From,
			FilepathDst: cp.To,
		}
		return nil
	}
	// Failed
	s.summary[StatusFailed]++
	if err == context.DeadlineExceeded || strings.HasSuffix(err.Error(), "would exceed context deadline") { // first is ctx.Err() and the second is a shapeio.Reder error
		s.logger.C <- LogEntry{
			Severity:    ERROR,
			Status:      StatusFailed,
			FilepathSrc: cp.From,
			FilepathDst: cp.To,
		}
		return nil
	}

	// Fatal error
	return err
}
<commit_msg>Fix file status in the terminal<commit_after>package copier

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/juju/errors"
)

type (
	// A Supervisor is batch copier handler.
	Supervisor struct {
		context.Context
		base          string
		logger        *Logger
		summary       map[string]int
		FilePaths     []string
		Destination   string
		ExecTimeout   time.Duration
		Progress      chan ProgressInfo
		Speed         float64
		Retries       int
		RetryInterval time.Duration
	}

	// ProgressInfo handle data for building a progress bar.
	ProgressInfo struct {
		Name        string
		Size        int64
		ProxyReader *ProxyReader
		Status      string
	}
)

// NewSupervisor returns a new Supervisor.
func NewSupervisor(paths []string, destination string) (*Supervisor, error) {
	base, err := GetBaseDirectory(paths)
	ctx := context.Background()
	return &Supervisor{
		Context: ctx,
		base:    base,
		logger:  NewLoggerWithContext(ctx),
		summary: map[string]int{
			StatusAlreadyExist: 0,
			StatusOverwritten:  0,
			StatusCopied:       0,
			StatusFailed:       0,
		},
		FilePaths:     paths,
		Destination:   destination,
		ExecTimeout:   10 * time.Minute,
		Progress:      make(chan ProgressInfo),
		Speed:         float64(512 * KBps),
		Retries:       5,
		RetryInterval: 2 * time.Second,
	}, err
}

// Logger returns the Supervisor logger.
func (s *Supervisor) Logger() *Logger {
	return s.logger
}

// Summary returns the summary of an execution.
func (s *Supervisor) Summary() map[string]int {
	return s.summary
}

// Execute runs the Supervisor copy batch.
func (s *Supervisor) Execute() error {
	fmt.Println("Start copy")
	fmt.Println("  - Speed:", int64(s.Speed), "Bps")
	fmt.Println("  - Timeout:", s.ExecTimeout)
	fmt.Println("  - Retries:", s.Retries)
	fmt.Println("  - Retry Interval:", s.RetryInterval)

	for _, from := range s.FilePaths {
		to := strings.Replace(from, s.base, s.Destination, 1)

		if err := s.execute(from, to, s.Retries, nil); err != nil {
			// Ignore returned error; go to next file.
			s.logger.C <- LogEntry{
				Severity:    ERROR,
				Status:      StatusFailed,
				FilepathSrc: from,
				FilepathDst: err.Error(),
			}
		}
	}
	return nil
}

func (s *Supervisor) execute(from, to string, retries int, err2 error) error {
	if retries == 0 {
		return err2 // keep last error
	}

	errc := make(chan error)
	var err error

	ctx, cancel := context.WithTimeout(s.Context, s.ExecTimeout)
	defer cancel()

	cp := NewExecWithContext(ctx, from, to, s.Speed)
	cp.Speed = s.Speed
	go func() {
		if err = cp.Execute(); err != nil {
			errc <- errors.Annotate(err, "Exec#Execute")
		}
		errc <- nil
	}()

	ready := false
	terminated := false
	for {
		select {
		case <-cp.Ready:
			s.Progress <- ProgressInfo{
				Name:        cp.Name(),
				Size:        cp.Size(),
				ProxyReader: cp.Reader(),
				Status:      cp.Status(),
			}
			ready = true
		case <-ctx.Done():
			if ctx.Err() != nil {
				time.Sleep(s.RetryInterval)
				err = s.execute(from, to, retries-1, ctx.Err())
			}

			terminated = true
		case err = <-errc:
			if err != nil {
				time.Sleep(s.RetryInterval)
				err = s.execute(from, to, retries-1, err)
			}

			terminated = true
		}

		if terminated {
			if !ready && err == nil {
				// Already exist
				s.Progress <- ProgressInfo{
					Name:        cp.Name(),
					Size:        cp.Size(),
					ProxyReader: cp.Reader(),
					Status:      cp.Status(),
				}
			}

			if err != nil {
				// failed
				s.Progress <- ProgressInfo{
					Name:        cp.Name(),
					Size:        cp.Size(),
					ProxyReader: cp.Reader(),
					Status:      StatusFailed,
				}
			}

			if err = s.logsOrReturnError(cp, err); err != nil {
				return err
			}
			break
		}
	}
	return nil
}

func (s *Supervisor) logsOrReturnError(cp *Exec, err error) error {
	// Succeed
	if err == nil {
		s.summary[cp.Status()]++
		s.logger.C <- LogEntry{
			Severity:    INFO,
			Status:      cp.Status(),
			FilepathSrc: cp.From,
			FilepathDst: cp.To,
		}
		return nil
	}
	// Failed
	s.summary[StatusFailed]++
	if err == context.DeadlineExceeded || strings.HasSuffix(err.Error(), "would exceed context deadline") { // first is ctx.Err() and the second is a shapeio.Reder error
		s.logger.C <- LogEntry{
			Severity:    ERROR,
			Status:      StatusFailed,
			FilepathSrc: cp.From,
			FilepathDst: cp.To,
		}
		return nil
	}

	// Fatal error
	return err
}
<|endoftext|>


<commit_before>package tablib

import (
	"bytes"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	typePostgres = "postgres"
	typeMySQL    = "mysql"
)

// columnSQLType determines the type of a column
// if throughout the whole column values have the same type then this type is
// returned, otherwise the VARCHAR/TEXT type is returned.
// numeric types are coerced into DOUBLE/NUMERIC
func (d *Dataset) columnSQLType(header, dbType string) (string, []interface{}) {
	types := 0
	currentType := ""
	maxString := 0
	values := d.Column(header)
	for _, c := range values {
		switch c.(type) {
		case uint, uint8, uint16, uint32, uint64,
			int, int8, int16, int32, int64,
			float32, float64:
			if currentType != "numeric" {
				currentType = "numeric"
				types++
			}
		case time.Time:
			if currentType != "time" {
				currentType = "time"
				types++
			}
		case string:
			if currentType != "string" {
				currentType = "string"
				types++
			}
			if len(c.(string)) > maxString {
				maxString = len(c.(string))
			}
		}
	}

	if types > 1 {
		if dbType == typePostgres {
			return "TEXT", values
		}
		return "VARCHAR(100)", values
	} else {
		switch currentType {
		case "numeric":
			if dbType == typePostgres {
				return "NUMERIC", values
			}
			return "DOUBLE", values
		case "time":
			return "TIMESTAMP", values
		default:
			if dbType == typePostgres {
				return "TEXT", values
			}
			return "VARCHAR(" + strconv.Itoa(maxString) + ")", values
		}
	}
}

// isStringColumn returns whether a column is VARCHAR/TEXT
func isStringColumn(c string) bool {
	return strings.HasPrefix(c, "VARCHAR") || strings.HasPrefix(c, "TEXT")
}

// MySQL returns a string representing a suite of MySQL commands
// recreating the Dataset into a table.
func (d *Dataset) MySQL(table string) string {
	return d.sql(table, typeMySQL)
}

// Postgres returns a string representing a suite of Postgres commands
// recreating the Dataset into a table.
func (d *Dataset) Postgres(table string) string {
	return d.sql(table, typePostgres)
}

// sql returns a string representing a suite of SQL commands
// recreating the Dataset into a table.
func (d *Dataset) sql(table, dbType string) string {
	var b bytes.Buffer

	columnValues := make(map[string][]interface{})
	columnTypes := make(map[string]string)

	// create table
	b.WriteString("CREATE TABLE IF NOT EXISTS " + table)
	if dbType == typePostgres {
		b.WriteString("\n(\n\tid SERIAL PRIMARY KEY,\n")
	} else {
		b.WriteString("\n(\n\tid INT NOT NULL AUTO_INCREMENT PRIMARY KEY,\n")
	}
	for i, h := range d.headers {
		b.WriteString("\t" + h)
		t, v := d.columnSQLType(h, dbType)
		columnValues[h] = v
		columnTypes[h] = t
		b.WriteString(" " + t)
		if i < len(d.headers)-1 {
			b.WriteString(",")
		}
		b.WriteString("\n")
	}

	b.WriteString(");\n\n")

	reg, _ := regexp.Compile("[']")
	// inserts
	for i := range d.data {
		b.WriteString("INSERT INTO " + table + " VALUES(" + strconv.Itoa(i+1) + ", ")
		for j, col := range d.headers {
			asStr := d.asString(columnValues[col][i])
			if isStringColumn(columnTypes[col]) {
				b.WriteString("'" + reg.ReplaceAllString(asStr, "''") + "'")
			} else {
				b.WriteString(asStr)
			}
			if j < len(d.headers)-1 {
				b.WriteString(", ")
			}
		}
		b.WriteString(");\n")
	}
	b.WriteString("\nCOMMIT;\n")

	return b.String()
}
<commit_msg>MySQL, Postgres<commit_after>package tablib

import (
	"bytes"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	typePostgres = "postgres"
	typeMySQL    = "mysql"
)

// columnSQLType determines the type of a column
// if throughout the whole column values have the same type then this type is
// returned, otherwise the VARCHAR/TEXT type is returned.
// numeric types are coerced into DOUBLE/NUMERIC
func (d *Dataset) columnSQLType(header, dbType string) (string, []interface{}) {
	types := 0
	currentType := ""
	maxString := 0
	values := d.Column(header)
	for _, c := range values {
		switch c.(type) {
		case uint, uint8, uint16, uint32, uint64,
			int, int8, int16, int32, int64,
			float32, float64:
			if currentType != "numeric" {
				currentType = "numeric"
				types++
			}
		case time.Time:
			if currentType != "time" {
				currentType = "time"
				types++
			}
		case string:
			if currentType != "string" {
				currentType = "string"
				types++
			}
			if len(c.(string)) > maxString {
				maxString = len(c.(string))
			}
		}
	}

	if types > 1 {
		if dbType == typePostgres {
			return "TEXT", values
		}
		return "VARCHAR(100)", values
	}
	switch currentType {
	case "numeric":
		if dbType == typePostgres {
			return "NUMERIC", values
		}
		return "DOUBLE", values
	case "time":
		return "TIMESTAMP", values
	default:
		if dbType == typePostgres {
			return "TEXT", values
		}
		return "VARCHAR(" + strconv.Itoa(maxString) + ")", values
	}
}

// isStringColumn returns whether a column is VARCHAR/TEXT
func isStringColumn(c string) bool {
	return strings.HasPrefix(c, "VARCHAR") || strings.HasPrefix(c, "TEXT")
}

// MySQL returns a string representing a suite of MySQL commands
// recreating the Dataset into a table.
func (d *Dataset) MySQL(table string) string {
	return d.sql(table, typeMySQL)
}

// Postgres returns a string representing a suite of Postgres commands
// recreating the Dataset into a table.
func (d *Dataset) Postgres(table string) string {
	return d.sql(table, typePostgres)
}

// sql returns a string representing a suite of SQL commands
// recreating the Dataset into a table.
func (d *Dataset) sql(table, dbType string) string {
	var b bytes.Buffer

	columnValues := make(map[string][]interface{})
	columnTypes := make(map[string]string)

	// create table
	b.WriteString("CREATE TABLE IF NOT EXISTS " + table)
	if dbType == typePostgres {
		b.WriteString("\n(\n\tid SERIAL PRIMARY KEY,\n")
	} else {
		b.WriteString("\n(\n\tid INT NOT NULL AUTO_INCREMENT PRIMARY KEY,\n")
	}
	for i, h := range d.headers {
		b.WriteString("\t" + h)
		t, v := d.columnSQLType(h, dbType)
		columnValues[h] = v
		columnTypes[h] = t
		b.WriteString(" " + t)
		if i < len(d.headers)-1 {
			b.WriteString(",")
		}
		b.WriteString("\n")
	}

	b.WriteString(");\n\n")

	reg, _ := regexp.Compile("[']")
	// inserts
	for i := range d.data {
		b.WriteString("INSERT INTO " + table + " VALUES(" + strconv.Itoa(i+1) + ", ")
		for j, col := range d.headers {
			asStr := d.asString(columnValues[col][i])
			if isStringColumn(columnTypes[col]) {
				b.WriteString("'" + reg.ReplaceAllString(asStr, "''") + "'")
			} else {
				b.WriteString(asStr)
			}
			if j < len(d.headers)-1 {
				b.WriteString(", ")
			}
		}
		b.WriteString(");\n")
	}
	b.WriteString("\nCOMMIT;\n")

	return b.String()
}
<|endoftext|>


<commit_before>// +build task

package main

import (
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
)

func sourceDir() (dir string, ok bool) {
	_, file, _, ok := runtime.Caller(1)
	if !ok {
		return "", ok
	}
	dir = filepath.Dir(file)

	// if the last segment is "task", strip that out; allows
	// segregating task files in a subdir
	parent, taskDir := filepath.Split(dir)
	if taskDir == "task" {
		dir = parent
	}

	return dir, ok
}

func goBuild(src string, action string, args ...string) error {
	cmd := exec.Command(
		"go", action,
		"-v",
	)
	cmd.Args = append(cmd.Args, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

func gitVersion() (string, error) {
	cmd := exec.Command(
		"git", "describe",
		"--match", "release/*",
		"--dirty=-edited",
	)
	cmd.Stderr = os.Stderr
	buf, err := cmd.Output()
	if err != nil {
		return "", err
	}
	return string(buf), nil
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("# ")
	src, ok := sourceDir()
	if !ok {
		log.Fatal("cannot determine source directory")
	}
	err := os.Chdir(src)
	if err != nil {
		log.Fatalf("cannot change to source directory: %v", err)
	}

	log.Print("build bazil")
	version, err := gitVersion()
	if err != nil {
		log.Fatalf("git describe: %v", err)
	}
	err = goBuild(src, "build", "-v",
		"-ldflags", "-X bazil.org/bazil/version.Version "+version,
		"bazil.org/bazil",
	)
	if err != nil {
		log.Fatalf("go build of bazil: %v", err)
	}
}
<commit_msg>task: New go build -X syntax<commit_after>// +build task

package main

import (
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
)

func sourceDir() (dir string, ok bool) {
	_, file, _, ok := runtime.Caller(1)
	if !ok {
		return "", ok
	}
	dir = filepath.Dir(file)

	// if the last segment is "task", strip that out; allows
	// segregating task files in a subdir
	parent, taskDir := filepath.Split(dir)
	if taskDir == "task" {
		dir = parent
	}

	return dir, ok
}

func goBuild(src string, action string, args ...string) error {
	cmd := exec.Command(
		"go", action,
		"-v",
	)
	cmd.Args = append(cmd.Args, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

func gitVersion() (string, error) {
	cmd := exec.Command(
		"git", "describe",
		"--match", "release/*",
		"--dirty=-edited",
	)
	cmd.Stderr = os.Stderr
	buf, err := cmd.Output()
	if err != nil {
		return "", err
	}
	return string(buf), nil
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("# ")
	src, ok := sourceDir()
	if !ok {
		log.Fatal("cannot determine source directory")
	}
	err := os.Chdir(src)
	if err != nil {
		log.Fatalf("cannot change to source directory: %v", err)
	}

	log.Print("build bazil")
	version, err := gitVersion()
	if err != nil {
		log.Fatalf("git describe: %v", err)
	}
	err = goBuild(src, "build", "-v",
		"-ldflags", "-X bazil.org/bazil/version.Version="+version,
		"bazil.org/bazil",
	)
	if err != nil {
		log.Fatalf("go build of bazil: %v", err)
	}
}
<|endoftext|>


<commit_before>package config

// DeviceNamed contains the name of a device and its config.
type DeviceNamed struct {
	Name   string
	Config Device
}

// DevicesSortable is a sortable slice of device names and config.
type DevicesSortable []DeviceNamed

func (devices DevicesSortable) Len() int {
	return len(devices)
}

func (devices DevicesSortable) Less(i, j int) bool {
	a := devices[i]
	b := devices[j]

	// First sort by types.
	if a.Config["type"] != b.Config["type"] {
		// In VMs, network interface names are derived from PCI
		// location. As a result of that, we must ensure that nic devices will
		// always show up at the same spot regardless of what other devices may be
		// added. Easiest way to do this is to always have them show up first.
		if a.Config["type"] == "nic" {
			return true
		}

		if b.Config["type"] == "nic" {
			return false
		}

		return a.Config["type"] < b.Config["type"]
	}

	// Special case disk paths.
	if a.Config["type"] == "disk" && b.Config["type"] == "disk" {
		if a.Config["path"] != b.Config["path"] {
			return a.Config["path"] < b.Config["path"]
		}
	}

	// Fallback to sorting by names.
	return a.Name < b.Name
}

func (devices DevicesSortable) Swap(i, j int) {
	devices[i], devices[j] = devices[j], devices[i]
}
<commit_msg>lxd/device/config/devices/sort: Sort disks between nics and other types of devices<commit_after>package config

// DeviceNamed contains the name of a device and its config.
type DeviceNamed struct {
	Name   string
	Config Device
}

// DevicesSortable is a sortable slice of device names and config.
type DevicesSortable []DeviceNamed

func (devices DevicesSortable) Len() int {
	return len(devices)
}

func (devices DevicesSortable) Less(i, j int) bool {
	a := devices[i]
	b := devices[j]

	// First sort by types.
	if a.Config["type"] != b.Config["type"] {
		// In VMs, network interface names are derived from PCI
		// location. As a result of that, we must ensure that nic devices will
		// always show up at the same spot regardless of what other devices may be
		// added. Easiest way to do this is to always have them show up first.
		if a.Config["type"] == "nic" {
			return true
		}

		if b.Config["type"] == "nic" {
			return false
		}

		// Start disks before other non-nic devices so that any unmounts triggered by pre-start resize
		// occur first and the rest of the devices can rely on the instance's root disk being mounted.
		if a.Config["type"] == "disk" {
			return true
		}

		if b.Config["type"] == "disk" {
			return false
		}

		// Otherwise start devices of same type together.
		return a.Config["type"] > b.Config["type"]
	}

	// Special case disk paths.
	if a.Config["type"] == "disk" && b.Config["type"] == "disk" {
		if a.Config["path"] != b.Config["path"] {
			return a.Config["path"] < b.Config["path"]
		}
	}

	// Fallback to sorting by names.
	return a.Name < b.Name
}

func (devices DevicesSortable) Swap(i, j int) {
	devices[i], devices[j] = devices[j], devices[i]
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// do a fast type check of kubernetes code, for all platforms.
package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"go/types"
	"io"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"golang.org/x/crypto/ssh/terminal"

	srcimporter "k8s.io/kubernetes/third_party/go-srcimporter"
)

var (
	verbose    = flag.Bool("verbose", false, "print more information")
	cross      = flag.Bool("cross", true, "build for all platforms")
	platforms  = flag.String("platform", "", "comma-separated list of platforms to typecheck")
	timings    = flag.Bool("time", false, "output times taken for each phase")
	defuses    = flag.Bool("defuse", false, "output defs/uses")
	serial     = flag.Bool("serial", false, "don't type check platforms in parallel")
	skipTest   = flag.Bool("skip-test", false, "don't type check test code")
	tags       = flag.String("tags", "", "comma-separated list of build tags to apply in addition to go's defaults")
	ignoreDirs = flag.String("ignore-dirs", "", "comma-separated list of directories to ignore in addition to the default hardcoded list including staging, vendor, and hidden dirs")

	isTerminal = terminal.IsTerminal(int(os.Stdout.Fd()))
	logPrefix  = ""

	// When processed in order, windows and darwin are early to make
	// interesting OS-based errors happen earlier.
	crossPlatforms = []string{
		"linux/amd64", "windows/386",
		"darwin/amd64", "linux/arm",
		"linux/386", "windows/amd64",
		"linux/arm64", "linux/ppc64le",
		"linux/s390x", "darwin/386",
	}
	darwinPlatString  = "darwin/386,darwin/amd64"
	windowsPlatString = "windows/386,windows/amd64"

	// directories we always ignore
	standardIgnoreDirs = []string{
		// Staging code is symlinked from vendor/k8s.io, and uses import
		// paths as if it were inside of vendor/. It fails typechecking
		// inside of staging/, but works when typechecked as part of vendor/.
		"staging",
		// OS-specific vendor code tends to be imported by OS-specific
		// packages. We recursively typecheck imported vendored packages for
		// each OS, but don't typecheck everything for every OS.
		"vendor",
		"_output",
		// This is a weird one. /testdata/ is *mostly* ignored by Go,
		// and this translates to kubernetes/vendor not working.
		// edit/record.go doesn't compile without gopkg.in/yaml.v2
		// in $GOSRC/$GOROOT (both typecheck and the shell script).
		"pkg/kubectl/cmd/testdata/edit",
		// Tools we use for maintaining the code base but not necessarily
		// ship as part of the release
		"hack/tools",
	}
)

type analyzer struct {
	fset      *token.FileSet // positions are relative to fset
	conf      types.Config
	ctx       build.Context
	failed    bool
	platform  string
	donePaths map[string]interface{}
	errors    []string
}

func newAnalyzer(platform string) *analyzer {
	ctx := build.Default
	platSplit := strings.Split(platform, "/")
	ctx.GOOS, ctx.GOARCH = platSplit[0], platSplit[1]
	ctx.CgoEnabled = true
	if *tags != "" {
		tagsSplit := strings.Split(*tags, ",")
		ctx.BuildTags = append(ctx.BuildTags, tagsSplit...)
	}

	// add selinux tag explicitly
	ctx.BuildTags = append(ctx.BuildTags, "selinux")

	a := &analyzer{
		platform:  platform,
		fset:      token.NewFileSet(),
		ctx:       ctx,
		donePaths: make(map[string]interface{}),
	}
	a.conf = types.Config{
		FakeImportC: true,
		Error:       a.handleError,
		Sizes:       types.SizesFor("gc", a.ctx.GOARCH),
	}

	a.conf.Importer = srcimporter.New(
		&a.ctx, a.fset, make(map[string]*types.Package))

	if *verbose {
		fmt.Printf("context: %#v\n", ctx)
	}

	return a
}

func (a *analyzer) handleError(err error) {
	a.errors = append(a.errors, err.Error())
	if *serial {
		fmt.Fprintf(os.Stderr, "%sERROR(%s) %s\n", logPrefix, a.platform, err)
	}
	a.failed = true
}

func (a *analyzer) dumpAndResetErrors() []string {
	es := a.errors
	a.errors = nil
	return es
}

// collect extracts test metadata from a file.
func (a *analyzer) collect(dir string) {
	if _, ok := a.donePaths[dir]; ok {
		return
	}
	a.donePaths[dir] = nil

	// Create the AST by parsing src.
	fs, err := parser.ParseDir(a.fset, dir, nil, parser.AllErrors)

	if err != nil {
		fmt.Println(logPrefix+"ERROR(syntax)", err)
		a.failed = true
		return
	}

	if len(fs) > 1 && *verbose {
		fmt.Println("multiple packages in dir:", dir)
	}

	for _, p := range fs {
		// returns first error, but a.handleError deals with it
		files := a.filterFiles(p.Files)
		if *verbose {
			fmt.Printf("path: %s package: %s files: ", dir, p.Name)
			for _, f := range files {
				fname := filepath.Base(a.fset.File(f.Pos()).Name())
				fmt.Printf("%s ", fname)
			}
			fmt.Printf("\n")
		}
		a.typeCheck(dir, files)
	}
}

// filterFiles restricts a list of files to only those that should be built by
// the current platform. This includes both build suffixes (_windows.go) and build
// tags ("// +build !linux" at the beginning).
func (a *analyzer) filterFiles(fs map[string]*ast.File) []*ast.File {
	files := []*ast.File{}
	for _, f := range fs {
		fpath := a.fset.File(f.Pos()).Name()
		if *skipTest && strings.HasSuffix(fpath, "_test.go") {
			continue
		}
		dir, name := filepath.Split(fpath)
		matches, err := a.ctx.MatchFile(dir, name)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%sERROR reading %s: %s\n", logPrefix, fpath, err)
			a.failed = true
			continue
		}
		if matches {
			files = append(files, f)
		}
	}
	return files
}

func (a *analyzer) typeCheck(dir string, files []*ast.File) error {
	info := types.Info{
		Defs: make(map[*ast.Ident]types.Object),
		Uses: make(map[*ast.Ident]types.Object),
	}

	// NOTE: this type check does a *recursive* import, but srcimporter
	// doesn't do a full type check (ignores function bodies)-- this has
	// some additional overhead.
	//
	// This means that we need to ensure that typeCheck runs on all
	// code we will be compiling.
	//
	// TODO(rmmh): Customize our forked srcimporter to do this better.
	pkg, err := a.conf.Check(dir, a.fset, files, &info)
	if err != nil {
		return err // type error
	}

	// A significant fraction of vendored code only compiles on Linux,
	// but it's only imported by code that has build-guards for Linux.
	// Track vendored code to type-check it in a second pass.
	for _, imp := range pkg.Imports() {
		if strings.HasPrefix(imp.Path(), "k8s.io/kubernetes/vendor/") {
			vendorPath := imp.Path()[len("k8s.io/kubernetes/"):]
			if *verbose {
				fmt.Println("recursively checking vendor path:", vendorPath)
			}
			a.collect(vendorPath)
		}
	}

	if *defuses {
		for id, obj := range info.Defs {
			fmt.Printf("%s: %q defines %v\n",
				a.fset.Position(id.Pos()), id.Name, obj)
		}
		for id, obj := range info.Uses {
			fmt.Printf("%s: %q uses %v\n",
				a.fset.Position(id.Pos()), id.Name, obj)
		}
	}

	return nil
}

type collector struct {
	dirs       []string
	ignoreDirs []string
}

// handlePath walks the filesystem recursively, collecting directories,
// ignoring some unneeded directories (hidden/vendored) that are handled
// specially later.
func (c *collector) handlePath(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if info.IsDir() {
		// Ignore hidden directories (.git, .cache, etc)
		if len(path) > 1 && path[0] == '.' {
			return filepath.SkipDir
		}
		for _, dir := range c.ignoreDirs {
			if path == dir {
				return filepath.SkipDir
			}
		}
		c.dirs = append(c.dirs, path)
	}
	return nil
}

type analyzerResult struct {
	platform string
	dir      string
	errors   []string
}

func dedupeErrors(out io.Writer, results chan analyzerResult, nDirs, nPlatforms int) {
	pkgRes := make(map[string][]analyzerResult)
	for done := 0; done < nDirs; {
		res := <-results
		pkgRes[res.dir] = append(pkgRes[res.dir], res)
		if len(pkgRes[res.dir]) != nPlatforms {
			continue // expect more results for dir
		}
		done++
		// Collect list of platforms for each error
		errPlats := map[string][]string{}
		for _, res := range pkgRes[res.dir] {
			for _, err := range res.errors {
				errPlats[err] = append(errPlats[err], res.platform)
			}
		}
		// Print each error (in the same order!) once.
		for _, res := range pkgRes[res.dir] {
			for _, err := range res.errors {
				if errPlats[err] == nil {
					continue // already printed
				}
				sort.Strings(errPlats[err])
				plats := strings.Join(errPlats[err], ",")
				if len(errPlats[err]) == len(crossPlatforms) {
					plats = "all"
				} else if plats == darwinPlatString {
					plats = "darwin"
				} else if plats == windowsPlatString {
					plats = "windows"
				}
				fmt.Fprintf(out, "%sERROR(%s) %s\n", logPrefix, plats, err)
				delete(errPlats, err)
			}
		}
		delete(pkgRes, res.dir)
	}
}

func main() {
	flag.Parse()
	args := flag.Args()

	if *verbose {
		*serial = true // to avoid confusing interleaved logs
	}

	if len(args) == 0 {
		args = append(args, ".")
	}

	c := collector{
		ignoreDirs: append([]string(nil), standardIgnoreDirs...),
	}
	if *ignoreDirs != "" {
		c.ignoreDirs = append(c.ignoreDirs, strings.Split(*ignoreDirs, ",")...)
	}

	for _, arg := range args {
		err := filepath.Walk(arg, c.handlePath)
		if err != nil {
			log.Fatalf("Error walking: %v", err)
		}
	}
	sort.Strings(c.dirs)

	ps := crossPlatforms[:]
	if *platforms != "" {
		ps = strings.Split(*platforms, ",")
	} else if !*cross {
		ps = ps[:1]
	}

	fmt.Println("type-checking: ", strings.Join(ps, ", "))

	var wg sync.WaitGroup
	var processedDirs int64
	var currentWork int64 // (dir_index << 8) | platform_index
	statuses := make([]int, len(ps))
	var results chan analyzerResult
	if !*serial {
		results = make(chan analyzerResult)
		wg.Add(1)
		go func() {
			dedupeErrors(os.Stderr, results, len(c.dirs), len(ps))
			wg.Done()
		}()
	}
	for i, p := range ps {
		wg.Add(1)
		fn := func(i int, p string) {
			start := time.Now()
			a := newAnalyzer(p)
			for n, dir := range c.dirs {
				a.collect(dir)
				atomic.AddInt64(&processedDirs, 1)
				atomic.StoreInt64(&currentWork, int64(n<<8|i))
				if results != nil {
					results <- analyzerResult{p, dir, a.dumpAndResetErrors()}
				}
			}
			if a.failed {
				statuses[i] = 1
			}
			if *timings {
				fmt.Printf("%s took %.1fs\n", p, time.Since(start).Seconds())
			}
			wg.Done()
		}
		if *serial {
			fn(i, p)
		} else {
			go fn(i, p)
		}
	}
	if isTerminal {
		logPrefix = "\r" // clear status bar when printing
		// Display a status bar so devs can estimate completion times.
		wg.Add(1)
		go func() {
			total := len(ps) * len(c.dirs)
			for proc := 0; ; proc = int(atomic.LoadInt64(&processedDirs)) {
				work := atomic.LoadInt64(&currentWork)
				dir := c.dirs[work>>8]
				platform := ps[work&0xFF]
				if len(dir) > 80 {
					dir = dir[:80]
				}
				fmt.Printf("\r%d/%d \033[2m%-13s\033[0m %-80s", proc, total, platform, dir)
				if proc == total {
					fmt.Println()
					break
				}
				time.Sleep(50 * time.Millisecond)
			}
			wg.Done()
		}()
	}
	wg.Wait()
	for _, status := range statuses {
		if status != 0 {
			os.Exit(status)
		}
	}
}
<commit_msg>Don't typecheck directories that start with _<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// do a fast type check of kubernetes code, for all platforms.
package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"go/types"
	"io"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"golang.org/x/crypto/ssh/terminal"

	srcimporter "k8s.io/kubernetes/third_party/go-srcimporter"
)

var (
	verbose    = flag.Bool("verbose", false, "print more information")
	cross      = flag.Bool("cross", true, "build for all platforms")
	platforms  = flag.String("platform", "", "comma-separated list of platforms to typecheck")
	timings    = flag.Bool("time", false, "output times taken for each phase")
	defuses    = flag.Bool("defuse", false, "output defs/uses")
	serial     = flag.Bool("serial", false, "don't type check platforms in parallel")
	skipTest   = flag.Bool("skip-test", false, "don't type check test code")
	tags       = flag.String("tags", "", "comma-separated list of build tags to apply in addition to go's defaults")
	ignoreDirs = flag.String("ignore-dirs", "", "comma-separated list of directories to ignore in addition to the default hardcoded list including staging, vendor, and hidden dirs")

	isTerminal = terminal.IsTerminal(int(os.Stdout.Fd()))
	logPrefix  = ""

	// When processed in order, windows and darwin are early to make
	// interesting OS-based errors happen earlier.
	crossPlatforms = []string{
		"linux/amd64", "windows/386",
		"darwin/amd64", "linux/arm",
		"linux/386", "windows/amd64",
		"linux/arm64", "linux/ppc64le",
		"linux/s390x", "darwin/386",
	}
	darwinPlatString  = "darwin/386,darwin/amd64"
	windowsPlatString = "windows/386,windows/amd64"

	// directories we always ignore
	standardIgnoreDirs = []string{
		// Staging code is symlinked from vendor/k8s.io, and uses import
		// paths as if it were inside of vendor/. It fails typechecking
		// inside of staging/, but works when typechecked as part of vendor/.
		"staging",
		// OS-specific vendor code tends to be imported by OS-specific
		// packages. We recursively typecheck imported vendored packages for
		// each OS, but don't typecheck everything for every OS.
		"vendor",
		"_output",
		// This is a weird one. /testdata/ is *mostly* ignored by Go,
		// and this translates to kubernetes/vendor not working.
		// edit/record.go doesn't compile without gopkg.in/yaml.v2
		// in $GOSRC/$GOROOT (both typecheck and the shell script).
		"pkg/kubectl/cmd/testdata/edit",
		// Tools we use for maintaining the code base but not necessarily
		// ship as part of the release
		"hack/tools",
	}
)

type analyzer struct {
	fset      *token.FileSet // positions are relative to fset
	conf      types.Config
	ctx       build.Context
	failed    bool
	platform  string
	donePaths map[string]interface{}
	errors    []string
}

func newAnalyzer(platform string) *analyzer {
	ctx := build.Default
	platSplit := strings.Split(platform, "/")
	ctx.GOOS, ctx.GOARCH = platSplit[0], platSplit[1]
	ctx.CgoEnabled = true
	if *tags != "" {
		tagsSplit := strings.Split(*tags, ",")
		ctx.BuildTags = append(ctx.BuildTags, tagsSplit...)
	}

	// add selinux tag explicitly
	ctx.BuildTags = append(ctx.BuildTags, "selinux")

	a := &analyzer{
		platform:  platform,
		fset:      token.NewFileSet(),
		ctx:       ctx,
		donePaths: make(map[string]interface{}),
	}
	a.conf = types.Config{
		FakeImportC: true,
		Error:       a.handleError,
		Sizes:       types.SizesFor("gc", a.ctx.GOARCH),
	}

	a.conf.Importer = srcimporter.New(
		&a.ctx, a.fset, make(map[string]*types.Package))

	if *verbose {
		fmt.Printf("context: %#v\n", ctx)
	}

	return a
}

func (a *analyzer) handleError(err error) {
	a.errors = append(a.errors, err.Error())
	if *serial {
		fmt.Fprintf(os.Stderr, "%sERROR(%s) %s\n", logPrefix, a.platform, err)
	}
	a.failed = true
}

func (a *analyzer) dumpAndResetErrors() []string {
	es := a.errors
	a.errors = nil
	return es
}

// collect extracts test metadata from a file.
func (a *analyzer) collect(dir string) {
	if _, ok := a.donePaths[dir]; ok {
		return
	}
	a.donePaths[dir] = nil

	// Create the AST by parsing src.
	fs, err := parser.ParseDir(a.fset, dir, nil, parser.AllErrors)

	if err != nil {
		fmt.Println(logPrefix+"ERROR(syntax)", err)
		a.failed = true
		return
	}

	if len(fs) > 1 && *verbose {
		fmt.Println("multiple packages in dir:", dir)
	}

	for _, p := range fs {
		// returns first error, but a.handleError deals with it
		files := a.filterFiles(p.Files)
		if *verbose {
			fmt.Printf("path: %s package: %s files: ", dir, p.Name)
			for _, f := range files {
				fname := filepath.Base(a.fset.File(f.Pos()).Name())
				fmt.Printf("%s ", fname)
			}
			fmt.Printf("\n")
		}
		a.typeCheck(dir, files)
	}
}

// filterFiles restricts a list of files to only those that should be built by
// the current platform. This includes both build suffixes (_windows.go) and build
// tags ("// +build !linux" at the beginning).
func (a *analyzer) filterFiles(fs map[string]*ast.File) []*ast.File {
	files := []*ast.File{}
	for _, f := range fs {
		fpath := a.fset.File(f.Pos()).Name()
		if *skipTest && strings.HasSuffix(fpath, "_test.go") {
			continue
		}
		dir, name := filepath.Split(fpath)
		matches, err := a.ctx.MatchFile(dir, name)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%sERROR reading %s: %s\n", logPrefix, fpath, err)
			a.failed = true
			continue
		}
		if matches {
			files = append(files, f)
		}
	}
	return files
}

func (a *analyzer) typeCheck(dir string, files []*ast.File) error {
	info := types.Info{
		Defs: make(map[*ast.Ident]types.Object),
		Uses: make(map[*ast.Ident]types.Object),
	}

	// NOTE: this type check does a *recursive* import, but srcimporter
	// doesn't do a full type check (ignores function bodies)-- this has
	// some additional overhead.
	//
	// This means that we need to ensure that typeCheck runs on all
	// code we will be compiling.
	//
	// TODO(rmmh): Customize our forked srcimporter to do this better.
	pkg, err := a.conf.Check(dir, a.fset, files, &info)
	if err != nil {
		return err // type error
	}

	// A significant fraction of vendored code only compiles on Linux,
	// but it's only imported by code that has build-guards for Linux.
	// Track vendored code to type-check it in a second pass.
	for _, imp := range pkg.Imports() {
		if strings.HasPrefix(imp.Path(), "k8s.io/kubernetes/vendor/") {
			vendorPath := imp.Path()[len("k8s.io/kubernetes/"):]
			if *verbose {
				fmt.Println("recursively checking vendor path:", vendorPath)
			}
			a.collect(vendorPath)
		}
	}

	if *defuses {
		for id, obj := range info.Defs {
			fmt.Printf("%s: %q defines %v\n",
				a.fset.Position(id.Pos()), id.Name, obj)
		}
		for id, obj := range info.Uses {
			fmt.Printf("%s: %q uses %v\n",
				a.fset.Position(id.Pos()), id.Name, obj)
		}
	}

	return nil
}

type collector struct {
	dirs       []string
	ignoreDirs []string
}

// handlePath walks the filesystem recursively, collecting directories,
// ignoring some unneeded directories (hidden/vendored) that are handled
// specially later.
func (c *collector) handlePath(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if info.IsDir() {
		// Ignore hidden directories (.git, .cache, etc)
		if len(path) > 1 && (path[0] == '.' || path[0] == '_') {
			return filepath.SkipDir
		}
		for _, dir := range c.ignoreDirs {
			if path == dir {
				return filepath.SkipDir
			}
		}
		c.dirs = append(c.dirs, path)
	}
	return nil
}

type analyzerResult struct {
	platform string
	dir      string
	errors   []string
}

func dedupeErrors(out io.Writer, results chan analyzerResult, nDirs, nPlatforms int) {
	pkgRes := make(map[string][]analyzerResult)
	for done := 0; done < nDirs; {
		res := <-results
		pkgRes[res.dir] = append(pkgRes[res.dir], res)
		if len(pkgRes[res.dir]) != nPlatforms {
			continue // expect more results for dir
		}
		done++
		// Collect list of platforms for each error
		errPlats := map[string][]string{}
		for _, res := range pkgRes[res.dir] {
			for _, err := range res.errors {
				errPlats[err] = append(errPlats[err], res.platform)
			}
		}
		// Print each error (in the same order!) once.
		for _, res := range pkgRes[res.dir] {
			for _, err := range res.errors {
				if errPlats[err] == nil {
					continue // already printed
				}
				sort.Strings(errPlats[err])
				plats := strings.Join(errPlats[err], ",")
				if len(errPlats[err]) == len(crossPlatforms) {
					plats = "all"
				} else if plats == darwinPlatString {
					plats = "darwin"
				} else if plats == windowsPlatString {
					plats = "windows"
				}
				fmt.Fprintf(out, "%sERROR(%s) %s\n", logPrefix, plats, err)
				delete(errPlats, err)
			}
		}
		delete(pkgRes, res.dir)
	}
}

func main() {
	flag.Parse()
	args := flag.Args()

	if *verbose {
		*serial = true // to avoid confusing interleaved logs
	}

	if len(args) == 0 {
		args = append(args, ".")
	}

	c := collector{
		ignoreDirs: append([]string(nil), standardIgnoreDirs...),
	}
	if *ignoreDirs != "" {
		c.ignoreDirs = append(c.ignoreDirs, strings.Split(*ignoreDirs, ",")...)
	}

	for _, arg := range args {
		err := filepath.Walk(arg, c.handlePath)
		if err != nil {
			log.Fatalf("Error walking: %v", err)
		}
	}
	sort.Strings(c.dirs)

	ps := crossPlatforms[:]
	if *platforms != "" {
		ps = strings.Split(*platforms, ",")
	} else if !*cross {
		ps = ps[:1]
	}

	fmt.Println("type-checking: ", strings.Join(ps, ", "))

	var wg sync.WaitGroup
	var processedDirs int64
	var currentWork int64 // (dir_index << 8) | platform_index
	statuses := make([]int, len(ps))
	var results chan analyzerResult
	if !*serial {
		results = make(chan analyzerResult)
		wg.Add(1)
		go func() {
			dedupeErrors(os.Stderr, results, len(c.dirs), len(ps))
			wg.Done()
		}()
	}
	for i, p := range ps {
		wg.Add(1)
		fn := func(i int, p string) {
			start := time.Now()
			a := newAnalyzer(p)
			for n, dir := range c.dirs {
				a.collect(dir)
				atomic.AddInt64(&processedDirs, 1)
				atomic.StoreInt64(&currentWork, int64(n<<8|i))
				if results != nil {
					results <- analyzerResult{p, dir, a.dumpAndResetErrors()}
				}
			}
			if a.failed {
				statuses[i] = 1
			}
			if *timings {
				fmt.Printf("%s took %.1fs\n", p, time.Since(start).Seconds())
			}
			wg.Done()
		}
		if *serial {
			fn(i, p)
		} else {
			go fn(i, p)
		}
	}
	if isTerminal {
		logPrefix = "\r" // clear status bar when printing
		// Display a status bar so devs can estimate completion times.
		wg.Add(1)
		go func() {
			total := len(ps) * len(c.dirs)
			for proc := 0; ; proc = int(atomic.LoadInt64(&processedDirs)) {
				work := atomic.LoadInt64(&currentWork)
				dir := c.dirs[work>>8]
				platform := ps[work&0xFF]
				if len(dir) > 80 {
					dir = dir[:80]
				}
				fmt.Printf("\r%d/%d \033[2m%-13s\033[0m %-80s", proc, total, platform, dir)
				if proc == total {
					fmt.Println()
					break
				}
				time.Sleep(50 * time.Millisecond)
			}
			wg.Done()
		}()
	}
	wg.Wait()
	for _, status := range statuses {
		if status != 0 {
			os.Exit(status)
		}
	}
}
<|endoftext|>