<commit_before>package main_test

import (
	"fmt"
	"os"
	"syscall"
	"time"

	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"

	"github.com/cloudfoundry-incubator/converger/converger_runner"
	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
)

var _ = Describe("Main", func() {
	var (
		etcdRunner *etcdstorerunner.ETCDClusterRunner
		bbs        *Bbs.BBS
		runner     *converger_runner.ConvergerRunner

		fileServerPresence ifrit.Process

		taskKickInterval = 1 * time.Second

		etcdClient storeadapter.StoreAdapter

		convergeRepeatInterval = time.Second
	)

	SynchronizedBeforeSuite(func() []byte {
		convergerBinPath, err := gexec.Build("github.com/cloudfoundry-incubator/converger", "-race")
		Ω(err).ShouldNot(HaveOccurred())
		return []byte(convergerBinPath)
	}, func(convergerBinPath []byte) {
		etcdPort := 5001 + config.GinkgoConfig.ParallelNode
		etcdCluster := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
		etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1)

		etcdClient = etcdRunner.Adapter()
		bbs = Bbs.NewBBS(etcdClient, timeprovider.NewTimeProvider(), lagertest.NewTestLogger("test"))

		runner = converger_runner.New(string(convergerBinPath), etcdCluster, "info")
	})

	SynchronizedAfterSuite(func() {
		etcdRunner.Stop()
	}, func() {
		gexec.CleanupBuildArtifacts()
	})

	BeforeEach(func() {
		etcdRunner.Start()

		fileServerPresence = ifrit.Envoke(bbs.NewFileServerHeartbeat("http://some.file.server", "file-server-id", time.Second))

		executorPresence := models.ExecutorPresence{
			ExecutorID: "the-executor-id",
			Stack:      "the-stack",
		}

		etcdClient.Create(storeadapter.StoreNode{
			Key:   shared.ExecutorSchemaPath(executorPresence.ExecutorID),
			Value: executorPresence.ToJSON(),
		})
	})

	AfterEach(func() {
		fileServerPresence.Signal(os.Interrupt)
		Eventually(fileServerPresence.Wait()).Should(Receive(BeNil()))

		runner.KillWithFire()
		etcdRunner.Stop()
	})

	startConverger := func() {
		runner.Start(convergeRepeatInterval, taskKickInterval, 30*time.Minute, 30*time.Second, 300*time.Second)
		time.Sleep(convergeRepeatInterval)
	}

	createClaimedTaskWithDeadExecutor := func() {
		task := models.Task{
			Domain: "tests",

			Guid:  "task-guid",
			Stack: "stack",
			Actions: []models.ExecutorAction{
				{
					Action: models.RunAction{
						Path: "cat",
						Args: []string{"/tmp/file"},
					},
				},
			},
		}

		err := bbs.DesireTask(task)
		Ω(err).ShouldNot(HaveOccurred())

		err = bbs.ClaimTask(task.Guid, "dead-executor")
		Ω(err).ShouldNot(HaveOccurred())
	}

	desireLRP := func() {
		err := bbs.DesireLRP(models.DesiredLRP{
			Domain: "tests",

			ProcessGuid: "the-guid",

			Stack: "some-stack",

			Instances: 3,
			MemoryMB:  128,
			DiskMB:    512,

			Actions: []models.ExecutorAction{
				{
					Action: models.RunAction{
						Path: "the-start-command",
					},
				},
			},
		})
		Ω(err).ShouldNot(HaveOccurred())
	}

	itIsInactive := func() {
		Describe("when an LRP is desired", func() {
			JustBeforeEach(desireLRP)

			It("does not create start auctions for apps that are missing instances", func() {
				Consistently(bbs.GetAllLRPStartAuctions, 0.5).Should(BeEmpty())
			})
		})

		Context("when a claimed task with a dead executor is present", func() {
			JustBeforeEach(createClaimedTaskWithDeadExecutor)

			It("does not change the task", func() {
				Consistently(bbs.GetAllCompletedTasks, taskKickInterval*2).Should(BeEmpty())
			})
		})
	}

	Context("when the converger has the lock", func() {
		BeforeEach(startConverger)

		Describe("when an LRP is desired", func() {
			JustBeforeEach(desireLRP)

			Context("for an app that is not running at all", func() {
				It("desires N start auctions in the BBS", func() {
					Eventually(bbs.GetAllLRPStartAuctions, 0.5).Should(HaveLen(3))
				})
			})

			Context("for an app that is missing instances", func() {
				BeforeEach(func() {
					bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "a",
						Index:        0,
					}, "the-executor-id")
				})

				It("start auctions for the missing instances", func() {
					Eventually(bbs.GetAllLRPStartAuctions, 0.5).Should(HaveLen(2))
					auctions, err := bbs.GetAllLRPStartAuctions()
					Ω(err).ShouldNot(HaveOccurred())

					indices := []int{auctions[0].Index, auctions[1].Index}
					Ω(indices).Should(ContainElement(1))
					Ω(indices).Should(ContainElement(2))

					Consistently(bbs.GetAllLRPStartAuctions).Should(HaveLen(2))
				})
			})

			Context("for an app that has extra instances", func() {
				BeforeEach(func() {
					bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "a",
						Index:        0,
					}, "the-executor-id")

					bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "b",
						Index:        1,
					}, "the-executor-id")

					bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "c",
						Index:        2,
					}, "the-executor-id")

					bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "d-extra",
						Index:        3,
					}, "the-executor-id")
				})

				It("stops the extra instances", func() {
					Consistently(bbs.GetAllLRPStartAuctions, 0.5).Should(BeEmpty())
					Eventually(bbs.GetAllStopLRPInstances).Should(HaveLen(1))
					stopInstances, err := bbs.GetAllStopLRPInstances()
					Ω(err).ShouldNot(HaveOccurred())

					Ω(stopInstances[0].ProcessGuid).Should(Equal("the-guid"))
					Ω(stopInstances[0].Index).Should(Equal(3))
					Ω(stopInstances[0].InstanceGuid).Should(Equal("d-extra"))
				})
			})
		})

		Describe("when a claimed task with a dead executor is present", func() {
			JustBeforeEach(createClaimedTaskWithDeadExecutor)

			It("eventually marks the task as failed", func() {
				Eventually(bbs.GetAllCompletedTasks, taskKickInterval*2).Should(HaveLen(1))
				tasks, err := bbs.GetAllTasks()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(tasks).Should(HaveLen(1))
				Ω(tasks[0].State).Should(Equal(models.TaskStateCompleted))
				Ω(tasks[0].Failed).Should(BeTrue())
			})
		})
	})

	Context("when the converger loses the lock", func() {
		BeforeEach(func() {
			startConverger()
			err := etcdClient.Update(storeadapter.StoreNode{
				Key:   shared.LockSchemaPath("converge_lock"),
				Value: []byte("something-else"),
			})
			Ω(err).ShouldNot(HaveOccurred())

			time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
		})

		itIsInactive()

		It("exits with an error", func() {
			Eventually(runner.Session.ExitCode).Should(Equal(1))
		})
	})

	Context("when the converger initially does not have the lock", func() {
		BeforeEach(func() {
			err := etcdClient.Create(storeadapter.StoreNode{
				Key:   shared.LockSchemaPath("converge_lock"),
				Value: []byte("something-else"),
			})
			Ω(err).ShouldNot(HaveOccurred())

			startConverger()
		})

		itIsInactive()

		Describe("when the lock becomes available", func() {
			BeforeEach(func() {
				err := etcdClient.Delete(shared.LockSchemaPath("converge_lock"))
				Ω(err).ShouldNot(HaveOccurred())

				time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
			})

			Describe("when an LRP is desired", func() {
				JustBeforeEach(desireLRP)

				Context("for an app that is not running at all", func() {
					It("desires N start auctions in the BBS", func() {
						Eventually(bbs.GetAllLRPStartAuctions, 0.5).Should(HaveLen(3))
					})
				})
			})

			Describe("when a claimed task with a dead executor is present", func() {
				JustBeforeEach(createClaimedTaskWithDeadExecutor)

				It("eventually marks the task as failed", func() {
					Eventually(bbs.GetAllCompletedTasks, taskKickInterval*2).Should(HaveLen(1))
				})
			})
		})
	})

	Describe("signal handling", func() {
		BeforeEach(func() {
			startConverger()
		})

		Describe("when it receives SIGINT", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGINT)
				Eventually(runner.Session, 4).Should(gexec.Exit(0))
			})
		})

		Describe("when it receives SIGTERM", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGTERM)
				Eventually(runner.Session, 4).Should(gexec.Exit(0))
			})
		})
	})
})
<commit_msg>Reorder the startup of Converger, DesiredLRP and LRPRunning<commit_after>package main_test

import (
	"fmt"
	"os"
	"syscall"
	"time"

	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"

	"github.com/cloudfoundry-incubator/converger/converger_runner"
	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
)

var _ = Describe("Main", func() {
	var (
		etcdRunner *etcdstorerunner.ETCDClusterRunner
		bbs        *Bbs.BBS
		runner     *converger_runner.ConvergerRunner

		fileServerPresence ifrit.Process

		taskKickInterval = 1 * time.Second

		etcdClient storeadapter.StoreAdapter

		convergeRepeatInterval = time.Second
	)

	SynchronizedBeforeSuite(func() []byte {
		convergerBinPath, err := gexec.Build("github.com/cloudfoundry-incubator/converger", "-race")
		Ω(err).ShouldNot(HaveOccurred())
		return []byte(convergerBinPath)
	}, func(convergerBinPath []byte) {
		etcdPort := 5001 + config.GinkgoConfig.ParallelNode
		etcdCluster := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
		etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1)

		etcdClient = etcdRunner.Adapter()
		bbs = Bbs.NewBBS(etcdClient, timeprovider.NewTimeProvider(), lagertest.NewTestLogger("test"))

		runner = converger_runner.New(string(convergerBinPath), etcdCluster, "info")
	})

	SynchronizedAfterSuite(func() {
		etcdRunner.Stop()
	}, func() {
		gexec.CleanupBuildArtifacts()
	})

	BeforeEach(func() {
		etcdRunner.Start()

		fileServerPresence = ifrit.Envoke(bbs.NewFileServerHeartbeat("http://some.file.server", "file-server-id", time.Second))

		executorPresence := models.ExecutorPresence{
			ExecutorID: "the-executor-id",
			Stack:      "the-stack",
		}

		etcdClient.Create(storeadapter.StoreNode{
			Key:   shared.ExecutorSchemaPath(executorPresence.ExecutorID),
			Value: executorPresence.ToJSON(),
		})
	})

	AfterEach(func() {
		fileServerPresence.Signal(os.Interrupt)
		Eventually(fileServerPresence.Wait()).Should(Receive(BeNil()))

		runner.KillWithFire()
		etcdRunner.Stop()
	})

	startConverger := func() {
		runner.Start(convergeRepeatInterval, taskKickInterval, 30*time.Minute, 30*time.Second, 300*time.Second)
		time.Sleep(convergeRepeatInterval)
	}

	createClaimedTaskWithDeadExecutor := func() {
		task := models.Task{
			Domain: "tests",

			Guid:  "task-guid",
			Stack: "stack",
			Actions: []models.ExecutorAction{
				{
					Action: models.RunAction{
						Path: "cat",
						Args: []string{"/tmp/file"},
					},
				},
			},
		}

		err := bbs.DesireTask(task)
		Ω(err).ShouldNot(HaveOccurred())

		err = bbs.ClaimTask(task.Guid, "dead-executor")
		Ω(err).ShouldNot(HaveOccurred())
	}

	desireLRP := func() {
		err := bbs.DesireLRP(models.DesiredLRP{
			Domain: "tests",

			ProcessGuid: "the-guid",

			Stack: "some-stack",

			Instances: 3,
			MemoryMB:  128,
			DiskMB:    512,

			Actions: []models.ExecutorAction{
				{
					Action: models.RunAction{
						Path: "the-start-command",
					},
				},
			},
		})
		Ω(err).ShouldNot(HaveOccurred())
	}

	itIsInactive := func() {
		Describe("when an LRP is desired", func() {
			JustBeforeEach(desireLRP)

			It("does not create start auctions for apps that are missing instances", func() {
				Consistently(bbs.GetAllLRPStartAuctions, 0.5).Should(BeEmpty())
			})
		})

		Context("when a claimed task with a dead executor is present", func() {
			JustBeforeEach(createClaimedTaskWithDeadExecutor)

			It("does not change the task", func() {
				Consistently(bbs.GetAllCompletedTasks, taskKickInterval*2).Should(BeEmpty())
			})
		})
	}

	Context("when the converger has the lock", func() {
		JustBeforeEach(startConverger)

		Describe("when an LRP is desired", func() {
			BeforeEach(desireLRP)

			Context("for an app that is not running at all", func() {
				It("desires N start auctions in the BBS", func() {
					Eventually(bbs.GetAllLRPStartAuctions, 0.5).Should(HaveLen(3))
				})
			})

			Context("for an app that is missing instances", func() {
				BeforeEach(func() {
					bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "a",
						Index:        0,
					}, "the-executor-id")
				})

				It("start auctions for the missing instances", func() {
					Eventually(bbs.GetAllLRPStartAuctions, 0.5).Should(HaveLen(2))
					auctions, err := bbs.GetAllLRPStartAuctions()
					Ω(err).ShouldNot(HaveOccurred())

					indices := []int{auctions[0].Index, auctions[1].Index}
					Ω(indices).Should(ContainElement(1))
					Ω(indices).Should(ContainElement(2))

					Consistently(bbs.GetAllLRPStartAuctions).Should(HaveLen(2))
				})
			})

			Context("for an app that has extra instances", func() {
				BeforeEach(func() {
					bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "a",
						Index:        0,
					}, "the-executor-id")

					bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "b",
						Index:        1,
					}, "the-executor-id")

					bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "c",
						Index:        2,
					}, "the-executor-id")

					bbs.ReportActualLRPAsRunning(models.ActualLRP{
						ProcessGuid:  "the-guid",
						InstanceGuid: "d-extra",
						Index:        3,
					}, "the-executor-id")
				})

				It("stops the extra instances", func() {
					Consistently(bbs.GetAllLRPStartAuctions, 0.5).Should(BeEmpty())
					Eventually(bbs.GetAllStopLRPInstances).Should(HaveLen(1))
					stopInstances, err := bbs.GetAllStopLRPInstances()
					Ω(err).ShouldNot(HaveOccurred())

					Ω(stopInstances[0].ProcessGuid).Should(Equal("the-guid"))
					Ω(stopInstances[0].Index).Should(Equal(3))
					Ω(stopInstances[0].InstanceGuid).Should(Equal("d-extra"))
				})
			})
		})

		Describe("when a claimed task with a dead executor is present", func() {
			JustBeforeEach(createClaimedTaskWithDeadExecutor)

			It("eventually marks the task as failed", func() {
				Eventually(bbs.GetAllCompletedTasks, taskKickInterval*2).Should(HaveLen(1))
				tasks, err := bbs.GetAllTasks()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(tasks).Should(HaveLen(1))
				Ω(tasks[0].State).Should(Equal(models.TaskStateCompleted))
				Ω(tasks[0].Failed).Should(BeTrue())
			})
		})
	})

	Context("when the converger loses the lock", func() {
		BeforeEach(func() {
			startConverger()
			err := etcdClient.Update(storeadapter.StoreNode{
				Key:   shared.LockSchemaPath("converge_lock"),
				Value: []byte("something-else"),
			})
			Ω(err).ShouldNot(HaveOccurred())

			time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
		})

		itIsInactive()

		It("exits with an error", func() {
			Eventually(runner.Session.ExitCode).Should(Equal(1))
		})
	})

	Context("when the converger initially does not have the lock", func() {
		BeforeEach(func() {
			err := etcdClient.Create(storeadapter.StoreNode{
				Key:   shared.LockSchemaPath("converge_lock"),
				Value: []byte("something-else"),
			})
			Ω(err).ShouldNot(HaveOccurred())

			startConverger()
		})

		itIsInactive()

		Describe("when the lock becomes available", func() {
			BeforeEach(func() {
				err := etcdClient.Delete(shared.LockSchemaPath("converge_lock"))
				Ω(err).ShouldNot(HaveOccurred())

				time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
			})

			Describe("when an LRP is desired", func() {
				JustBeforeEach(desireLRP)

				Context("for an app that is not running at all", func() {
					It("desires N start auctions in the BBS", func() {
						Eventually(bbs.GetAllLRPStartAuctions, 0.5).Should(HaveLen(3))
					})
				})
			})

			Describe("when a claimed task with a dead executor is present", func() {
				JustBeforeEach(createClaimedTaskWithDeadExecutor)

				It("eventually marks the task as failed", func() {
					Eventually(bbs.GetAllCompletedTasks, taskKickInterval*2).Should(HaveLen(1))
				})
			})
		})
	})

	Describe("signal handling", func() {
		BeforeEach(func() {
			startConverger()
		})

		Describe("when it receives SIGINT", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGINT)
				Eventually(runner.Session, 4).Should(gexec.Exit(0))
			})
		})

		Describe("when it receives SIGTERM", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGTERM)
				Eventually(runner.Session, 4).Should(gexec.Exit(0))
			})
		})
	})
})
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package source

import (
	"fmt"
	"os"
	"strings"

	"github.com/kubernetes-incubator/external-dns/endpoint"
	log "github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

// crdSource is an implementation of Source that provides endpoints by listing
// specified CRD and fetching Endpoints embedded in Spec.
type crdSource struct {
	crdClient   rest.Interface
	namespace   string
	crdResource string
	codec       runtime.ParameterCodec
}

func addKnownTypes(scheme *runtime.Scheme, groupVersion schema.GroupVersion) error {
	scheme.AddKnownTypes(groupVersion,
		&endpoint.DNSEndpoint{},
		&endpoint.DNSEndpointList{},
	)
	metav1.AddToGroupVersion(scheme, groupVersion)
	return nil
}

// NewCRDClientForAPIVersionKind return rest client for the given apiVersion and kind of the CRD
func NewCRDClientForAPIVersionKind(client kubernetes.Interface, kubeConfig, kubeMaster, apiVersion, kind string) (*rest.RESTClient, *runtime.Scheme, error) {
	if kubeConfig == "" {
		if _, err := os.Stat(clientcmd.RecommendedHomeFile); err == nil {
			kubeConfig = clientcmd.RecommendedHomeFile
		}
	}

	config, err := clientcmd.BuildConfigFromFlags(kubeMaster, kubeConfig)
	if err != nil {
		return nil, nil, err
	}

	groupVersion, err := schema.ParseGroupVersion(apiVersion)
	if err != nil {
		return nil, nil, err
	}
	apiResourceList, err := client.Discovery().ServerResourcesForGroupVersion(groupVersion.String())
	if err != nil {
		return nil, nil, fmt.Errorf("error listing resources in GroupVersion %q: %s", groupVersion.String(), err)
	}

	var crdAPIResource *metav1.APIResource
	for _, apiResource := range apiResourceList.APIResources {
		if apiResource.Kind == kind {
			crdAPIResource = &apiResource
			break
		}
	}
	if crdAPIResource == nil {
		return nil, nil, fmt.Errorf("unable to find Resource Kind %q in GroupVersion %q", kind, apiVersion)
	}

	scheme := runtime.NewScheme()
	addKnownTypes(scheme, groupVersion)

	config.ContentConfig.GroupVersion = &groupVersion
	config.APIPath = "/apis"
	config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: serializer.NewCodecFactory(scheme)}

	crdClient, err := rest.UnversionedRESTClientFor(config)
	if err != nil {
		return nil, nil, err
	}
	return crdClient, scheme, nil
}

// NewCRDSource creates a new crdSource with the given config.
func NewCRDSource(crdClient rest.Interface, namespace, kind string, scheme *runtime.Scheme) (Source, error) {
	return &crdSource{
		crdResource: strings.ToLower(kind) + "s",
		namespace:   namespace,
		crdClient:   crdClient,
		codec:       runtime.NewParameterCodec(scheme),
	}, nil
}

// Endpoints returns endpoint objects.
func (cs *crdSource) Endpoints() ([]*endpoint.Endpoint, error) {
	endpoints := []*endpoint.Endpoint{}

	result, err := cs.List(&metav1.ListOptions{})
	if err != nil {
		return nil, err
	}

	for _, dnsEndpoint := range result.Items {
		// Make sure that all endpoints have targets for A or CNAME type
		for _, endPoint := range dnsEndpoint.Spec.Endpoints {
			if (endPoint.RecordType == "CNAME" || endPoint.RecordType == "A") && len(endPoint.Targets) < 1 {
				log.Warnf("Endpoint %s with DNSName %s has an empty list of targets", dnsEndpoint.ObjectMeta.Name, endPoint.DNSName)
				continue
			}
			endpoints = append(endpoints, endPoint)
		}

		if dnsEndpoint.Status.ObservedGeneration == dnsEndpoint.Generation {
			continue
		}

		dnsEndpoint.Status.ObservedGeneration = dnsEndpoint.Generation
		// Update the ObservedGeneration
		_, err = cs.UpdateStatus(&dnsEndpoint)
		if err != nil {
			log.Warnf("Could not update ObservedGeneration of the CRD: %v", err)
		}
	}

	return endpoints, nil
}

func (cs *crdSource) List(opts *metav1.ListOptions) (result *endpoint.DNSEndpointList, err error) {
	result = &endpoint.DNSEndpointList{}
	err = cs.crdClient.Get().
		Namespace(cs.namespace).
		Resource(cs.crdResource).
		VersionedParams(opts, cs.codec).
		Do().
		Into(result)
	return
}

func (cs *crdSource) UpdateStatus(dnsEndpoint *endpoint.DNSEndpoint) (result *endpoint.DNSEndpoint, err error) {
	result = &endpoint.DNSEndpoint{}
	err = cs.crdClient.Put().
		Namespace(dnsEndpoint.Namespace).
		Resource(cs.crdResource).
		Name(dnsEndpoint.Name).
		SubResource("status").
		Body(dnsEndpoint).
		Do().
		Into(result)
	return
}
<commit_msg>Rename endPoint to endpoint for crd implementation.<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package source

import (
	"fmt"
	"os"
	"strings"

	"github.com/kubernetes-incubator/external-dns/endpoint"
	log "github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

// crdSource is an implementation of Source that provides endpoints by listing
// specified CRD and fetching Endpoints embedded in Spec.
type crdSource struct {
	crdClient   rest.Interface
	namespace   string
	crdResource string
	codec       runtime.ParameterCodec
}

func addKnownTypes(scheme *runtime.Scheme, groupVersion schema.GroupVersion) error {
	scheme.AddKnownTypes(groupVersion,
		&endpoint.DNSEndpoint{},
		&endpoint.DNSEndpointList{},
	)
	metav1.AddToGroupVersion(scheme, groupVersion)
	return nil
}

// NewCRDClientForAPIVersionKind return rest client for the given apiVersion and kind of the CRD
func NewCRDClientForAPIVersionKind(client kubernetes.Interface, kubeConfig, kubeMaster, apiVersion, kind string) (*rest.RESTClient, *runtime.Scheme, error) {
	if kubeConfig == "" {
		if _, err := os.Stat(clientcmd.RecommendedHomeFile); err == nil {
			kubeConfig = clientcmd.RecommendedHomeFile
		}
	}

	config, err := clientcmd.BuildConfigFromFlags(kubeMaster, kubeConfig)
	if err != nil {
		return nil, nil, err
	}

	groupVersion, err := schema.ParseGroupVersion(apiVersion)
	if err != nil {
		return nil, nil, err
	}
	apiResourceList, err := client.Discovery().ServerResourcesForGroupVersion(groupVersion.String())
	if err != nil {
		return nil, nil, fmt.Errorf("error listing resources in GroupVersion %q: %s", groupVersion.String(), err)
	}

	var crdAPIResource *metav1.APIResource
	for _, apiResource := range apiResourceList.APIResources {
		if apiResource.Kind == kind {
			crdAPIResource = &apiResource
			break
		}
	}
	if crdAPIResource == nil {
		return nil, nil, fmt.Errorf("unable to find Resource Kind %q in GroupVersion %q", kind, apiVersion)
	}

	scheme := runtime.NewScheme()
	addKnownTypes(scheme, groupVersion)

	config.ContentConfig.GroupVersion = &groupVersion
	config.APIPath = "/apis"
	config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: serializer.NewCodecFactory(scheme)}

	crdClient, err := rest.UnversionedRESTClientFor(config)
	if err != nil {
		return nil, nil, err
	}
	return crdClient, scheme, nil
}

// NewCRDSource creates a new crdSource with the given config.
func NewCRDSource(crdClient rest.Interface, namespace, kind string, scheme *runtime.Scheme) (Source, error) {
	return &crdSource{
		crdResource: strings.ToLower(kind) + "s",
		namespace:   namespace,
		crdClient:   crdClient,
		codec:       runtime.NewParameterCodec(scheme),
	}, nil
}

// Endpoints returns endpoint objects.
func (cs *crdSource) Endpoints() ([]*endpoint.Endpoint, error) {
	endpoints := []*endpoint.Endpoint{}

	result, err := cs.List(&metav1.ListOptions{})
	if err != nil {
		return nil, err
	}

	for _, dnsEndpoint := range result.Items {
		// Make sure that all endpoints have targets for A or CNAME type
		for _, endpoint := range dnsEndpoint.Spec.Endpoints {
			if (endpoint.RecordType == "CNAME" || endpoint.RecordType == "A") && len(endpoint.Targets) < 1 {
				log.Warnf("Endpoint %s with DNSName %s has an empty list of targets", dnsEndpoint.ObjectMeta.Name, endpoint.DNSName)
				continue
			}
			endpoints = append(endpoints, endpoint)
		}

		if dnsEndpoint.Status.ObservedGeneration == dnsEndpoint.Generation {
			continue
		}

		dnsEndpoint.Status.ObservedGeneration = dnsEndpoint.Generation
		// Update the ObservedGeneration
		_, err = cs.UpdateStatus(&dnsEndpoint)
		if err != nil {
			log.Warnf("Could not update ObservedGeneration of the CRD: %v", err)
		}
	}

	return endpoints, nil
}

func (cs *crdSource) List(opts *metav1.ListOptions) (result *endpoint.DNSEndpointList, err error) {
	result = &endpoint.DNSEndpointList{}
	err = cs.crdClient.Get().
		Namespace(cs.namespace).
		Resource(cs.crdResource).
		VersionedParams(opts, cs.codec).
		Do().
		Into(result)
	return
}

func (cs *crdSource) UpdateStatus(dnsEndpoint *endpoint.DNSEndpoint) (result *endpoint.DNSEndpoint, err error) {
	result = &endpoint.DNSEndpoint{}
	err = cs.crdClient.Put().
		Namespace(dnsEndpoint.Namespace).
		Resource(cs.crdResource).
		Name(dnsEndpoint.Name).
		SubResource("status").
		Body(dnsEndpoint).
		Do().
		Into(result)
	return
}
<|endoftext|>


<commit_before>// Copyright 2016 HenryLee. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package faygo

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"html/template"
	"path/filepath"
	"strconv"
	"strings"
	"time"
	"unsafe"

	json "github.com/json-iterator/go"

	"github.com/henrylee2cn/faygo/acceptencoder"
)

// Size returns the current size, in bytes, of the response.
func (ctx *Context) Size() int64 {
	return ctx.W.Size()
}

// Committed returns whether the response has been submitted or not.
func (ctx *Context) Committed() bool {
	return ctx.W.committed
}

// Status returns the HTTP status code of the response.
func (ctx *Context) Status() int {
	return ctx.W.status
}

// IsCachable returns boolean of this request is cached.
// HTTP 304 means cached.
func (ctx *Context) IsCachable() bool {
	return ctx.W.status >= 200 && ctx.W.status < 300 || ctx.W.status == 304
}

// IsEmpty returns boolean of this request is empty.
// HTTP 201，204 and 304 means empty.
func (ctx *Context) IsEmpty() bool {
	return ctx.W.status == 201 || ctx.W.status == 204 || ctx.W.status == 304
}

// IsOk returns boolean of this request runs well.
// HTTP 200 means ok.
func (ctx *Context) IsOk() bool {
	return ctx.W.status == 200
}

// IsSuccessful returns boolean of this request runs successfully.
// HTTP 2xx means ok.
func (ctx *Context) IsSuccessful() bool {
	return ctx.W.status >= 200 && ctx.W.status < 300
}

// IsRedirect returns boolean of this request is redirection header.
// HTTP 301,302,307 means redirection.
func (ctx *Context) IsRedirect() bool {
	return ctx.W.status == 301 || ctx.W.status == 302 || ctx.W.status == 303 || ctx.W.status == 307
}

// IsForbidden returns boolean of this request is forbidden.
// HTTP 403 means forbidden.
func (ctx *Context) IsForbidden() bool {
	return ctx.W.status == 403
}

// IsNotFound returns boolean of this request is not found.
// HTTP 404 means forbidden.
func (ctx *Context) IsNotFound() bool {
	return ctx.W.status == 404
}

// IsClientError returns boolean of this request client sends error data.
// HTTP 4xx means forbidden.
func (ctx *Context) IsClientError() bool {
	return ctx.W.status >= 400 && ctx.W.status < 500
}

// IsServerError returns boolean of this server handler errors.
// HTTP 5xx means server internal error.
func (ctx *Context) IsServerError() bool {
	return ctx.W.status >= 500 && ctx.W.status < 600
}

// SetHeader sets response header item string via given key.
func (ctx *Context) SetHeader(key, val string) {
	ctx.W.Header().Set(key, val)
}

// SetCookie sets cookie value via given key.
// others are ordered as cookie's max age time, path, domain, secure and httponly.
func (ctx *Context) SetCookie(name string, value string, others ...interface{}) {
	var b bytes.Buffer
	fmt.Fprintf(&b, "%s=%s", sanitizeName(name), sanitizeValue(value))
	//fix cookie not work in IE
	if len(others) > 0 {
		var maxAge int64
		switch v := others[0].(type) {
		case int:
			maxAge = int64(v)
		case int32:
			maxAge = int64(v)
		case int64:
			maxAge = v
		}
		switch {
		case maxAge > 0:
			fmt.Fprintf(&b, "; Expires=%s; Max-Age=%d", time.Now().Add(time.Duration(maxAge)*time.Second).UTC().Format(time.RFC1123), maxAge)
		case maxAge < 0:
			fmt.Fprintf(&b, "; Max-Age=0")
		}
	}
	// the settings below
	// Path, Domain, Secure, HttpOnly
	// can use nil skip set

	// default "/"
	if len(others) > 1 {
		if v, ok := others[1].(string); ok && len(v) > 0 {
			fmt.Fprintf(&b, "; Path=%s", sanitizeValue(v))
		}
	} else {
		fmt.Fprintf(&b, "; Path=%s", "/")
	}

	// default empty
	if len(others) > 2 {
		if v, ok := others[2].(string); ok && len(v) > 0 {
			fmt.Fprintf(&b, "; Domain=%s", sanitizeValue(v))
		}
	}

	// default empty
	if len(others) > 3 {
		var secure bool
		switch v := others[3].(type) {
		case bool:
			secure = v
		default:
			if others[3] != nil {
				secure = true
			}
		}
		if secure {
			fmt.Fprintf(&b, "; Secure")
		}
	}

	// default false. for session cookie default true
	httponly := false
	if len(others) > 4 {
		if v, ok := others[4].(bool); ok && v {
			// HttpOnly = true
			httponly = true
		}
	}

	if httponly {
		fmt.Fprintf(&b, "; HttpOnly")
	}

	ctx.W.Header().Add(HeaderSetCookie, b.String())
}

var cookieNameSanitizer = strings.NewReplacer("\n", "-", "\r", "-")

func sanitizeName(n string) string {
	return cookieNameSanitizer.Replace(n)
}

var cookieValueSanitizer = strings.NewReplacer("\n", " ", "\r", " ", ";", " ")

func sanitizeValue(v string) string {
	return cookieValueSanitizer.Replace(v)
}

// SetSecureCookie Set Secure cookie for response.
func (ctx *Context) SetSecureCookie(secret, name, value string, others ...interface{}) {
	vs := base64.URLEncoding.EncodeToString([]byte(value))
	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
	h := hmac.New(sha1.New, []byte(secret))
	fmt.Fprintf(h, "%s%s", vs, timestamp)
	sig := fmt.Sprintf("%02x", h.Sum(nil))
	cookie := strings.Join([]string{vs, timestamp, sig}, "|")
	ctx.SetCookie(name, cookie, others...)
}

// NoContent sends a response with no body and a status code.
func (ctx *Context) NoContent(status int) {
	ctx.W.WriteHeader(status)
}

// Send error message and stop handler chain.
func (ctx *Context) Error(status int, errStr string) {
	global.errorFunc(ctx, errStr, status)
	ctx.Stop()
}

// Bytes writes the data bytes to the connection as part of an HTTP reply.
func (ctx *Context) Bytes(status int, contentType string, content []byte) error {
	if ctx.W.committed {
		ctx.W.multiCommitted()
		return nil
	}
	ctx.W.Header().Set(HeaderContentType, contentType)
	if ctx.enableGzip && len(ctx.W.Header()[HeaderContentEncoding]) == 0 {
		buf := &bytes.Buffer{}
		ok, encoding, _ := acceptencoder.WriteBody(acceptencoder.ParseEncoding(ctx.R), buf, content)
		if ok {
			ctx.W.Header().Set(HeaderContentEncoding, encoding)
			content = buf.Bytes()
		}
	}
	ctx.W.Header().Set(HeaderContentLength, strconv.Itoa(len(content)))
	ctx.W.WriteHeader(status)
	_, err := ctx.W.Write(content)
	return err
}

// String writes a string to the client, something like fmt.Fprintf
func (ctx *Context) String(status int, format string, s ...interface{}) error {
	if len(s) == 0 {
		return ctx.Bytes(status, MIMETextPlainCharsetUTF8, []byte(format))
	}
	return ctx.Bytes(status, MIMETextPlainCharsetUTF8, []byte(fmt.Sprintf(format, s...)))
}

// HTML sends an HTTP response with status code.
func (ctx *Context) HTML(status int, html string) error {
	x := (*[2]uintptr)(unsafe.Pointer(&html))
	h := [3]uintptr{x[0], x[1], x[1]}
	return ctx.Bytes(status, MIMETextHTMLCharsetUTF8, *(*[]byte)(unsafe.Pointer(&h)))
}

// JSON sends a JSON response with status code.
func (ctx *Context) JSON(status int, data interface{}, isIndent ...bool) error {
	var (
		b   []byte
		err error
	)
	if len(isIndent) > 0 && isIndent[0] {
		b, err = json.MarshalIndent(data, "", "  ")
	} else {
		b, err = json.Marshal(data)
	}
	if err != nil {
		return err
	}
	return ctx.JSONBlob(status, b)
}

// JSONBlob sends a JSON blob response with status code.
func (ctx *Context) JSONBlob(status int, b []byte) error {
	return ctx.Bytes(status, MIMEApplicationJSONCharsetUTF8, b)
}

// JSONP sends a JSONP response with status code. It uses `callback` to construct
// the JSONP payload.
func (ctx *Context) JSONP(status int, callback string, data interface{}, isIndent ...bool) error {
	var (
		b   []byte
		err error
	)
	if len(isIndent) > 0 && isIndent[0] {
		b, err = json.MarshalIndent(data, "", "  ")
	} else {
		b, err = json.Marshal(data)
	}
	if err != nil {
		return err
	}
	callback = template.JSEscapeString(callback)
	callbackContent := bytes.NewBufferString(" if(window." + callback + ")" + callback)
	callbackContent.WriteString("(")
	callbackContent.Write(b)
	callbackContent.WriteString(");\r\n")
	return ctx.Bytes(status, MIMEApplicationJavaScriptCharsetUTF8, callbackContent.Bytes())
}

// JSONMsg sends a JSON with JSONMsg format.
func (ctx *Context) JSONMsg(status int, msgcode int, info interface{}, isIndent ...bool) error {
	var (
		b    []byte
		err  error
		data = JSONMsg{
			Code: msgcode,
			Info: info,
		}
	)
	if len(isIndent) > 0 && isIndent[0] {
		b, err = json.MarshalIndent(data, "", "  ")
	} else {
		b, err = json.Marshal(data)
	}
	if err != nil {
		return err
	}
	return ctx.JSONBlob(status, b)
}

// XML sends an XML response with status code.
func (ctx *Context) XML(status int, data interface{}, isIndent ...bool) error {
	var (
		b   []byte
		err error
	)
	if len(isIndent) > 0 && isIndent[0] {
		b, err = xml.MarshalIndent(data, "", "  ")
	} else {
		b, err = xml.Marshal(data)
	}
	if err != nil {
		return err
	}
	return ctx.XMLBlob(status, b)
}

// XMLBlob sends a XML blob response with status code.
func (ctx *Context) XMLBlob(status int, b []byte) error {
	content := bytes.NewBufferString(xml.Header)
	content.Write(b)
	return ctx.Bytes(status, MIMEApplicationXMLCharsetUTF8, content.Bytes())
}

// JSONOrXML serve Xml OR Json, depending on the value of the Accept header
func (ctx *Context) JSONOrXML(status int, data interface{}, isIndent ...bool) error {
	if ctx.AcceptJSON() || !ctx.AcceptXML() {
		return ctx.JSON(status, data, isIndent...)
	}
	return ctx.XML(status, data, isIndent...)
}

// File forces response for download file.
// it prepares the download response header automatically.
func (ctx *Context) File(file string, filename ...string) {
	ctx.W.Header().Set(HeaderContentDescription, "File Transfer")
	ctx.W.Header().Set(HeaderContentType, MIMEOctetStream)
	if len(filename) > 0 && filename[0] != "" {
		ctx.W.Header().Set(HeaderContentDisposition, "attachment; filename="+filename[0])
	} else {
		ctx.W.Header().Set(HeaderContentDisposition, "attachment; filename="+filepath.Base(file))
	}
	ctx.W.Header().Set(HeaderContentTransferEncoding, "binary")
	ctx.W.Header().Set(HeaderExpires, "0")
	ctx.W.Header().Set(HeaderCacheControl, "must-revalidate")
	ctx.W.Header().Set(HeaderPragma, "public")
	global.fsManager.ServeFile(ctx, file)
}

// Render renders a template with data and sends a text/html response with status code.
func (ctx *Context) Render(status int, name string, data Map) error {
	b, err := global.render.Render(name, data)
	if err != nil {
		return err
	}
	return ctx.Bytes(status, MIMETextHTMLCharsetUTF8, b)
}
<commit_msg>Optimize ctx.File args name<commit_after>// Copyright 2016 HenryLee. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package faygo

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"html/template"
	"path/filepath"
	"strconv"
	"strings"
	"time"
	"unsafe"

	json "github.com/json-iterator/go"

	"github.com/henrylee2cn/faygo/acceptencoder"
)

// Size returns the current size, in bytes, of the response.
func (ctx *Context) Size() int64 {
	return ctx.W.Size()
}

// Committed returns whether the response has been submitted or not.
func (ctx *Context) Committed() bool {
	return ctx.W.committed
}

// Status returns the HTTP status code of the response.
func (ctx *Context) Status() int {
	return ctx.W.status
}

// IsCachable returns boolean of this request is cached.
// HTTP 304 means cached.
func (ctx *Context) IsCachable() bool {
	return ctx.W.status >= 200 && ctx.W.status < 300 || ctx.W.status == 304
}

// IsEmpty returns boolean of this request is empty.
// HTTP 201，204 and 304 means empty.
func (ctx *Context) IsEmpty() bool {
	return ctx.W.status == 201 || ctx.W.status == 204 || ctx.W.status == 304
}

// IsOk returns boolean of this request runs well.
// HTTP 200 means ok.
func (ctx *Context) IsOk() bool {
	return ctx.W.status == 200
}

// IsSuccessful returns boolean of this request runs successfully.
// HTTP 2xx means ok.
func (ctx *Context) IsSuccessful() bool {
	return ctx.W.status >= 200 && ctx.W.status < 300
}

// IsRedirect returns boolean of this request is redirection header.
// HTTP 301,302,307 means redirection.
func (ctx *Context) IsRedirect() bool {
	return ctx.W.status == 301 || ctx.W.status == 302 || ctx.W.status == 303 || ctx.W.status == 307
}

// IsForbidden returns boolean of this request is forbidden.
// HTTP 403 means forbidden.
func (ctx *Context) IsForbidden() bool {
	return ctx.W.status == 403
}

// IsNotFound returns boolean of this request is not found.
// HTTP 404 means forbidden.
func (ctx *Context) IsNotFound() bool {
	return ctx.W.status == 404
}

// IsClientError returns boolean of this request client sends error data.
// HTTP 4xx means forbidden.
func (ctx *Context) IsClientError() bool {
	return ctx.W.status >= 400 && ctx.W.status < 500
}

// IsServerError returns boolean of this server handler errors.
// HTTP 5xx means server internal error.
func (ctx *Context) IsServerError() bool {
	return ctx.W.status >= 500 && ctx.W.status < 600
}

// SetHeader sets response header item string via given key.
func (ctx *Context) SetHeader(key, val string) {
	ctx.W.Header().Set(key, val)
}

// SetCookie sets cookie value via given key.
// others are ordered as cookie's max age time, path, domain, secure and httponly.
func (ctx *Context) SetCookie(name string, value string, others ...interface{}) {
	var b bytes.Buffer
	fmt.Fprintf(&b, "%s=%s", sanitizeName(name), sanitizeValue(value))
	//fix cookie not work in IE
	if len(others) > 0 {
		var maxAge int64
		switch v := others[0].(type) {
		case int:
			maxAge = int64(v)
		case int32:
			maxAge = int64(v)
		case int64:
			maxAge = v
		}
		switch {
		case maxAge > 0:
			fmt.Fprintf(&b, "; Expires=%s; Max-Age=%d", time.Now().Add(time.Duration(maxAge)*time.Second).UTC().Format(time.RFC1123), maxAge)
		case maxAge < 0:
			fmt.Fprintf(&b, "; Max-Age=0")
		}
	}
	// the settings below
	// Path, Domain, Secure, HttpOnly
	// can use nil skip set

	// default "/"
	if len(others) > 1 {
		if v, ok := others[1].(string); ok && len(v) > 0 {
			fmt.Fprintf(&b, "; Path=%s", sanitizeValue(v))
		}
	} else {
		fmt.Fprintf(&b, "; Path=%s", "/")
	}

	// default empty
	if len(others) > 2 {
		if v, ok := others[2].(string); ok && len(v) > 0 {
			fmt.Fprintf(&b, "; Domain=%s", sanitizeValue(v))
		}
	}

	// default empty
	if len(others) > 3 {
		var secure bool
		switch v := others[3].(type) {
		case bool:
			secure = v
		default:
			if others[3] != nil {
				secure = true
			}
		}
		if secure {
			fmt.Fprintf(&b, "; Secure")
		}
	}

	// default false. for session cookie default true
	httponly := false
	if len(others) > 4 {
		if v, ok := others[4].(bool); ok && v {
			// HttpOnly = true
			httponly = true
		}
	}

	if httponly {
		fmt.Fprintf(&b, "; HttpOnly")
	}

	ctx.W.Header().Add(HeaderSetCookie, b.String())
}

var cookieNameSanitizer = strings.NewReplacer("\n", "-", "\r", "-")

func sanitizeName(n string) string {
	return cookieNameSanitizer.Replace(n)
}

var cookieValueSanitizer = strings.NewReplacer("\n", " ", "\r", " ", ";", " ")

func sanitizeValue(v string) string {
	return cookieValueSanitizer.Replace(v)
}

// SetSecureCookie Set Secure cookie for response.
func (ctx *Context) SetSecureCookie(secret, name, value string, others ...interface{}) {
	vs := base64.URLEncoding.EncodeToString([]byte(value))
	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
	h := hmac.New(sha1.New, []byte(secret))
	fmt.Fprintf(h, "%s%s", vs, timestamp)
	sig := fmt.Sprintf("%02x", h.Sum(nil))
	cookie := strings.Join([]string{vs, timestamp, sig}, "|")
	ctx.SetCookie(name, cookie, others...)
}

// NoContent sends a response with no body and a status code.
func (ctx *Context) NoContent(status int) {
	ctx.W.WriteHeader(status)
}

// Send error message and stop handler chain.
func (ctx *Context) Error(status int, errStr string) {
	global.errorFunc(ctx, errStr, status)
	ctx.Stop()
}

// Bytes writes the data bytes to the connection as part of an HTTP reply.
func (ctx *Context) Bytes(status int, contentType string, content []byte) error {
	if ctx.W.committed {
		ctx.W.multiCommitted()
		return nil
	}
	ctx.W.Header().Set(HeaderContentType, contentType)
	if ctx.enableGzip && len(ctx.W.Header()[HeaderContentEncoding]) == 0 {
		buf := &bytes.Buffer{}
		ok, encoding, _ := acceptencoder.WriteBody(acceptencoder.ParseEncoding(ctx.R), buf, content)
		if ok {
			ctx.W.Header().Set(HeaderContentEncoding, encoding)
			content = buf.Bytes()
		}
	}
	ctx.W.Header().Set(HeaderContentLength, strconv.Itoa(len(content)))
	ctx.W.WriteHeader(status)
	_, err := ctx.W.Write(content)
	return err
}

// String writes a string to the client, something like fmt.Fprintf
func (ctx *Context) String(status int, format string, s ...interface{}) error {
	if len(s) == 0 {
		return ctx.Bytes(status, MIMETextPlainCharsetUTF8, []byte(format))
	}
	return ctx.Bytes(status, MIMETextPlainCharsetUTF8, []byte(fmt.Sprintf(format, s...)))
}

// HTML sends an HTTP response with status code.
func (ctx *Context) HTML(status int, html string) error {
	x := (*[2]uintptr)(unsafe.Pointer(&html))
	h := [3]uintptr{x[0], x[1], x[1]}
	return ctx.Bytes(status, MIMETextHTMLCharsetUTF8, *(*[]byte)(unsafe.Pointer(&h)))
}

// JSON sends a JSON response with status code.
func (ctx *Context) JSON(status int, data interface{}, isIndent ...bool) error {
	var (
		b   []byte
		err error
	)
	if len(isIndent) > 0 && isIndent[0] {
		b, err = json.MarshalIndent(data, "", "  ")
	} else {
		b, err = json.Marshal(data)
	}
	if err != nil {
		return err
	}
	return ctx.JSONBlob(status, b)
}

// JSONBlob sends a JSON blob response with status code.
func (ctx *Context) JSONBlob(status int, b []byte) error {
	return ctx.Bytes(status, MIMEApplicationJSONCharsetUTF8, b)
}

// JSONP sends a JSONP response with status code. It uses `callback` to construct
// the JSONP payload.
func (ctx *Context) JSONP(status int, callback string, data interface{}, isIndent ...bool) error {
	var (
		b   []byte
		err error
	)
	if len(isIndent) > 0 && isIndent[0] {
		b, err = json.MarshalIndent(data, "", "  ")
	} else {
		b, err = json.Marshal(data)
	}
	if err != nil {
		return err
	}
	callback = template.JSEscapeString(callback)
	callbackContent := bytes.NewBufferString(" if(window." + callback + ")" + callback)
	callbackContent.WriteString("(")
	callbackContent.Write(b)
	callbackContent.WriteString(");\r\n")
	return ctx.Bytes(status, MIMEApplicationJavaScriptCharsetUTF8, callbackContent.Bytes())
}

// JSONMsg sends a JSON with JSONMsg format.
func (ctx *Context) JSONMsg(status int, msgcode int, info interface{}, isIndent ...bool) error {
	var (
		b    []byte
		err  error
		data = JSONMsg{
			Code: msgcode,
			Info: info,
		}
	)
	if len(isIndent) > 0 && isIndent[0] {
		b, err = json.MarshalIndent(data, "", "  ")
	} else {
		b, err = json.Marshal(data)
	}
	if err != nil {
		return err
	}
	return ctx.JSONBlob(status, b)
}

// XML sends an XML response with status code.
func (ctx *Context) XML(status int, data interface{}, isIndent ...bool) error {
	var (
		b   []byte
		err error
	)
	if len(isIndent) > 0 && isIndent[0] {
		b, err = xml.MarshalIndent(data, "", "  ")
	} else {
		b, err = xml.Marshal(data)
	}
	if err != nil {
		return err
	}
	return ctx.XMLBlob(status, b)
}

// XMLBlob sends a XML blob response with status code.
func (ctx *Context) XMLBlob(status int, b []byte) error {
	content := bytes.NewBufferString(xml.Header)
	content.Write(b)
	return ctx.Bytes(status, MIMEApplicationXMLCharsetUTF8, content.Bytes())
}

// JSONOrXML serve Xml OR Json, depending on the value of the Accept header
func (ctx *Context) JSONOrXML(status int, data interface{}, isIndent ...bool) error {
	if ctx.AcceptJSON() || !ctx.AcceptXML() {
		return ctx.JSON(status, data, isIndent...)
	}
	return ctx.XML(status, data, isIndent...)
}

// File forces response for download file.
// it prepares the download response header automatically.
func (ctx *Context) File(localFilename string, showFilename ...string) {
	ctx.W.Header().Set(HeaderContentDescription, "File Transfer")
	ctx.W.Header().Set(HeaderContentType, MIMEOctetStream)
	if len(showFilename) > 0 && showFilename[0] != "" {
		ctx.W.Header().Set(HeaderContentDisposition, "attachment; filename="+showFilename[0])
	} else {
		ctx.W.Header().Set(HeaderContentDisposition, "attachment; filename="+filepath.Base(localFilename))
	}
	ctx.W.Header().Set(HeaderContentTransferEncoding, "binary")
	ctx.W.Header().Set(HeaderExpires, "0")
	ctx.W.Header().Set(HeaderCacheControl, "must-revalidate")
	ctx.W.Header().Set(HeaderPragma, "public")
	global.fsManager.ServeFile(ctx, localFilename)
}

// Render renders a template with data and sends a text/html response with status code.
func (ctx *Context) Render(status int, name string, data Map) error {
	b, err := global.render.Render(name, data)
	if err != nil {
		return err
	}
	return ctx.Bytes(status, MIMETextHTMLCharsetUTF8, b)
}
<|endoftext|>


<commit_before>package api

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"errors"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"time"

	"github.com/STNS/STNS/middleware"
	"github.com/STNS/STNS/model"
	"github.com/STNS/STNS/stns"
	"github.com/facebookgo/pidfile"
	"github.com/labstack/echo"
	emiddleware "github.com/labstack/echo/middleware"

	"github.com/labstack/gommon/log"

	"github.com/lestrrat/go-server-starter/listener"
)

type httpServer struct {
	baseServer
}

func newHTTPServer(confPath string) (*httpServer, error) {
	conf, err := stns.NewConfig(confPath)
	if err != nil {
		return nil, err
	}

	s := &httpServer{
		baseServer{config: &conf},
	}
	return s, nil
}

func status(c echo.Context) error {
	return c.String(http.StatusOK, "OK")
}

// Run サーバの起動
func (s *httpServer) Run() error {
	var backends model.Backends
	e := echo.New()
	if os.Getenv("STNS_LOG") != "" {
		f, err := os.OpenFile(os.Getenv("STNS_LOG"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
		if err != nil {
			return errors.New("error opening file :" + err.Error())
		}
		e.Logger.SetOutput(f)
	} else {
		e.Logger.SetLevel(log.DEBUG)
	}
	e.GET("/status", status)

	if err := pidfile.Write(); err != nil {
		return err
	}

	defer func() {
		if err := os.Remove(pidfile.GetPidfilePath()); err != nil {
			e.Logger.Fatalf("Error removing %s: %s", pidfile.GetPidfilePath(), err)
		}
	}()

	b, err := model.NewBackendTomlFile(s.config.Users, s.config.Groups)
	if err != nil {
		return err
	}
	backends = append(backends, b)

	err = s.loadModules(e.Logger.(*log.Logger), &backends)
	if err != nil {
		return err
	}

	e.Use(middleware.Backends(backends))
	e.Use(middleware.AddHeader(backends))

	e.Use(emiddleware.Recover())
	e.Use(emiddleware.LoggerWithConfig(emiddleware.LoggerConfig{
		Format: `{"time":"${time_rfc3339_nano}","remote_ip":"${remote_ip}","host":"${host}",` +
			`"method":"${method}","uri":"${uri}","status":${status}}` + "\n",
	}))

	if s.config.BasicAuth != nil {
		e.Use(emiddleware.BasicAuthWithConfig(
			emiddleware.BasicAuthConfig{
				Validator: func(username, password string, c echo.Context) (bool, error) {
					if username == s.config.BasicAuth.User && password == s.config.BasicAuth.Password {
						return true, nil
					}
					return false, nil
				},
				Skipper: func(c echo.Context) bool {
					if c.Path() == "/" || c.Path() == "/status" || len(os.Getenv("CI")) > 0 {
						return true
					}
					return false
				},
			}))
	}

	if s.config.TokenAuth != nil {
		e.Use(middleware.TokenAuthWithConfig(middleware.TokenAuthConfig{
			Skipper: func(c echo.Context) bool {

				if c.Path() == "/" || c.Path() == "/status" || len(os.Getenv("CI")) > 0 {
					return true
				}

				return false
			},
			Validator: func(token string) bool {
				for _, a := range s.config.TokenAuth.Tokens {
					if a == token {
						return true
					}
				}
				return false
			},
		}))
	}

	if s.config.UseServerStarter {
		listeners, err := listener.ListenAll()
		if listeners == nil || err != nil {
			return err
		}
		e.Listener = listeners[0]
	}

	go func() {
		customServer := &http.Server{
			WriteTimeout: 1 * time.Minute,
		}
		if e.Listener == nil {
			p := strconv.Itoa(s.config.Port)
			customServer.Addr = ":" + p
		}

		// tls client authentication
		if s.config.TLS != nil {
			if _, err := os.Stat(s.config.TLS.CA); err == nil {
				ca, err := ioutil.ReadFile(s.config.TLS.CA)
				if err != nil {
					e.Logger.Fatal(err)
				}
				caPool := x509.NewCertPool()
				caPool.AppendCertsFromPEM(ca)

				tlsConfig := &tls.Config{
					ClientCAs:              caPool,
					SessionTicketsDisabled: true,
					ClientAuth:             tls.RequireAndVerifyClientCert,
				}

				tlsConfig.BuildNameToCertificate()
				customServer.TLSConfig = tlsConfig
			}
		}

		if s.config.TLS != nil && s.config.TLS.Cert != "" && s.config.TLS.Key != "" {
			if customServer.TLSConfig == nil {
				customServer.TLSConfig = new(tls.Config)
			}
			customServer.TLSConfig.Certificates = make([]tls.Certificate, 1)
			customServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(s.config.TLS.Cert, s.config.TLS.Key)
			if err != nil {
				e.Logger.Fatal(err)
			}

		}

		if err := e.StartServer(customServer); err != nil {
			e.Logger.Fatalf("shutting down the server: %s", err)
		}
	}()

	v1 := e.Group("/v1")
	UserEndpoints(v1)
	GroupEndpoints(v1)

	e.GET("/", func(c echo.Context) error {
		return c.String(http.StatusOK, "Hello! STNS!!1")
	})

	quit := make(chan os.Signal)
	signal.Notify(quit, os.Interrupt)
	<-quit
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	if err := e.Shutdown(ctx); err != nil {
		return err
	}
	return nil
}
<commit_msg>output HTTP(S) request log to LOGFILE(stns.log)<commit_after>package api

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"errors"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"time"

	"github.com/STNS/STNS/middleware"
	"github.com/STNS/STNS/model"
	"github.com/STNS/STNS/stns"
	"github.com/facebookgo/pidfile"
	"github.com/labstack/echo"
	emiddleware "github.com/labstack/echo/middleware"

	"github.com/labstack/gommon/log"

	"github.com/lestrrat/go-server-starter/listener"
)

type httpServer struct {
	baseServer
}

func newHTTPServer(confPath string) (*httpServer, error) {
	conf, err := stns.NewConfig(confPath)
	if err != nil {
		return nil, err
	}

	s := &httpServer{
		baseServer{config: &conf},
	}
	return s, nil
}

func status(c echo.Context) error {
	return c.String(http.StatusOK, "OK")
}

func switchLogOutput() (*os.File, error) {
	if os.Getenv("STNS_LOG") != "" {
		f, err := os.OpenFile(os.Getenv("STNS_LOG"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
		return f, err
	}
	return os.Stdout, nil
}

// Run サーバの起動
func (s *httpServer) Run() error {
	var backends model.Backends
	e := echo.New()
	f, err := switchLogOutput()
	if err != nil {
		return errors.New("error opening file :" + err.Error())
	}
	if f != os.Stdout {
		e.Logger.SetOutput(f)
	} else {
		e.Logger.SetLevel(log.DEBUG)
	}
	e.GET("/status", status)

	if err := pidfile.Write(); err != nil {
		return err
	}

	defer func() {
		if err := os.Remove(pidfile.GetPidfilePath()); err != nil {
			e.Logger.Fatalf("Error removing %s: %s", pidfile.GetPidfilePath(), err)
		}
	}()

	b, err := model.NewBackendTomlFile(s.config.Users, s.config.Groups)
	if err != nil {
		return err
	}
	backends = append(backends, b)

	err = s.loadModules(e.Logger.(*log.Logger), &backends)
	if err != nil {
		return err
	}

	e.Use(middleware.Backends(backends))
	e.Use(middleware.AddHeader(backends))

	e.Use(emiddleware.Recover())
	e.Use(emiddleware.LoggerWithConfig(emiddleware.LoggerConfig{
		Format: `{"time":"${time_rfc3339_nano}","remote_ip":"${remote_ip}","host":"${host}",` +
			`"method":"${method}","uri":"${uri}","status":${status}}` + "\n",
		Output: f,
	}))

	if s.config.BasicAuth != nil {
		e.Use(emiddleware.BasicAuthWithConfig(
			emiddleware.BasicAuthConfig{
				Validator: func(username, password string, c echo.Context) (bool, error) {
					if username == s.config.BasicAuth.User && password == s.config.BasicAuth.Password {
						return true, nil
					}
					return false, nil
				},
				Skipper: func(c echo.Context) bool {
					if c.Path() == "/" || c.Path() == "/status" || len(os.Getenv("CI")) > 0 {
						return true
					}
					return false
				},
			}))
	}

	if s.config.TokenAuth != nil {
		e.Use(middleware.TokenAuthWithConfig(middleware.TokenAuthConfig{
			Skipper: func(c echo.Context) bool {

				if c.Path() == "/" || c.Path() == "/status" || len(os.Getenv("CI")) > 0 {
					return true
				}

				return false
			},
			Validator: func(token string) bool {
				for _, a := range s.config.TokenAuth.Tokens {
					if a == token {
						return true
					}
				}
				return false
			},
		}))
	}

	if s.config.UseServerStarter {
		listeners, err := listener.ListenAll()
		if listeners == nil || err != nil {
			return err
		}
		e.Listener = listeners[0]
	}

	go func() {
		customServer := &http.Server{
			WriteTimeout: 1 * time.Minute,
		}
		if e.Listener == nil {
			p := strconv.Itoa(s.config.Port)
			customServer.Addr = ":" + p
		}

		// tls client authentication
		if s.config.TLS != nil {
			if _, err := os.Stat(s.config.TLS.CA); err == nil {
				ca, err := ioutil.ReadFile(s.config.TLS.CA)
				if err != nil {
					e.Logger.Fatal(err)
				}
				caPool := x509.NewCertPool()
				caPool.AppendCertsFromPEM(ca)

				tlsConfig := &tls.Config{
					ClientCAs:              caPool,
					SessionTicketsDisabled: true,
					ClientAuth:             tls.RequireAndVerifyClientCert,
				}

				tlsConfig.BuildNameToCertificate()
				customServer.TLSConfig = tlsConfig
			}
		}

		if s.config.TLS != nil && s.config.TLS.Cert != "" && s.config.TLS.Key != "" {
			if customServer.TLSConfig == nil {
				customServer.TLSConfig = new(tls.Config)
			}
			customServer.TLSConfig.Certificates = make([]tls.Certificate, 1)
			customServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(s.config.TLS.Cert, s.config.TLS.Key)
			if err != nil {
				e.Logger.Fatal(err)
			}

		}

		if err := e.StartServer(customServer); err != nil {
			e.Logger.Fatalf("shutting down the server: %s", err)
		}
	}()

	v1 := e.Group("/v1")
	UserEndpoints(v1)
	GroupEndpoints(v1)

	e.GET("/", func(c echo.Context) error {
		return c.String(http.StatusOK, "Hello! STNS!!1")
	})

	quit := make(chan os.Signal)
	signal.Notify(quit, os.Interrupt)
	<-quit
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	if err := e.Shutdown(ctx); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Package bitutils provides a collection of utilities to deal with bits.
package bitutils

import (
	"fmt"
)

// W is the length of a machine word.
const W = 64

// Magic constants.
const (
	lowers2 = 0x5555555555555555
	lowers4 = 0x3333333333333333
	lowers8 = 0x0f0f0f0f0f0f0f0f
	lowest8 = 0x0101010101010101
)

// Word represents a 64-bit binary string.
type Word uint64

// exp2[i] is 2^i.
var exp2 [W]Word

func init() {
	for i := 0; i < len(exp2); i++ {
		exp2[i] = Word(1) << uint(i)
	}
}

// String returns binary string w[0]w[1]...w[63].
func (w Word) String() string {
	return fmt.Sprintf("%064b", w)
}

// Count1 returns the number of ones contained in w.
func (w Word) Count1() int {
	w -= (w >> 1) & lowers2
	w = (w & lowers4) + ((w >> 2) & lowers4)
	w = (w + (w >> 4)) & lowers8
	return int((w * lowest8) >> 56)
}

// Count0 returns the number of zeros contained in w.
func (w Word) Count0() int {
	w = ^w
	return w.Count1()
}

// Count returns the number of b[0]'s contained in w.
func (w Word) Count(b int) int {
	w = w ^ (^Word(0) + Word(b))
	return w.Count1()
}

// Get returns w[i].
func (w Word) Get(i int) Word {
	w = w >> uint(i)
	return w & exp2[0]
}

// Set1 sets w[i] to 1.
func (w Word) Set1(i int) Word {
	return w | exp2[i]
}

// Set0 sets w[i] to 0.
func (w Word) Set0(i int) Word {
	return w & ^exp2[i]
}

// Flip flips w[i].
func (w Word) Flip(i int) Word {
	return w ^ exp2[i]
}

// Rank1 returns the number of ones in w[0]...w[i].
func (w Word) Rank1(i int) int {
	w = w << uint(W-i-1)
	return w.Count1()
}

// Rank0 returns the number of zeros in w[0]...w[i].
func (w Word) Rank0(i int) int {
	w = ^w << uint(W-i-1)
	return w.Count1()
}
<commit_msg>Refactor code involving shifts<commit_after>// Package bitutils provides a collection of utilities to deal with bits.
package bitutils

import (
	"fmt"
)

// W is the length of a machine word.
const W = 64

// Magic constants.
const (
	lowers2 = 0x5555555555555555
	lowers4 = 0x3333333333333333
	lowers8 = 0x0f0f0f0f0f0f0f0f
	lowest8 = 0x0101010101010101
)

// Word represents a 64-bit binary string.
type Word uint64

var (
	// shift[i] is (1 << i).
	shift [W]Word
	// shiftNot[i] is ^shift[i].
	shiftNot [W]Word
)

func init() {
	for i := 0; i < len(shift); i++ {
		shift[i] = Word(1) << uint(i)
		shiftNot[i] = ^shift[i]
	}
}

// String returns binary string w[0]w[1]...w[63].
func (w Word) String() string {
	return fmt.Sprintf("%064b", w)
}

// Count1 returns the number of ones contained in w.
func (w Word) Count1() int {
	w -= (w >> 1) & lowers2
	w = (w & lowers4) + ((w >> 2) & lowers4)
	w = (w + (w >> 4)) & lowers8
	return int((w * lowest8) >> 56)
}

// Count0 returns the number of zeros contained in w.
func (w Word) Count0() int {
	w = ^w
	return w.Count1()
}

// Count returns the number of b[0]'s contained in w.
func (w Word) Count(b int) int {
	w = w ^ (^Word(0) + Word(b))
	return w.Count1()
}

// Get returns w[i].
func (w Word) Get(i int) Word {
	w = w >> uint(i)
	return w & shift[0]
}

// Set1 sets w[i] to 1.
func (w Word) Set1(i int) Word {
	return w | shift[i]
}

// Set0 sets w[i] to 0.
func (w Word) Set0(i int) Word {
	return w & shiftNot[i]
}

// Flip flips w[i].
func (w Word) Flip(i int) Word {
	return w ^ shift[i]
}

// Rank1 returns the number of ones in w[0]...w[i].
func (w Word) Rank1(i int) int {
	w = w << uint(W-i-1)
	return w.Count1()
}

// Rank0 returns the number of zeros in w[0]...w[i].
func (w Word) Rank0(i int) int {
	w = ^w << uint(W-i-1)
	return w.Count1()
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package conventions

// OpenTelemetry semantic convention values for AWS-specific resource attributes
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/cloud_provider/aws/README.md
const (
	AttributeAWSECSContainerARN = "aws.ecs.container.arn"
	AttributeAWSECSClusterARN   = "aws.ecs.cluster.arn"
	AttributeAWSECSLaunchType   = "aws.ecs.launchtype"
	AttributeAWSECSTaskARN      = "aws.ecs.task.arn"
	AttributeAWSECSTaskFamily   = "aws.ecs.task.family"
	AttributeAWSLogGroupNames   = "aws.log.group.names"
	AttributeAWSLogGroupARNs    = "aws.log.group.arns"
	AttributeAWSLogStreamNames  = "aws.log.stream.names"
	AttributeAWSLogStreamARNs   = "aws.log.stream.arns"
)

// OpenTelemetry Semantic Convention values for Resource attribute "aws.ecs.launchtype" values.
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/cloud_provider/aws/ecs.md
const (
	AttributeAWSECSLaunchTypeEC2     = "ec2"
	AttributeAWSECSLaunchTypeFargate = "fargate"
)
<commit_msg>Add `aws.ecs.task.revision` (#2816)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package conventions

// OpenTelemetry semantic convention values for AWS-specific resource attributes
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/cloud_provider/aws/README.md
const (
	AttributeAWSECSContainerARN = "aws.ecs.container.arn"
	AttributeAWSECSClusterARN   = "aws.ecs.cluster.arn"
	AttributeAWSECSLaunchType   = "aws.ecs.launchtype"
	AttributeAWSECSTaskARN      = "aws.ecs.task.arn"
	AttributeAWSECSTaskFamily   = "aws.ecs.task.family"
	AttributeAWSECSTaskRevision = "aws.ecs.task.revision"
	AttributeAWSLogGroupNames   = "aws.log.group.names"
	AttributeAWSLogGroupARNs    = "aws.log.group.arns"
	AttributeAWSLogStreamNames  = "aws.log.stream.names"
	AttributeAWSLogStreamARNs   = "aws.log.stream.arns"
)

// OpenTelemetry Semantic Convention values for Resource attribute "aws.ecs.launchtype" values.
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/cloud_provider/aws/ecs.md
const (
	AttributeAWSECSLaunchTypeEC2     = "ec2"
	AttributeAWSECSLaunchTypeFargate = "fargate"
)
<|endoftext|>


<commit_before>// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package core provides transport-agnostic implementation of Migrillian tool.
package core

import (
	"bytes"
	"context"
	"fmt"
	"strconv"
	"sync"

	"github.com/golang/glog"

	ct "github.com/google/certificate-transparency-go"
	"github.com/google/certificate-transparency-go/client"
	"github.com/google/certificate-transparency-go/scanner"
	"github.com/google/certificate-transparency-go/trillian/migrillian/configpb"

	"github.com/google/trillian/merkle"
	_ "github.com/google/trillian/merkle/rfc6962" // Register hasher.
	"github.com/google/trillian/monitoring"
	"github.com/google/trillian/types"
	"github.com/google/trillian/util/election2"
)

var (
	metrics     treeMetrics
	metricsOnce sync.Once
)

// treeMetrics holds metrics keyed by Tree ID.
type treeMetrics struct {
	masterRuns     monitoring.Counter
	masterCancels  monitoring.Counter
	isMaster       monitoring.Gauge
	entriesFetched monitoring.Counter
	entriesSeen    monitoring.Counter
	entriesStored  monitoring.Counter
	// TODO(pavelkalinnikov): Add latency histograms, latest STH, tree size, etc.
}

// initMetrics creates metrics using the factory, if not yet created.
func initMetrics(mf monitoring.MetricFactory) {
	const treeID = "tree_id"
	metricsOnce.Do(func() {
		metrics = treeMetrics{
			masterRuns:     mf.NewCounter("master_runs", "Number of mastership runs.", treeID),
			masterCancels:  mf.NewCounter("master_cancels", "Number of unexpected mastership cancelations.", treeID),
			isMaster:       mf.NewGauge("is_master", "The instance is currently the master.", treeID),
			entriesFetched: mf.NewCounter("entries_fetched", "Entries fetched from the source log.", treeID),
			entriesSeen:    mf.NewCounter("entries_seen", "Entries seen by the submitters.", treeID),
			entriesStored:  mf.NewCounter("entries_stored", "Entries successfully submitted to Trillian.", treeID),
		}
	})
}

// Options holds configuration for a Controller.
type Options struct {
	scanner.FetcherOptions
	Submitters         int
	ChannelSize        int
	NoConsistencyCheck bool
}

// OptionsFromConfig returns Options created from the passed in config.
func OptionsFromConfig(cfg *configpb.MigrationConfig) Options {
	opts := Options{
		FetcherOptions: scanner.FetcherOptions{
			BatchSize:     int(cfg.BatchSize),
			ParallelFetch: int(cfg.NumFetchers),
			StartIndex:    cfg.StartIndex,
			EndIndex:      cfg.EndIndex,
			Continuous:    cfg.IsContinuous,
		},
		Submitters:         int(cfg.NumSubmitters),
		ChannelSize:        int(cfg.ChannelSize),
		NoConsistencyCheck: cfg.NoConsistencyCheck,
	}
	if cfg.NumFetchers == 0 {
		opts.ParallelFetch = 1
	}
	if cfg.NumSubmitters == 0 {
		opts.Submitters = 1
	}
	return opts
}

// Controller coordinates migration from a CT log to a Trillian tree.
//
// TODO(pavelkalinnikov):
// - Schedule a distributed fetch to increase throughput.
// - Store CT STHs in Trillian or make this tool stateful on its own.
// - Make fetching stateful to reduce master resigning aftermath.
type Controller struct {
	opts     Options
	batches  chan scanner.EntryBatch
	ctClient *client.LogClient
	plClient *PreorderedLogClient
	ef       election2.Factory
	label    string
}

// NewController creates a Controller configured by the passed in options, CT
// and Trillian clients, and a master election factory.
//
// The passed in MetricFactory is used to create per-tree metrics, and it
// should be the same for all instances. However, it is used only once.
func NewController(
	opts Options,
	ctClient *client.LogClient,
	plClient *PreorderedLogClient,
	ef election2.Factory,
	mf monitoring.MetricFactory,
) *Controller {
	initMetrics(mf)
	l := strconv.FormatInt(plClient.tree.TreeId, 10)
	return &Controller{opts: opts, ctClient: ctClient, plClient: plClient, ef: ef, label: l}
}

// RunWhenMaster is a master-elected version of Run method. It executes Run
// whenever this instance captures mastership of the tree ID. As soon as the
// instance stops being the master, Run is canceled. The method returns if a
// severe error occurs, the passed in context is canceled, or fetching is
// completed (in non-Continuous mode). Releases mastership when terminates.
func (c *Controller) RunWhenMaster(ctx context.Context) error {
	treeID := strconv.FormatInt(c.plClient.tree.TreeId, 10)

	el, err := c.ef.NewElection(ctx, treeID)
	if err != nil {
		return err
	}
	defer func(ctx context.Context) {
		metrics.isMaster.Set(0, c.label)
		if err := el.Close(ctx); err != nil {
			glog.Warningf("%s: Election.Close(): %v", treeID, err)
		}
	}(ctx)

	for {
		if err := el.Await(ctx); err != nil {
			return err
		}
		metrics.isMaster.Set(1, c.label)

		mctx, err := el.WithMastership(ctx)
		if err != nil {
			return err
		} else if err := mctx.Err(); err != nil {
			return err
		}

		glog.Infof("%s: running as master", treeID)
		metrics.masterRuns.Inc(c.label)

		// Run while still master (or until an error).
		err = c.Run(mctx)
		if ctx.Err() != nil {
			// We have been externally canceled, so return the current error (which
			// could be nil or a cancelation-related error).
			return err
		} else if mctx.Err() == nil {
			// We are still the master, so emit the real error.
			return err
		}

		// Otherwise the mastership has been canceled, retry.
		metrics.isMaster.Set(0, c.label)
		metrics.masterCancels.Inc(c.label)
	}
}

// Run transfers CT log entries obtained via the CT log client to a Trillian
// pre-ordered log via Trillian client. If Options.Continuous is true then the
// migration process runs continuously trying to keep up with the target CT
// log. Returns if an error occurs, the context is canceled, or all the entries
// have been transferred (in non-Continuous mode).
func (c *Controller) Run(ctx context.Context) error {
	treeID := c.plClient.tree.TreeId

	root, err := c.plClient.getVerifiedRoot(ctx)
	if err != nil {
		return err
	}
	if c.opts.Continuous { // Ignore range parameters in Continuous mode.
		// TODO(pavelkalinnikov): Restore fetching state from storage in a better
		// way than "take the current tree size".
		c.opts.StartIndex, c.opts.EndIndex = int64(root.TreeSize), 0
		glog.Warningf("%d: updated entry range to [%d, INF)", treeID, c.opts.StartIndex)
	} else if c.opts.StartIndex < 0 {
		c.opts.StartIndex = int64(root.TreeSize)
		glog.Warningf("%d: updated start index to %d", treeID, c.opts.StartIndex)
	}

	fetcher := scanner.NewFetcher(c.ctClient, &c.opts.FetcherOptions)
	sth, err := fetcher.Prepare(ctx)
	if err != nil {
		return err
	}
	if err := c.verifyConsistency(ctx, root, sth); err != nil {
		return err
	}

	var wg sync.WaitGroup
	c.batches = make(chan scanner.EntryBatch, c.opts.ChannelSize)
	cctx, cancel := context.WithCancel(ctx)
	defer cancel()

	// TODO(pavelkalinnikov): Share the submitters pool between multiple trees.
	for w, cnt := 0, c.opts.Submitters; w < cnt; w++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			if err := c.runSubmitter(cctx); err != nil {
				glog.Errorf("%d: Stopping due to submitter error: %v", treeID, err)
				cancel() // Stop the other submitters and the Fetcher.
			}
		}()
	}

	handler := func(b scanner.EntryBatch) {
		metrics.entriesFetched.Add(float64(len(b.Entries)), c.label)
		c.batches <- b
	}
	result := fetcher.Run(cctx, handler)
	close(c.batches)
	wg.Wait()
	return result
}

// verifyConsistency checks that the provided verified Trillian root is
// consistent with the CT log's STH.
func (c *Controller) verifyConsistency(ctx context.Context, root *types.LogRootV1, sth *ct.SignedTreeHead) error {
	h := c.plClient.verif.Hasher
	if root.TreeSize == 0 {
		if got, want := root.RootHash, h.EmptyRoot(); !bytes.Equal(got, want) {
			return fmt.Errorf("invalid empty tree hash %x, want %x", got, want)
		}
		return nil
	}
	if c.opts.NoConsistencyCheck {
		glog.Warningf("%s: skipping consistency check", c.label)
		return nil
	}

	resp, err := c.ctClient.GetEntryAndProof(ctx, root.TreeSize-1, sth.TreeSize)
	if err != nil {
		return err
	}
	leafHash, err := h.HashLeaf(resp.LeafInput)
	if err != nil {
		return err
	}

	hash, err := merkle.NewLogVerifier(h).VerifiedPrefixHashFromInclusionProof(
		int64(root.TreeSize), int64(sth.TreeSize),
		resp.AuditPath, sth.SHA256RootHash[:], leafHash)
	if err != nil {
		return err
	}

	if got := root.RootHash; !bytes.Equal(got, hash) {
		return fmt.Errorf("inconsistent root hash %x, want %x", got, hash)
	}
	return nil
}

// runSubmitter obtains CT log entry batches from the controller's channel and
// submits them through Trillian client. Returns when the channel is closed, or
// the client returns a non-recoverable error (an example of a recoverable
// error is when Trillian write quota is exceeded).
func (c *Controller) runSubmitter(ctx context.Context) error {
	treeID := c.plClient.tree.TreeId
	for b := range c.batches {
		entries := float64(len(b.Entries))
		metrics.entriesSeen.Add(entries, c.label)

		end := b.Start + int64(len(b.Entries))
		if err := c.plClient.addSequencedLeaves(ctx, &b); err != nil {
			// addSequencedLeaves failed to submit entries despite retries. At this
			// point there is not much we can do. Seemingly the best strategy is to
			// shut down the Controller.
			// TODO(pavelkalinnikov): Restart Controller and/or expose some metrics
			// allowing a log operator to set up alerts and react accordingly.
			return fmt.Errorf("failed to add batch [%d, %d): %v", b.Start, end, err)
		}
		glog.Infof("%d: added batch [%d, %d)", treeID, b.Start, end)
		metrics.entriesStored.Add(entries, c.label)
	}
	return nil
}
<commit_msg>[Migrillian] Fix deadlock when terminating submitters (#447)<commit_after>// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package core provides transport-agnostic implementation of Migrillian tool.
package core

import (
	"bytes"
	"context"
	"fmt"
	"strconv"
	"sync"

	"github.com/golang/glog"

	ct "github.com/google/certificate-transparency-go"
	"github.com/google/certificate-transparency-go/client"
	"github.com/google/certificate-transparency-go/scanner"
	"github.com/google/certificate-transparency-go/trillian/migrillian/configpb"

	"github.com/google/trillian/merkle"
	_ "github.com/google/trillian/merkle/rfc6962" // Register hasher.
	"github.com/google/trillian/monitoring"
	"github.com/google/trillian/types"
	"github.com/google/trillian/util/election2"
)

var (
	metrics     treeMetrics
	metricsOnce sync.Once
)

// treeMetrics holds metrics keyed by Tree ID.
type treeMetrics struct {
	masterRuns     monitoring.Counter
	masterCancels  monitoring.Counter
	isMaster       monitoring.Gauge
	entriesFetched monitoring.Counter
	entriesSeen    monitoring.Counter
	entriesStored  monitoring.Counter
	// TODO(pavelkalinnikov): Add latency histograms, latest STH, tree size, etc.
}

// initMetrics creates metrics using the factory, if not yet created.
func initMetrics(mf monitoring.MetricFactory) {
	const treeID = "tree_id"
	metricsOnce.Do(func() {
		metrics = treeMetrics{
			masterRuns:     mf.NewCounter("master_runs", "Number of mastership runs.", treeID),
			masterCancels:  mf.NewCounter("master_cancels", "Number of unexpected mastership cancelations.", treeID),
			isMaster:       mf.NewGauge("is_master", "The instance is currently the master.", treeID),
			entriesFetched: mf.NewCounter("entries_fetched", "Entries fetched from the source log.", treeID),
			entriesSeen:    mf.NewCounter("entries_seen", "Entries seen by the submitters.", treeID),
			entriesStored:  mf.NewCounter("entries_stored", "Entries successfully submitted to Trillian.", treeID),
		}
	})
}

// Options holds configuration for a Controller.
type Options struct {
	scanner.FetcherOptions
	Submitters         int
	ChannelSize        int
	NoConsistencyCheck bool
}

// OptionsFromConfig returns Options created from the passed in config.
func OptionsFromConfig(cfg *configpb.MigrationConfig) Options {
	opts := Options{
		FetcherOptions: scanner.FetcherOptions{
			BatchSize:     int(cfg.BatchSize),
			ParallelFetch: int(cfg.NumFetchers),
			StartIndex:    cfg.StartIndex,
			EndIndex:      cfg.EndIndex,
			Continuous:    cfg.IsContinuous,
		},
		Submitters:         int(cfg.NumSubmitters),
		ChannelSize:        int(cfg.ChannelSize),
		NoConsistencyCheck: cfg.NoConsistencyCheck,
	}
	if cfg.NumFetchers == 0 {
		opts.ParallelFetch = 1
	}
	if cfg.NumSubmitters == 0 {
		opts.Submitters = 1
	}
	return opts
}

// Controller coordinates migration from a CT log to a Trillian tree.
//
// TODO(pavelkalinnikov):
// - Schedule a distributed fetch to increase throughput.
// - Store CT STHs in Trillian or make this tool stateful on its own.
// - Make fetching stateful to reduce master resigning aftermath.
type Controller struct {
	opts     Options
	batches  chan scanner.EntryBatch
	ctClient *client.LogClient
	plClient *PreorderedLogClient
	ef       election2.Factory
	label    string
}

// NewController creates a Controller configured by the passed in options, CT
// and Trillian clients, and a master election factory.
//
// The passed in MetricFactory is used to create per-tree metrics, and it
// should be the same for all instances. However, it is used only once.
func NewController(
	opts Options,
	ctClient *client.LogClient,
	plClient *PreorderedLogClient,
	ef election2.Factory,
	mf monitoring.MetricFactory,
) *Controller {
	initMetrics(mf)
	l := strconv.FormatInt(plClient.tree.TreeId, 10)
	return &Controller{opts: opts, ctClient: ctClient, plClient: plClient, ef: ef, label: l}
}

// RunWhenMaster is a master-elected version of Run method. It executes Run
// whenever this instance captures mastership of the tree ID. As soon as the
// instance stops being the master, Run is canceled. The method returns if a
// severe error occurs, the passed in context is canceled, or fetching is
// completed (in non-Continuous mode). Releases mastership when terminates.
func (c *Controller) RunWhenMaster(ctx context.Context) error {
	treeID := strconv.FormatInt(c.plClient.tree.TreeId, 10)

	el, err := c.ef.NewElection(ctx, treeID)
	if err != nil {
		return err
	}
	defer func(ctx context.Context) {
		metrics.isMaster.Set(0, c.label)
		if err := el.Close(ctx); err != nil {
			glog.Warningf("%s: Election.Close(): %v", treeID, err)
		}
	}(ctx)

	for {
		if err := el.Await(ctx); err != nil {
			return err
		}
		metrics.isMaster.Set(1, c.label)

		mctx, err := el.WithMastership(ctx)
		if err != nil {
			return err
		} else if err := mctx.Err(); err != nil {
			return err
		}

		glog.Infof("%s: running as master", treeID)
		metrics.masterRuns.Inc(c.label)

		// Run while still master (or until an error).
		err = c.Run(mctx)
		if ctx.Err() != nil {
			// We have been externally canceled, so return the current error (which
			// could be nil or a cancelation-related error).
			return err
		} else if mctx.Err() == nil {
			// We are still the master, so emit the real error.
			return err
		}

		// Otherwise the mastership has been canceled, retry.
		metrics.isMaster.Set(0, c.label)
		metrics.masterCancels.Inc(c.label)
	}
}

// Run transfers CT log entries obtained via the CT log client to a Trillian
// pre-ordered log via Trillian client. If Options.Continuous is true then the
// migration process runs continuously trying to keep up with the target CT
// log. Returns if an error occurs, the context is canceled, or all the entries
// have been transferred (in non-Continuous mode).
func (c *Controller) Run(ctx context.Context) error {
	treeID := c.plClient.tree.TreeId

	root, err := c.plClient.getVerifiedRoot(ctx)
	if err != nil {
		return err
	}
	if c.opts.Continuous { // Ignore range parameters in Continuous mode.
		// TODO(pavelkalinnikov): Restore fetching state from storage in a better
		// way than "take the current tree size".
		c.opts.StartIndex, c.opts.EndIndex = int64(root.TreeSize), 0
		glog.Warningf("%d: updated entry range to [%d, INF)", treeID, c.opts.StartIndex)
	} else if c.opts.StartIndex < 0 {
		c.opts.StartIndex = int64(root.TreeSize)
		glog.Warningf("%d: updated start index to %d", treeID, c.opts.StartIndex)
	}

	fetcher := scanner.NewFetcher(c.ctClient, &c.opts.FetcherOptions)
	sth, err := fetcher.Prepare(ctx)
	if err != nil {
		return err
	}
	if err := c.verifyConsistency(ctx, root, sth); err != nil {
		return err
	}

	var wg sync.WaitGroup
	c.batches = make(chan scanner.EntryBatch, c.opts.ChannelSize)
	cctx, cancel := context.WithCancel(ctx)
	defer cancel()

	// TODO(pavelkalinnikov): Share the submitters pool between multiple trees.
	for w, cnt := 0, c.opts.Submitters; w < cnt; w++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			if err := c.runSubmitter(cctx); err != nil {
				glog.Errorf("%d: Stopping due to submitter error: %v", treeID, err)
				cancel() // Stop the other submitters and the Fetcher.
			}
		}()
	}

	handler := func(b scanner.EntryBatch) {
		metrics.entriesFetched.Add(float64(len(b.Entries)), c.label)
		select {
		case c.batches <- b:
		case <-cctx.Done(): // Avoid deadlock when shutting down.
		}
	}
	result := fetcher.Run(cctx, handler)
	close(c.batches)
	wg.Wait()
	return result
}

// verifyConsistency checks that the provided verified Trillian root is
// consistent with the CT log's STH.
func (c *Controller) verifyConsistency(ctx context.Context, root *types.LogRootV1, sth *ct.SignedTreeHead) error {
	h := c.plClient.verif.Hasher
	if root.TreeSize == 0 {
		if got, want := root.RootHash, h.EmptyRoot(); !bytes.Equal(got, want) {
			return fmt.Errorf("invalid empty tree hash %x, want %x", got, want)
		}
		return nil
	}
	if c.opts.NoConsistencyCheck {
		glog.Warningf("%s: skipping consistency check", c.label)
		return nil
	}

	resp, err := c.ctClient.GetEntryAndProof(ctx, root.TreeSize-1, sth.TreeSize)
	if err != nil {
		return err
	}
	leafHash, err := h.HashLeaf(resp.LeafInput)
	if err != nil {
		return err
	}

	hash, err := merkle.NewLogVerifier(h).VerifiedPrefixHashFromInclusionProof(
		int64(root.TreeSize), int64(sth.TreeSize),
		resp.AuditPath, sth.SHA256RootHash[:], leafHash)
	if err != nil {
		return err
	}

	if got := root.RootHash; !bytes.Equal(got, hash) {
		return fmt.Errorf("inconsistent root hash %x, want %x", got, hash)
	}
	return nil
}

// runSubmitter obtains CT log entry batches from the controller's channel and
// submits them through Trillian client. Returns when the channel is closed, or
// the client returns a non-recoverable error (an example of a recoverable
// error is when Trillian write quota is exceeded).
func (c *Controller) runSubmitter(ctx context.Context) error {
	treeID := c.plClient.tree.TreeId
	for b := range c.batches {
		entries := float64(len(b.Entries))
		metrics.entriesSeen.Add(entries, c.label)

		end := b.Start + int64(len(b.Entries))
		if err := c.plClient.addSequencedLeaves(ctx, &b); err != nil {
			// addSequencedLeaves failed to submit entries despite retries. At this
			// point there is not much we can do. Seemingly the best strategy is to
			// shut down the Controller.
			// TODO(pavelkalinnikov): Restart Controller and/or expose some metrics
			// allowing a log operator to set up alerts and react accordingly.
			return fmt.Errorf("failed to add batch [%d, %d): %v", b.Start, end, err)
		}
		glog.Infof("%d: added batch [%d, %d)", treeID, b.Start, end)
		metrics.entriesStored.Add(entries, c.label)
	}
	return nil
}
<|endoftext|>


<commit_before>package apostle

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
)

var (
	conf = loadConfig()
)

func Send(templateId string, email string, name string, data map[string]string) error {
	// generate json request body
	m := map[string]interface{}{
		"recipients": map[string]interface{}{
			email: map[string]interface{}{
				"template_id": templateId,
				"name":        name,
				"data":        data,
			},
		},
	}
	requestJson, err := json.Marshal(m)
	if err != nil {
		return err
	}
	log.Printf("Sending Apostle: %v", string(requestJson))
	postData := bytes.NewBuffer(requestJson)

	// prepare request
	req, err := http.NewRequest("POST", "http://deliver.apostle.io", postData)

	// set appropriate headers for auth/content type etc
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", os.Getenv("APOSTLE_BEARER_TOKEN")))

	// do request
	response, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	// apostle returns 202 Accepted if it's gonna send
	if response.StatusCode != http.StatusAccepted {
		return errors.New(fmt.Sprintf("HTTP error from apostle of code %v", response.StatusCode))
	}
	readBody, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Print(err)
	}
	log.Printf("Apostle returned: %v", string(readBody))

	return err
}
<commit_msg>Remove old apostle.go code<commit_after>package apostle

var (
	conf = loadConfig()
)
<|endoftext|>


<commit_before>package asm

import (
	"testing"
)

// These are the old names, retained here to check what
// changes have been made.
func TestOpCodeString(t *testing.T) {
	t.Skip()

	testcases := map[string]OpCode{
		// AddImm  add dst, imm   |  dst += imm
		"AddImm": 0x07,
		// AddSrc  add dst, src   |  dst += src
		"AddSrc": 0x0f,
		// SubImm  sub dst, imm   |  dst -= imm
		"SubImm": 0x17,
		// SubSrc  sub dst, src   |  dst -= src
		"SubSrc": 0x1f,
		// MulImm  mul dst, imm   |  dst *= imm
		"MulImm": 0x27,
		// MulSrc  mul dst, src   |  dst *= src
		"MulSrc": 0x2f,
		// DivImm  div dst, imm   |  dst /= imm
		"DivImm": 0x37,
		// DivSrc  div dst, src   |  dst /= src
		"DivSrc": 0x3f,
		// OrImm   or dst, imm    |  dst  |= imm
		"OrImm": 0x47,
		// OrSrc   or dst, src    |  dst  |= src
		"OrSrc": 0x4f,
		// AndImm  and dst, imm   |  dst &= imm
		"AndImm": 0x57,
		// AndSrc  and dst, src   |  dst &= src
		"AndSrc": 0x5f,
		// LShImm  lsh dst, imm   |  dst <<= imm
		"LShImm": 0x67,
		// LShSrc  lsh dst, src   |  dst <<= src
		"LShSrc": 0x6f,
		// RShImm  rsh dst, imm   |  dst >>= imm (logical)
		"RShImm": 0x77,
		// RShSrc  rsh dst, src   |  dst >>= src (logical)
		"RShSrc": 0x7f,
		// Neg     neg dst        |  dst = -dst
		"Neg": 0x87,
		// ModImm  mod dst, imm   |  dst %= imm
		"ModImm": 0x97,
		// ModSrc  mod dst, src   |  dst %= src
		"ModSrc": 0x9f,
		// XorImm  xor dst, imm   |  dst ^= imm
		"XorImm": 0xa7,
		// XorSrc  xor dst, src   |  dst ^= src
		"XorSrc": 0xaf,
		// MovImm  mov dst, imm   |  dst = imm
		"MovImm": 0xb7,
		// MovSrc  mov dst, src   |  dst = src
		"MovSrc": 0xbf,
		// ArShImm arsh dst, imm  |  dst >>= imm (arithmetic)
		"ArShImm": 0xc7,
		// ArShSrc arsh dst, src  |  dst >>= src (arithmetic)
		"ArShSrc": 0xcf,
		// Add32Imm add32 dst, imm  |  dst += imm
		"Add32Imm": 0x04,
		// Add32Src add32 dst, src  |  dst += src
		"Add32Src": 0x0c,
		// Sub32Imm sub32 dst, imm  |  dst -= imm
		"Sub32Imm": 0x14,
		// Sub32Src sub32 dst, src  |  dst -= src
		"Sub32Src": 0x1c,
		// Mul32Imm mul32 dst, imm  |  dst *= imm
		"Mul32Imm": 0x24,
		// Mul32Src mul32 dst, src  |  dst *= src
		"Mul32Src": 0x2c,
		// Div32Imm div32 dst, imm  |  dst /= imm
		"Div32Imm": 0x34,
		// Div32Src div32 dst, src  |  dst /= src
		"Div32Src": 0x3c,
		// Or32Imm  or32 dst, imm   |  dst |= imm
		"Or32Imm": 0x44,
		// Or32Src  or32 dst, src   |  dst |= src
		"Or32Src": 0x4c,
		// And32Imm and32 dst, imm  |  dst &= imm
		"And32Imm": 0x54,
		// And32Src and32 dst, src  |  dst &= src
		"And32Src": 0x5c,
		// LSh32Imm lsh32 dst, imm  |  dst <<= imm
		"LSh32Imm": 0x64,
		// LSh32Src lsh32 dst, src  |  dst <<= src
		"LSh32Src": 0x6c,
		// RSh32Imm rsh32 dst, imm  |  dst >>= imm (logical)
		"RSh32Imm": 0x74,
		// RSh32Src rsh32 dst, src  |  dst >>= src (logical)
		"RSh32Src": 0x7c,
		// Neg32    neg32 dst       |  dst = -dst
		"Neg32": 0x84,
		// Mod32Imm mod32 dst, imm  |  dst %= imm
		"Mod32Imm": 0x94,
		// Mod32Src mod32 dst, src  |  dst %= src
		"Mod32Src": 0x9c,
		// Xor32Imm xor32 dst, imm  |  dst ^= imm
		"Xor32Imm": 0xa4,
		// Xor32Src xor32 dst, src  |  dst ^= src
		"Xor32Src": 0xac,
		// Mov32Imm mov32 dst, imm  |  dst eBPF only
		"Mov32Imm": 0xb4,
		// Mov32Src mov32 dst, src  |  dst eBPF only
		"Mov32Src": 0xbc,
		// LE16 le16 dst, imm == 16  |  dst = htole16(dst)
		"LE16": 0xd4,
		// LE32 le32 dst, imm == 32  |  dst = htole32(dst)
		"LE32": 0xd4,
		// LE64 le64 dst, imm == 64  |  dst = htole64(dst)
		"LE64": 0xd4,
		// BE16 be16 dst, imm == 16  |  dst = htobe16(dst)
		"BE16": 0xdc,
		// BE32 be32 dst, imm == 32  |  dst = htobe32(dst)
		"BE32": 0xdc,
		// BE64 be64 dst, imm == 64  |  dst = htobe64(dst)
		"BE64": 0xdc,
		// LdDW      lddw (src), dst, imm   |  dst = imm
		"LdDW": 0x18,
		// XAddStSrc xadd dst, src          |  *dst += src
		"XAddStSrc": 0xdb,
		// LdAbsB    ldabsb imm             |  r0 = (uint8_t *) (mem + imm)
		"LdAbsB": 0x30,
		// LdXW      ldxw dst, [src+off]    |  dst = *(uint32_t *) (src + off)
		"LdXW": 0x61,
		// LdXH      ldxh dst, [src+off]    |  dst = *(uint16_t *) (src + off)
		"LdXH": 0x69,
		// LdXB      ldxb dst, [src+off]    |  dst = *(uint8_t *) (src + off)
		"LdXB": 0x71,
		// LdXDW     ldxdw dst, [src+off]   |  dst = *(uint64_t *) (src + off)
		"LdXDW": 0x79,
		// StB       stb [dst+off], imm     |  *(uint8_t *) (dst + off) = imm
		"StB": 0x72,
		// StH       sth [dst+off], imm     |  *(uint16_t *) (dst + off) = imm
		"StH": 0x6a,
		// StW       stw [dst+off], imm     |  *(uint32_t *) (dst + off) = imm
		"StW": 0x62,
		// StDW      stdw [dst+off], imm    |  *(uint64_t *) (dst + off) = imm
		"StDW": 0x7a,
		// StXB      stxb [dst+off], src    |  *(uint8_t *) (dst + off) = src
		"StXB": 0x73,
		// StXH      stxh [dst+off], src    |  *(uint16_t *) (dst + off) = src
		"StXH": 0x6b,
		// StXW      stxw [dst+off], src    |  *(uint32_t *) (dst + off) = src
		"StXW": 0x63,
		// StXDW     stxdw [dst+off], src   |  *(uint64_t *) (dst + off) = src
		"StXDW": 0x7b,
		// LdAbsH  ldabsh imm             |  r0 = (uint16_t *) (imm)
		// Abs and Ind reference memory directly. This is always the context,
		// of whatever the eBPF program is. For example in a sock filter program
		// the memory context is the sk_buff struct.
		"LdAbsH": 0x28,
		// LdAbsW  ldabsw imm             |  r0 = (uint32_t *) (imm)
		"LdAbsW": 0x20,
		// LdAbsDW ldabsdw imm            |  r0 = (uint64_t *) (imm)
		"LdAbsDW": 0x38,
		// LdIndB  ldindb src, dst, imm   |  dst = (uint64_t *) (src + imm)
		"LdIndB": 0x50,
		// LdIndH  ldindh src, dst, imm   |  dst = (uint16_t *) (src + imm)
		"LdIndH": 0x48,
		// LdIndW  ldindw src, dst, imm   |  dst = (uint32_t *) (src + imm)
		"LdIndW": 0x40,
		// LdIndDW ldinddw src, dst, imm  |  dst = (uint64_t *) (src + imm)
		"LdIndDW": 0x58,
		// Ja      ja +off             |  PC += off
		"Ja": 0x05,
		// JEqImm  jeq dst, imm, +off  |  PC += off if dst == imm
		"JEqImm": 0x15,
		// JEqSrc  jeq dst, src, +off  |  PC += off if dst == src
		"JEqSrc": 0x1d,
		// JGTImm  jgt dst, imm, +off  |  PC += off if dst > imm
		"JGTImm": 0x25,
		// JGTSrc  jgt dst, src, +off  |  PC += off if dst > src
		"JGTSrc": 0x2d,
		// JGEImm  jge dst, imm, +off  |  PC += off if dst >= imm
		"JGEImm": 0x35,
		// JGESrc  jge dst, src, +off  |  PC += off if dst >= src
		"JGESrc": 0x3d,
		// JSETImm jset dst, imm, +off |  PC += off if dst & imm
		"JSETImm": 0x45,
		// JSETSrc jset dst, src, +off |  PC += off if dst & src
		"JSETSrc": 0x4d,
		// JNEImm  jne dst, imm, +off  |  PC += off if dst != imm
		"JNEImm": 0x55,
		// JNESrc  jne dst, src, +off  |  PC += off if dst != src
		"JNESrc": 0x5d,
		// JSGTImm jsgt dst, imm, +off |  PC += off if dst > imm (signed)
		"JSGTImm": 0x65,
		// JSGTSrc jsgt dst, src, +off |  PC += off if dst > src (signed)
		"JSGTSrc": 0x6d,
		// JSGEImm jsge dst, imm, +off |  PC += off if dst >= imm (signed)
		"JSGEImm": 0x75,
		// JSGESrc jsge dst, src, +off |  PC += off if dst >= src (signed)
		"JSGESrc": 0x7d,
		// Call    call imm            |  Function call
		"Call": 0x85,
		// Exit    exit                |  return r0
		"Exit": 0x95,
	}

	for want, op := range testcases {
		if have := op.String(); want != have {
			t.Errorf("Expected %s, got %s", want, have)
		}
	}
}
<commit_msg>asm: remove old opcode test<commit_after><|endoftext|>