<commit_before>package recipe

import (
	"github.com/watermint/toolbox/infra/app"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/infra/report/rp_model"
	"github.com/watermint/toolbox/infra/ui/app_msg"
	"runtime"
)

type VersionInfo struct {
	Key       string `json:"key"`
	Component string `json:"component"`
	Version   string `json:"version"`
}

type Version struct {
	Versions        rp_model.RowReport
	HeaderAppHash   app_msg.Message
	HeaderBuildTime app_msg.Message
	HeaderGoVersion app_msg.Message
}

func (z *Version) Preset() {
	z.Versions.SetModel(&VersionInfo{})
}

func (z *Version) Exec(c app_control.Control) error {
	ui := c.UI()
	if err := z.Versions.Open(); err != nil {
		return err
	}
	z.Versions.Row(&VersionInfo{
		Key:       "app.name",
		Component: app.Name,
		Version:   app.Version,
	})
	z.Versions.Row(&VersionInfo{
		Key:       "app.hash",
		Component: ui.Text(z.HeaderAppHash),
		Version:   app.Hash,
	})
	z.Versions.Row(&VersionInfo{
		Key:       "build.time",
		Component: ui.Text(z.HeaderBuildTime),
		Version:   app.BuildTimestamp,
	})
	z.Versions.Row(&VersionInfo{
		Key:       "go.version",
		Component: ui.Text(z.HeaderGoVersion),
		Version:   runtime.Version(),
	})
	return nil
}

func (z *Version) Test(c app_control.Control) error {
	return rc_exec.Exec(c, &Version{}, rc_recipe.NoCustomValues)
}
<commit_msg>#336 : fix name of the key<commit_after>package recipe

import (
	"github.com/watermint/toolbox/infra/app"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/infra/report/rp_model"
	"github.com/watermint/toolbox/infra/ui/app_msg"
	"runtime"
)

type VersionInfo struct {
	Key       string `json:"key"`
	Component string `json:"component"`
	Version   string `json:"version"`
}

type Version struct {
	Versions        rp_model.RowReport
	HeaderAppHash   app_msg.Message
	HeaderBuildTime app_msg.Message
	HeaderGoVersion app_msg.Message
}

func (z *Version) Preset() {
	z.Versions.SetModel(&VersionInfo{})
}

func (z *Version) Exec(c app_control.Control) error {
	ui := c.UI()
	if err := z.Versions.Open(); err != nil {
		return err
	}
	z.Versions.Row(&VersionInfo{
		Key:       "app.version",
		Component: app.Name,
		Version:   app.Version,
	})
	z.Versions.Row(&VersionInfo{
		Key:       "app.hash",
		Component: ui.Text(z.HeaderAppHash),
		Version:   app.Hash,
	})
	z.Versions.Row(&VersionInfo{
		Key:       "build.time",
		Component: ui.Text(z.HeaderBuildTime),
		Version:   app.BuildTimestamp,
	})
	z.Versions.Row(&VersionInfo{
		Key:       "go.version",
		Component: ui.Text(z.HeaderGoVersion),
		Version:   runtime.Version(),
	})
	return nil
}

func (z *Version) Test(c app_control.Control) error {
	return rc_exec.Exec(c, &Version{}, rc_recipe.NoCustomValues)
}
<|endoftext|>


<commit_before>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

package handlers

import (
	"appengine"
	"appengine/datastore"
	"code.google.com/p/mlab-ns2/gae/ns/data"
	"code.google.com/p/mlab-ns2/gae/ns/rtt"
	"errors"
	"fmt"
	"net"
	"net/http"
)

const (
	URLRTTMain = "/rtt/"
)

var (
	ErrNoToolIDSpecified = errors.New("rtt: No Tool ID specified in request.")
	ErrNotEnoughData     = errors.New("rtt: The RTT resolver has insufficient data to respond to this query.")
)

func init() {
	http.HandleFunc(URLRTTMain, RTTHandler)
}

// RTTHandler is a simple handler which uses the URL parameter 'url' or client's
// IP to find a Sliver with lowest RTT.
func RTTHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	// Get IP to use as client IP.
	ipStr := r.FormValue("ip")
	if ipStr == "" {
		ipStr = r.RemoteAddr
	}
	ip := net.ParseIP(ipStr)

	// Get Tool ID
	toolID := r.FormValue("tool_id")
	if toolID == "" {
		http.Error(w, ErrNoToolIDSpecified.Error(), http.StatusInternalServerError)
		return
	}

	// Query RTT resolver.
	resp, err := RTTResolver(c, toolID, ip)
	switch err {
	case ErrNotEnoughData:
		http.Error(w, err.Error(), http.StatusNotFound)
	case nil:
		fmt.Fprintln(w, resp)
	default:
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

// RTTResolver returns a Sliver from a Site with lowest RTT given a client's IP.
func RTTResolver(c appengine.Context, toolID string, ip net.IP) (net.IP, error) {
	cgIP := rtt.GetClientGroup(ip).IP
	rttKey := datastore.NewKey(c, "string", "rtt", 0, nil)
	key := datastore.NewKey(c, "ClientGroup", cgIP.String(), 0, rttKey)

	// Get ClientGroup from datastore.
	var cg rtt.ClientGroup
	err := data.GetData(c, MCKey_ClientGroup(cgIP), key, &cg)
	if err != nil {
		if err == datastore.ErrNoSuchEntity {
			return nil, ErrNotEnoughData
		}
		return nil, err
	}

	// Get first error-less Site and a random SliverTool from selected Site.
	var siteID string
	var sliverTool *data.SliverTool
	for _, sr := range cg.SiteRTTs {
		siteID = sr.SiteID
		sliverTool, err = data.GetRandomSliverFromSite(c, toolID, siteID)
		if err == nil {
			return net.ParseIP(sliverTool.SliverIPv4), nil
		}
	}
	// No valid Site found.
	return nil, ErrNotEnoughData
}

// MCKey_ClientGroup returns a key for use in memcache for rtt.ClientGroup data.
func MCKey_ClientGroup(ip net.IP) string {
	key := fmt.Sprintf("rtt:ClientGroup:%s", ip)
	return key
}
<commit_msg>Address CL13235045<commit_after>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

package handlers

import (
	"appengine"
	"appengine/datastore"
	"code.google.com/p/mlab-ns2/gae/ns/data"
	"code.google.com/p/mlab-ns2/gae/ns/rtt"
	"errors"
	"fmt"
	"net"
	"net/http"
)

const (
	URLRTTMain = "/rtt/"
)

var (
	ErrNoToolIDSpecified = errors.New("rtt: No Tool ID specified in request.")
	ErrNotEnoughData     = errors.New("rtt: The RTT resolver has insufficient data to respond to this query.")
)

func init() {
	http.HandleFunc(URLRTTMain, RTTHandler)
}

// RTTHandler is a simple handler which uses the URL parameter 'url' or client's
// IP to find a Sliver with lowest RTT.
func RTTHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	// Get IP to use as client IP.
	ipStr := r.FormValue("ip")
	if ipStr == "" {
		ipStr = r.RemoteAddr
	}
	ip := net.ParseIP(ipStr)

	// Get Tool ID
	toolID := r.FormValue("tool_id")
	if toolID == "" {
		http.Error(w, ErrNoToolIDSpecified.Error(), http.StatusInternalServerError)
		c.Errorf("rtt.RTTHandler: %s", ErrNoToolIDSpecified)
		return
	}

	// Query RTT resolver.
	resp, err := RTTResolver(c, toolID, ip)
	switch err {
	case ErrNotEnoughData:
		http.Error(w, err.Error(), http.StatusNotFound)
		c.Errorf("rtt.RTTHandler: %s", err)
	case nil:
		fmt.Fprintln(w, resp)
	default:
		http.Error(w, err.Error(), http.StatusInternalServerError)
		c.Errorf("rtt.RTTHandler: %s", err)
	}
}

// RTTResolver returns a Sliver from a Site with lowest RTT given a client's IP.
func RTTResolver(c appengine.Context, toolID string, ip net.IP) (net.IP, error) {
	cgIP := rtt.GetClientGroup(ip).IP
	rttKey := datastore.NewKey(c, "string", "rtt", 0, nil)
	key := datastore.NewKey(c, "ClientGroup", cgIP.String(), 0, rttKey)

	// Get ClientGroup from datastore.
	var cg rtt.ClientGroup
	err := data.GetData(c, MCKey_ClientGroup(cgIP), key, &cg)
	if err != nil {
		if err == datastore.ErrNoSuchEntity {
			return nil, ErrNotEnoughData
		}
		return nil, err
	}

	// Get first error-less Site and a random SliverTool from selected Site.
	var siteID string
	var sliverTool *data.SliverTool
	for _, sr := range cg.SiteRTTs {
		siteID = sr.SiteID
		sliverTool, err = data.GetRandomSliverFromSite(c, toolID, siteID)
		if err == nil {
			return net.ParseIP(sliverTool.SliverIPv4), nil
		}
	}
	// No valid Site found.
	return nil, ErrNotEnoughData
}

// MCKey_ClientGroup returns a key for use in memcache for rtt.ClientGroup data.
func MCKey_ClientGroup(ip net.IP) string {
	key := fmt.Sprintf("rtt:ClientGroup:%s", ip)
	return key
}
<|endoftext|>


<commit_before>package main

import (
	"../util"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"strconv"
)

func HttpServer(listener net.Listener) {
	log.Printf("HTTP: listening on %s", listener.Addr().String())
	http.HandleFunc("/ping", pingHandler)
	http.HandleFunc("/lookup", lookupHandler)
	err := http.Serve(listener, nil)
	if err != nil {
		log.Fatal("http.ListenAndServe:", err)
	}
}

func pingHandler(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Length", "2")
	io.WriteString(w, "OK")
}

func lookupHandler(w http.ResponseWriter, req *http.Request) {
	reqParams, err := util.NewReqParams(req)
	if err != nil {
		log.Printf("ERROR: failed to parse request params - %s", err.Error())
		w.Write([]byte(`{"status_code":500, "status_txt":"INVALID_REQUEST", "data":null}`))
		return
	}

	topicName, err := reqParams.Query("topic")
	if err != nil {
		w.Write([]byte(`{"status_code":500, "status_txt":"MISSING_ARG_TOPIC", "data":null}`))
		return
	}

	dataInterface, ok := sm.Get("topic." + topicName)
	if !ok {
		errorTxt := fmt.Sprintf(`{"status_code":500, "status_txt":"%s", "data":null}`)
		w.Write([]byte(errorTxt))
		return
	}

	data := dataInterface.(map[string]interface{})
	output := make(map[string]interface{})
	output["data"] = data
	output["status_code"] = 200
	output["status_txt"] = "OK"
	response, err := json.Marshal(&output)
	if err != nil {
		errorTxt := fmt.Sprintf(`{"status_code":500, "status_txt":"%s", "data":null}`, err.Error())
		w.Write([]byte(errorTxt))
		return
	}

	w.Header().Set("Content-Length", strconv.Itoa(len(response)))
	w.Write(response)
}
<commit_msg>fix error when topic is invalid<commit_after>package main

import (
	"../util"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"strconv"
)

func HttpServer(listener net.Listener) {
	log.Printf("HTTP: listening on %s", listener.Addr().String())
	http.HandleFunc("/ping", pingHandler)
	http.HandleFunc("/lookup", lookupHandler)
	err := http.Serve(listener, nil)
	if err != nil {
		log.Fatal("http.ListenAndServe:", err)
	}
}

func pingHandler(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Length", "2")
	io.WriteString(w, "OK")
}

func lookupHandler(w http.ResponseWriter, req *http.Request) {
	reqParams, err := util.NewReqParams(req)
	if err != nil {
		log.Printf("ERROR: failed to parse request params - %s", err.Error())
		w.Write([]byte(`{"status_code":500, "status_txt":"INVALID_REQUEST", "data":null}`))
		return
	}

	topicName, err := reqParams.Query("topic")
	if err != nil {
		w.Write([]byte(`{"status_code":500, "status_txt":"MISSING_ARG_TOPIC", "data":null}`))
		return
	}

	dataInterface, ok := sm.Get("topic." + topicName)
	if !ok {
		errorTxt := fmt.Sprintf(`{"status_code":500, "status_txt":"INVALID_ARG_TOPIC", "data":null}`)
		w.Write([]byte(errorTxt))
		return
	}

	data := dataInterface.(map[string]interface{})
	output := make(map[string]interface{})
	output["data"] = data
	output["status_code"] = 200
	output["status_txt"] = "OK"
	response, err := json.Marshal(&output)
	if err != nil {
		errorTxt := fmt.Sprintf(`{"status_code":500, "status_txt":"%s", "data":null}`, err.Error())
		w.Write([]byte(errorTxt))
		return
	}

	w.Header().Set("Content-Length", strconv.Itoa(len(response)))
	w.Write(response)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"bytes"
	"fmt"
	"reflect"
	"strings"
	"sync"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/clientset_generated/release_1_2"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/metrics"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	maxKubectlExecRetries = 5
)

// Framework supports common operations used by e2e tests; it will keep a client & a namespace for you.
// Eventual goal is to merge this with integration test framework.
type Framework struct {
	BaseName string

	Client        *client.Client
	Clientset_1_2 *release_1_2.Clientset

	Namespace                *api.Namespace   // Every test has at least one namespace
	namespacesToDelete       []*api.Namespace // Some tests have more than one.
	NamespaceDeletionTimeout time.Duration

	gatherer containerResourceGatherer
	// Constraints that passed to a check which is exectued after data is gathered to
	// see if 99% of results are within acceptable bounds. It as to be injected in the test,
	// as expectations vary greatly. Constraints are groupped by the container names.
	addonResourceConstraints map[string]resourceConstraint

	logsSizeWaitGroup    sync.WaitGroup
	logsSizeCloseChannel chan bool
	logsSizeVerifier     *LogsSizeVerifier

	// To make sure that this framework cleans up after itself, no matter what,
	// we install a cleanup action before each test and clear it after.  If we
	// should abort, the AfterSuite hook should run all cleanup actions.
	cleanupHandle CleanupActionHandle
}

type TestDataSummary interface {
	PrintHumanReadable() string
	PrintJSON() string
}

// NewFramework makes a new framework and sets up a BeforeEach/AfterEach for
// you (you can write additional before/after each functions).
func NewFramework(baseName string) *Framework {
	f := &Framework{
		BaseName:                 baseName,
		addonResourceConstraints: make(map[string]resourceConstraint),
	}

	BeforeEach(f.beforeEach)
	AfterEach(f.afterEach)

	return f
}

// beforeEach gets a client and makes a namespace.
func (f *Framework) beforeEach() {
	// The fact that we need this feels like a bug in ginkgo.
	// https://github.com/onsi/ginkgo/issues/222
	f.cleanupHandle = AddCleanupAction(f.afterEach)

	By("Creating a kubernetes client")
	c, err := loadClient()
	Expect(err).NotTo(HaveOccurred())

	f.Client = c
	f.Clientset_1_2 = release_1_2.FromUnversionedClient(c)

	By("Building a namespace api object")
	namespace, err := f.CreateNamespace(f.BaseName, map[string]string{
		"e2e-framework": f.BaseName,
	})
	Expect(err).NotTo(HaveOccurred())

	f.Namespace = namespace

	if testContext.VerifyServiceAccount {
		By("Waiting for a default service account to be provisioned in namespace")
		err = waitForDefaultServiceAccountInNamespace(c, namespace.Name)
		Expect(err).NotTo(HaveOccurred())
	} else {
		Logf("Skipping waiting for service account")
	}

	if testContext.GatherKubeSystemResourceUsageData {
		f.gatherer.startGatheringData(c, resourceDataGatheringPeriodSeconds*time.Second)
	}

	if testContext.GatherLogsSizes {
		f.logsSizeWaitGroup = sync.WaitGroup{}
		f.logsSizeWaitGroup.Add(1)
		f.logsSizeCloseChannel = make(chan bool)
		f.logsSizeVerifier = NewLogsVerifier(c, f.logsSizeCloseChannel)
		go func() {
			f.logsSizeVerifier.Run()
			f.logsSizeWaitGroup.Done()
		}()
	}
}

// afterEach deletes the namespace, after reading its events.
func (f *Framework) afterEach() {
	RemoveCleanupAction(f.cleanupHandle)

	// Print events if the test failed.
	if CurrentGinkgoTestDescription().Failed {
		By(fmt.Sprintf("Collecting events from namespace %q.", f.Namespace.Name))
		events, err := f.Client.Events(f.Namespace.Name).List(api.ListOptions{})
		Expect(err).NotTo(HaveOccurred())

		for _, e := range events.Items {
			Logf("event for %v: %v %v: %v", e.InvolvedObject.Name, e.Source, e.Reason, e.Message)
		}
		// Note that we don't wait for any cleanup to propagate, which means
		// that if you delete a bunch of pods right before ending your test,
		// you may or may not see the killing/deletion/cleanup events.

		dumpAllPodInfo(f.Client)

		dumpAllNodeInfo(f.Client)
	}

	summaries := make([]TestDataSummary, 0)
	if testContext.GatherKubeSystemResourceUsageData {
		summaries = append(summaries, f.gatherer.stopAndSummarize([]int{90, 99}, f.addonResourceConstraints))
	}

	if testContext.GatherLogsSizes {
		close(f.logsSizeCloseChannel)
		f.logsSizeWaitGroup.Wait()
		summaries = append(summaries, f.logsSizeVerifier.GetSummary())
	}

	if testContext.GatherMetricsAfterTest {
		// TODO: enable Scheduler and ControllerManager metrics grabbing when Master's Kubelet will be registered.
		grabber, err := metrics.NewMetricsGrabber(f.Client, true, false, false, true)
		if err != nil {
			Logf("Failed to create MetricsGrabber. Skipping metrics gathering.")
		} else {
			received, err := grabber.Grab(nil)
			if err != nil {
				Logf("MetricsGrabber failed grab metrics. Skipping metrics gathering.")
			} else {
				summaries = append(summaries, (*MetricsForE2E)(&received))
			}
		}
	}

	if testContext.DeleteNamespace {
		for _, ns := range f.namespacesToDelete {
			By(fmt.Sprintf("Destroying namespace %q for this suite.", ns.Name))

			timeout := 5 * time.Minute
			if f.NamespaceDeletionTimeout != 0 {
				timeout = f.NamespaceDeletionTimeout
			}
			if err := deleteNS(f.Client, ns.Name, timeout); err != nil {
				Failf("Couldn't delete ns %q: %s", ns.Name, err)
			}
		}
		f.namespacesToDelete = nil
	} else {
		Logf("Found DeleteNamespace=false, skipping namespace deletion!")
	}

	outputTypes := strings.Split(testContext.OutputPrintType, ",")
	for _, printType := range outputTypes {
		switch printType {
		case "hr":
			for i := range summaries {
				Logf(summaries[i].PrintHumanReadable())
			}
		case "json":
			for i := range summaries {
				typeName := reflect.TypeOf(summaries[i]).String()
				Logf("%v JSON\n%v", typeName[strings.LastIndex(typeName, ".")+1:len(typeName)], summaries[i].PrintJSON())
				Logf("Finished")
			}
		default:
			Logf("Unknown ouptut type: %v. Skipping.", printType)
		}
	}

	// Check whether all nodes are ready after the test.
	// This is explicitly done at the very end of the test, to avoid
	// e.g. not removing namespace in case of this failure.
	if err := allNodesReady(f.Client, time.Minute); err != nil {
		Failf("All nodes should be ready after test, %v", err)
	}

	// Paranoia-- prevent reuse!
	f.Namespace = nil
	f.Client = nil
}

func (f *Framework) CreateNamespace(baseName string, labels map[string]string) (*api.Namespace, error) {
	ns, err := createTestingNS(baseName, f.Client, labels)
	if err == nil {
		f.namespacesToDelete = append(f.namespacesToDelete, ns)
	}
	return ns, err
}

// WaitForPodTerminated waits for the pod to be terminated with the given reason.
func (f *Framework) WaitForPodTerminated(podName, reason string) error {
	return waitForPodTerminatedInNamespace(f.Client, podName, reason, f.Namespace.Name)
}

// WaitForPodRunning waits for the pod to run in the namespace.
func (f *Framework) WaitForPodRunning(podName string) error {
	return waitForPodRunningInNamespace(f.Client, podName, f.Namespace.Name)
}

// WaitForPodRunningSlow waits for the pod to run in the namespace.
// It has a longer timeout then WaitForPodRunning (util.slowPodStartTimeout).
func (f *Framework) WaitForPodRunningSlow(podName string) error {
	return waitForPodRunningInNamespaceSlow(f.Client, podName, f.Namespace.Name)
}

// Runs the given pod and verifies that the output of exact container matches the desired output.
func (f *Framework) TestContainerOutput(scenarioName string, pod *api.Pod, containerIndex int, expectedOutput []string) {
	testContainerOutput(scenarioName, f.Client, pod, containerIndex, expectedOutput, f.Namespace.Name)
}

// Runs the given pod and verifies that the output of exact container matches the desired regexps.
func (f *Framework) TestContainerOutputRegexp(scenarioName string, pod *api.Pod, containerIndex int, expectedOutput []string) {
	testContainerOutputRegexp(scenarioName, f.Client, pod, containerIndex, expectedOutput, f.Namespace.Name)
}

// WaitForAnEndpoint waits for at least one endpoint to become available in the
// service's corresponding endpoints object.
func (f *Framework) WaitForAnEndpoint(serviceName string) error {
	for {
		// TODO: Endpoints client should take a field selector so we
		// don't have to list everything.
		list, err := f.Client.Endpoints(f.Namespace.Name).List(api.ListOptions{})
		if err != nil {
			return err
		}
		rv := list.ResourceVersion

		isOK := func(e *api.Endpoints) bool {
			return e.Name == serviceName && len(e.Subsets) > 0 && len(e.Subsets[0].Addresses) > 0
		}
		for i := range list.Items {
			if isOK(&list.Items[i]) {
				return nil
			}
		}

		options := api.ListOptions{
			FieldSelector:   fields.Set{"metadata.name": serviceName}.AsSelector(),
			ResourceVersion: rv,
		}
		w, err := f.Client.Endpoints(f.Namespace.Name).Watch(options)
		if err != nil {
			return err
		}
		defer w.Stop()

		for {
			val, ok := <-w.ResultChan()
			if !ok {
				// reget and re-watch
				break
			}
			if e, ok := val.Object.(*api.Endpoints); ok {
				if isOK(e) {
					return nil
				}
			}
		}
	}
}

// Write a file using kubectl exec echo <contents> > <path> via specified container
// Because of the primitive technique we're using here, we only allow ASCII alphanumeric characters
func (f *Framework) WriteFileViaContainer(podName, containerName string, path string, contents string) error {
	By("writing a file in the container")
	allowedCharacters := "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
	for _, c := range contents {
		if !strings.ContainsRune(allowedCharacters, c) {
			return fmt.Errorf("Unsupported character in string to write: %v", c)
		}
	}
	command := fmt.Sprintf("echo '%s' > '%s'", contents, path)
	stdout, stderr, err := kubectlExecWithRetry(f.Namespace.Name, podName, containerName, "--", "/bin/sh", "-c", command)
	if err != nil {
		Logf("error running kubectl exec to write file: %v\nstdout=%v\nstderr=%v)", err, string(stdout), string(stderr))
	}
	return err
}

// Read a file using kubectl exec cat <path>
func (f *Framework) ReadFileViaContainer(podName, containerName string, path string) (string, error) {
	By("reading a file in the container")

	stdout, stderr, err := kubectlExecWithRetry(f.Namespace.Name, podName, containerName, "--", "cat", path)
	if err != nil {
		Logf("error running kubectl exec to read file: %v\nstdout=%v\nstderr=%v)", err, string(stdout), string(stderr))
	}
	return string(stdout), err
}

func kubectlExecWithRetry(namespace string, podName, containerName string, args ...string) ([]byte, []byte, error) {
	for numRetries := 0; numRetries < maxKubectlExecRetries; numRetries++ {
		if numRetries > 0 {
			Logf("Retrying kubectl exec (retry count=%v/%v)", numRetries+1, maxKubectlExecRetries)
		}

		stdOutBytes, stdErrBytes, err := kubectlExec(namespace, podName, containerName, args...)
		if err != nil {
			if strings.Contains(strings.ToLower(string(stdErrBytes)), "i/o timeout") {
				// Retry on "i/o timeout" errors
				Logf("Warning: kubectl exec encountered i/o timeout.\nerr=%v\nstdout=%v\nstderr=%v)", err, string(stdOutBytes), string(stdErrBytes))
				continue
			}
		}

		return stdOutBytes, stdErrBytes, err
	}
	err := fmt.Errorf("Failed: kubectl exec failed %d times with \"i/o timeout\". Giving up.", maxKubectlExecRetries)
	return nil, nil, err
}

func kubectlExec(namespace string, podName, containerName string, args ...string) ([]byte, []byte, error) {
	var stdout, stderr bytes.Buffer
	cmdArgs := []string{
		"exec",
		fmt.Sprintf("--namespace=%v", namespace),
		podName,
		fmt.Sprintf("-c=%v", containerName),
	}
	cmdArgs = append(cmdArgs, args...)

	cmd := kubectlCmd(cmdArgs...)
	cmd.Stdout, cmd.Stderr = &stdout, &stderr

	Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args, " "))
	err := cmd.Run()
	return stdout.Bytes(), stderr.Bytes(), err
}
<commit_msg>Always delete namespace in test framework<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"bytes"
	"fmt"
	"reflect"
	"strings"
	"sync"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/clientset_generated/release_1_2"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/metrics"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	maxKubectlExecRetries = 5
)

// Framework supports common operations used by e2e tests; it will keep a client & a namespace for you.
// Eventual goal is to merge this with integration test framework.
type Framework struct {
	BaseName string

	Client        *client.Client
	Clientset_1_2 *release_1_2.Clientset

	Namespace                *api.Namespace   // Every test has at least one namespace
	namespacesToDelete       []*api.Namespace // Some tests have more than one.
	NamespaceDeletionTimeout time.Duration

	gatherer containerResourceGatherer
	// Constraints that passed to a check which is exectued after data is gathered to
	// see if 99% of results are within acceptable bounds. It as to be injected in the test,
	// as expectations vary greatly. Constraints are groupped by the container names.
	addonResourceConstraints map[string]resourceConstraint

	logsSizeWaitGroup    sync.WaitGroup
	logsSizeCloseChannel chan bool
	logsSizeVerifier     *LogsSizeVerifier

	// To make sure that this framework cleans up after itself, no matter what,
	// we install a cleanup action before each test and clear it after.  If we
	// should abort, the AfterSuite hook should run all cleanup actions.
	cleanupHandle CleanupActionHandle
}

type TestDataSummary interface {
	PrintHumanReadable() string
	PrintJSON() string
}

// NewFramework makes a new framework and sets up a BeforeEach/AfterEach for
// you (you can write additional before/after each functions).
func NewFramework(baseName string) *Framework {
	f := &Framework{
		BaseName:                 baseName,
		addonResourceConstraints: make(map[string]resourceConstraint),
	}

	BeforeEach(f.beforeEach)
	AfterEach(f.afterEach)

	return f
}

// beforeEach gets a client and makes a namespace.
func (f *Framework) beforeEach() {
	// The fact that we need this feels like a bug in ginkgo.
	// https://github.com/onsi/ginkgo/issues/222
	f.cleanupHandle = AddCleanupAction(f.afterEach)

	By("Creating a kubernetes client")
	c, err := loadClient()
	Expect(err).NotTo(HaveOccurred())

	f.Client = c
	f.Clientset_1_2 = release_1_2.FromUnversionedClient(c)

	By("Building a namespace api object")
	namespace, err := f.CreateNamespace(f.BaseName, map[string]string{
		"e2e-framework": f.BaseName,
	})
	Expect(err).NotTo(HaveOccurred())

	f.Namespace = namespace

	if testContext.VerifyServiceAccount {
		By("Waiting for a default service account to be provisioned in namespace")
		err = waitForDefaultServiceAccountInNamespace(c, namespace.Name)
		Expect(err).NotTo(HaveOccurred())
	} else {
		Logf("Skipping waiting for service account")
	}

	if testContext.GatherKubeSystemResourceUsageData {
		f.gatherer.startGatheringData(c, resourceDataGatheringPeriodSeconds*time.Second)
	}

	if testContext.GatherLogsSizes {
		f.logsSizeWaitGroup = sync.WaitGroup{}
		f.logsSizeWaitGroup.Add(1)
		f.logsSizeCloseChannel = make(chan bool)
		f.logsSizeVerifier = NewLogsVerifier(c, f.logsSizeCloseChannel)
		go func() {
			f.logsSizeVerifier.Run()
			f.logsSizeWaitGroup.Done()
		}()
	}
}

// afterEach deletes the namespace, after reading its events.
func (f *Framework) afterEach() {
	RemoveCleanupAction(f.cleanupHandle)

	// DeleteNamespace at the very end in defer, to avoid any
	// expectation failures preventing deleting the namespace.
	defer func() {
		if testContext.DeleteNamespace {
			for _, ns := range f.namespacesToDelete {
				By(fmt.Sprintf("Destroying namespace %q for this suite.", ns.Name))

				timeout := 5 * time.Minute
				if f.NamespaceDeletionTimeout != 0 {
					timeout = f.NamespaceDeletionTimeout
				}
				if err := deleteNS(f.Client, ns.Name, timeout); err != nil {
					Failf("Couldn't delete ns %q: %s", ns.Name, err)
				}
			}
			f.namespacesToDelete = nil
		} else {
			Logf("Found DeleteNamespace=false, skipping namespace deletion!")
		}

		// Paranoia-- prevent reuse!
		f.Namespace = nil
		f.Client = nil
	}()

	// Print events if the test failed.
	if CurrentGinkgoTestDescription().Failed {
		By(fmt.Sprintf("Collecting events from namespace %q.", f.Namespace.Name))
		events, err := f.Client.Events(f.Namespace.Name).List(api.ListOptions{})
		Expect(err).NotTo(HaveOccurred())

		for _, e := range events.Items {
			Logf("event for %v: %v %v: %v", e.InvolvedObject.Name, e.Source, e.Reason, e.Message)
		}
		// Note that we don't wait for any cleanup to propagate, which means
		// that if you delete a bunch of pods right before ending your test,
		// you may or may not see the killing/deletion/cleanup events.

		dumpAllPodInfo(f.Client)

		dumpAllNodeInfo(f.Client)
	}

	summaries := make([]TestDataSummary, 0)
	if testContext.GatherKubeSystemResourceUsageData {
		summaries = append(summaries, f.gatherer.stopAndSummarize([]int{90, 99}, f.addonResourceConstraints))
	}

	if testContext.GatherLogsSizes {
		close(f.logsSizeCloseChannel)
		f.logsSizeWaitGroup.Wait()
		summaries = append(summaries, f.logsSizeVerifier.GetSummary())
	}

	if testContext.GatherMetricsAfterTest {
		// TODO: enable Scheduler and ControllerManager metrics grabbing when Master's Kubelet will be registered.
		grabber, err := metrics.NewMetricsGrabber(f.Client, true, false, false, true)
		if err != nil {
			Logf("Failed to create MetricsGrabber. Skipping metrics gathering.")
		} else {
			received, err := grabber.Grab(nil)
			if err != nil {
				Logf("MetricsGrabber failed grab metrics. Skipping metrics gathering.")
			} else {
				summaries = append(summaries, (*MetricsForE2E)(&received))
			}
		}
	}

	outputTypes := strings.Split(testContext.OutputPrintType, ",")
	for _, printType := range outputTypes {
		switch printType {
		case "hr":
			for i := range summaries {
				Logf(summaries[i].PrintHumanReadable())
			}
		case "json":
			for i := range summaries {
				typeName := reflect.TypeOf(summaries[i]).String()
				Logf("%v JSON\n%v", typeName[strings.LastIndex(typeName, ".")+1:len(typeName)], summaries[i].PrintJSON())
				Logf("Finished")
			}
		default:
			Logf("Unknown ouptut type: %v. Skipping.", printType)
		}
	}

	// Check whether all nodes are ready after the test.
	// This is explicitly done at the very end of the test, to avoid
	// e.g. not removing namespace in case of this failure.
	if err := allNodesReady(f.Client, time.Minute); err != nil {
		Failf("All nodes should be ready after test, %v", err)
	}
}

func (f *Framework) CreateNamespace(baseName string, labels map[string]string) (*api.Namespace, error) {
	ns, err := createTestingNS(baseName, f.Client, labels)
	if err == nil {
		f.namespacesToDelete = append(f.namespacesToDelete, ns)
	}
	return ns, err
}

// WaitForPodTerminated waits for the pod to be terminated with the given reason.
func (f *Framework) WaitForPodTerminated(podName, reason string) error {
	return waitForPodTerminatedInNamespace(f.Client, podName, reason, f.Namespace.Name)
}

// WaitForPodRunning waits for the pod to run in the namespace.
func (f *Framework) WaitForPodRunning(podName string) error {
	return waitForPodRunningInNamespace(f.Client, podName, f.Namespace.Name)
}

// WaitForPodRunningSlow waits for the pod to run in the namespace.
// It has a longer timeout then WaitForPodRunning (util.slowPodStartTimeout).
func (f *Framework) WaitForPodRunningSlow(podName string) error {
	return waitForPodRunningInNamespaceSlow(f.Client, podName, f.Namespace.Name)
}

// Runs the given pod and verifies that the output of exact container matches the desired output.
func (f *Framework) TestContainerOutput(scenarioName string, pod *api.Pod, containerIndex int, expectedOutput []string) {
	testContainerOutput(scenarioName, f.Client, pod, containerIndex, expectedOutput, f.Namespace.Name)
}

// Runs the given pod and verifies that the output of exact container matches the desired regexps.
func (f *Framework) TestContainerOutputRegexp(scenarioName string, pod *api.Pod, containerIndex int, expectedOutput []string) {
	testContainerOutputRegexp(scenarioName, f.Client, pod, containerIndex, expectedOutput, f.Namespace.Name)
}

// WaitForAnEndpoint waits for at least one endpoint to become available in the
// service's corresponding endpoints object.
func (f *Framework) WaitForAnEndpoint(serviceName string) error {
	for {
		// TODO: Endpoints client should take a field selector so we
		// don't have to list everything.
		list, err := f.Client.Endpoints(f.Namespace.Name).List(api.ListOptions{})
		if err != nil {
			return err
		}
		rv := list.ResourceVersion

		isOK := func(e *api.Endpoints) bool {
			return e.Name == serviceName && len(e.Subsets) > 0 && len(e.Subsets[0].Addresses) > 0
		}
		for i := range list.Items {
			if isOK(&list.Items[i]) {
				return nil
			}
		}

		options := api.ListOptions{
			FieldSelector:   fields.Set{"metadata.name": serviceName}.AsSelector(),
			ResourceVersion: rv,
		}
		w, err := f.Client.Endpoints(f.Namespace.Name).Watch(options)
		if err != nil {
			return err
		}
		defer w.Stop()

		for {
			val, ok := <-w.ResultChan()
			if !ok {
				// reget and re-watch
				break
			}
			if e, ok := val.Object.(*api.Endpoints); ok {
				if isOK(e) {
					return nil
				}
			}
		}
	}
}

// Write a file using kubectl exec echo <contents> > <path> via specified container
// Because of the primitive technique we're using here, we only allow ASCII alphanumeric characters
func (f *Framework) WriteFileViaContainer(podName, containerName string, path string, contents string) error {
	By("writing a file in the container")
	allowedCharacters := "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
	for _, c := range contents {
		if !strings.ContainsRune(allowedCharacters, c) {
			return fmt.Errorf("Unsupported character in string to write: %v", c)
		}
	}
	command := fmt.Sprintf("echo '%s' > '%s'", contents, path)
	stdout, stderr, err := kubectlExecWithRetry(f.Namespace.Name, podName, containerName, "--", "/bin/sh", "-c", command)
	if err != nil {
		Logf("error running kubectl exec to write file: %v\nstdout=%v\nstderr=%v)", err, string(stdout), string(stderr))
	}
	return err
}

// Read a file using kubectl exec cat <path>
func (f *Framework) ReadFileViaContainer(podName, containerName string, path string) (string, error) {
	By("reading a file in the container")

	stdout, stderr, err := kubectlExecWithRetry(f.Namespace.Name, podName, containerName, "--", "cat", path)
	if err != nil {
		Logf("error running kubectl exec to read file: %v\nstdout=%v\nstderr=%v)", err, string(stdout), string(stderr))
	}
	return string(stdout), err
}

func kubectlExecWithRetry(namespace string, podName, containerName string, args ...string) ([]byte, []byte, error) {
	for numRetries := 0; numRetries < maxKubectlExecRetries; numRetries++ {
		if numRetries > 0 {
			Logf("Retrying kubectl exec (retry count=%v/%v)", numRetries+1, maxKubectlExecRetries)
		}

		stdOutBytes, stdErrBytes, err := kubectlExec(namespace, podName, containerName, args...)
		if err != nil {
			if strings.Contains(strings.ToLower(string(stdErrBytes)), "i/o timeout") {
				// Retry on "i/o timeout" errors
				Logf("Warning: kubectl exec encountered i/o timeout.\nerr=%v\nstdout=%v\nstderr=%v)", err, string(stdOutBytes), string(stdErrBytes))
				continue
			}
		}

		return stdOutBytes, stdErrBytes, err
	}
	err := fmt.Errorf("Failed: kubectl exec failed %d times with \"i/o timeout\". Giving up.", maxKubectlExecRetries)
	return nil, nil, err
}

func kubectlExec(namespace string, podName, containerName string, args ...string) ([]byte, []byte, error) {
	var stdout, stderr bytes.Buffer
	cmdArgs := []string{
		"exec",
		fmt.Sprintf("--namespace=%v", namespace),
		podName,
		fmt.Sprintf("-c=%v", containerName),
	}
	cmdArgs = append(cmdArgs, args...)

	cmd := kubectlCmd(cmdArgs...)
	cmd.Stdout, cmd.Stderr = &stdout, &stderr

	Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args, " "))
	err := cmd.Run()
	return stdout.Bytes(), stderr.Bytes(), err
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pzse

import (
	"fmt"
	"net/http"
	"os/exec"
	"regexp"
	"strings"

	"github.com/venicegeo/pzsvc-exec/pzsvc"
)

func handleFList(s pzsvc.Session, fList, nameList []string, lFunc rangeFunc, fType, action string, output *OutStruct, fileRec map[string]string) {
	re := regexp.MustCompile(`^[\w\-\.]*$`)
	for i, f := range fList {
		name := ""
		if len(nameList) > i {
			name = nameList[i]
		}
		if !re.Match([]byte(name)) {
			pzsvc.LogAlert(s, `Illegal filename "`+name+`" entered for `+action+`.  Possible attempted security breach.`)
			addOutputError(output, `handleFlist error: Filename "`+name+`" contains illegal characters and is not permitted.`, http.StatusBadRequest)
			continue
		}
		outStr, err := lFunc(f, name, fType)
		if err != nil {
			addOutputError(output, "Error in "+action+" of "+name+".", http.StatusBadRequest)
		} else if outStr == "" {
			pzsvc.LogSimpleErr(s, `handleFlist error: type "`+fType+`", input "`+f+`" blank result.`, nil)
			addOutputError(output, "Blank Result Error in "+action+" of "+name+".", http.StatusBadRequest)
		} else {
			fileRec[f] = outStr
		}
	}
}

func addOutputError(output *OutStruct, errString string, httpStat int) {
	output.Errors = append(output.Errors, errString)
	output.HTTPStatus = httpStat
}

func splitOrNil(inString, knife string) []string {
	if inString == "" {
		return nil
	}
	return strings.Split(inString, knife)
}

// GetVersion spits out the best available version for the
// current software, based on the contents fo the config file
func GetVersion(s pzsvc.Session, configObj *ConfigType) string {
	vCmdSlice := splitOrNil(configObj.VersionCmd, " ")
	if vCmdSlice != nil {
		vCmd := exec.Command(vCmdSlice[0], vCmdSlice[1:]...)
		verB, err := vCmd.Output()
		if err != nil {
			pzsvc.LogSimpleErr(s, "VersionCmd failed: ", err)
		}
		if string(verB) != "" {
			return string(verB)
		}
	}
	return configObj.VersionStr
}

// CheckConfig takes an input config file, checks it over for issues,
// and outputs any issues or concerns to std.out.  It returns whether
// or not the config file permits autoregistration.
func CheckConfig(s pzsvc.Session, configObj *ConfigType) bool {
	canReg := true
	canPzFile := configObj.CanUpload || configObj.CanDownlPz
	if configObj.CliCmd == "" {
		pzsvc.LogAlert(s, `Config: Warning: CliCmd is blank.  This is a major security vulnerability.`)
	}

	if configObj.PzAddr == "" && configObj.PzAddrEnVar == "" {
		errStr := `Config: Did not specify PzAddr or PzAddrEnVar.  Autoregistration disabled.`
		if canPzFile {
			errStr += `  Client will have to provide Piazza Address for uploads and Piazza downloads.`
		}
		pzsvc.LogInfo(s, errStr)
		canReg = false
	} else if configObj.APIKeyEnVar == "" {
		errStr := `Config: APIKeyEnVar was not specified.  Autoregistration disabled.`
		if canPzFile {
			errStr += `  Client will have to provide authKey for uploads and Piazza downloads.`
		}
		pzsvc.LogInfo(s, errStr)
		canReg = false
	} else if configObj.SvcName == "" {
		pzsvc.LogInfo(s, `Config: SvcName not specified.  Autoregistration disabled.`)
		canReg = false
	} else if configObj.URL == "" && configObj.RegForTaskMgr == false {
		pzsvc.LogInfo(s, `Config: URL not specified.  URL required unless registering for task manager.  Autoregistration disabled.`)
		canReg = false
	} else if configObj.RegForTaskMgr && configObj.MaxRunTime == 0 {
		pzsvc.LogInfo(s, `Config: Cannot register for task manager use without MaxRunTime (in seconds).  Registration disabled.`)
		canReg = false
	}

	if !canReg {
		if configObj.VersionCmd != "" {
			pzsvc.LogInfo(s, `Config: VersionCmd was specified, but is much less useful without autoregistration.`)
		}
		if configObj.VersionStr != "" {
			pzsvc.LogInfo(s, `Config: VersionStr was specified, but is much less useful without without autoregistration.`)
		}
		if configObj.APIKeyEnVar != "" {
			if canPzFile {
				pzsvc.LogInfo(s, `Config: APIKeyEnVar was specified, but PzAddr was not.  APIKeyEnVar useless without a Pz instance to authenticate against.`)
			} else {
				pzsvc.LogInfo(s, `Config: APIKeyEnVar was specified, but is meaningless without autoregistration or Pz file interactions.`)
			}
		}
		if configObj.SvcName != "" {
			pzsvc.LogInfo(s, `Config: SvcName was specified, but is meaningless without autoregistration.`)
		}
		if configObj.URL != "" {
			pzsvc.LogInfo(s, `Config: URL was specified, but is meaningless without autoregistration.`)
		}
		if configObj.RegForTaskMgr {
			pzsvc.LogInfo(s, `Config: RegForTaskMgr was specified as true, but is meaningless without autoregistration.`)
		}
		if configObj.MaxRunTime == 0 {
			pzsvc.LogInfo(s, `Config: MaxRunTime was specified, but is meaningless without autoregistration.`)
		}
	} else {
		if configObj.PzAddr == "" && configObj.PzAddrEnVar == "" {
			pzsvc.LogInfo(s, `Config: Both PzAddr and PzAddrEnVar were specified.  Redundant.  Default to PzAddrEnVar.`)
		}
		if configObj.VersionCmd == "" && configObj.VersionStr == "" {
			pzsvc.LogInfo(s, `Config: Neither VersionCmd nor VersionStr were specified.  Version will be left blank.`)
		}
		if configObj.VersionCmd != "" && configObj.VersionStr != "" {
			pzsvc.LogInfo(s, `Config: Both VersionCmd and VersionStr were specified.  Redundant.  Default to VersionCmd.`)
		}
		if configObj.Description == "" {
			pzsvc.LogInfo(s, `Config: Description not specified.  When autoregistering, descriptions are strongly encouraged.`)
		}
		if configObj.MaxRunTime != 0 && !configObj.RegForTaskMgr {
			pzsvc.LogInfo(s, `Config: MaxRunTime not meaningful unless registering for task manager use.`)
		}

	}

	if configObj.Port <= 0 && configObj.PortEnVar == "" {
		pzsvc.LogInfo(s, `Config: Neither Port nor PortEnVar were properly specified.  Default to Port 8080.`)
	}
	if configObj.Port > 0 && configObj.PortEnVar != "" {
		pzsvc.LogInfo(s, `Config: Both Port and PortEnVar were specified.  Redundant.  Default to PortEnVar.`)
	}

	return canReg
}

// PrintHelp prints out a basic helpfile to make things easier on direct users
func PrintHelp(w http.ResponseWriter) {
	fmt.Fprintln(w, `The pzsvc-exec service endpoints are as follows:`)
	fmt.Fprintln(w, `- '/': entry point.  Displays base command if any, and suggests other endpoints.`)
	fmt.Fprintln(w, `- '/execute': The meat of the program.  Downloads files, executes on them, and uploads the results.`)
	fmt.Fprintln(w, `See the Service Request Format section of the Readme for interface details.`)
	fmt.Fprintln(w, `(Readme available at https://github.com/venicegeo/pzsvc-exec).`)
	fmt.Fprintln(w, `- '/description': When enabled, provides a description of this particular pzsvc-exec instance.`)
	fmt.Fprintln(w, `- '/attributes': When enabled, provides a list of key/value attributes for this pzsvc-exec instance.`)
	fmt.Fprintln(w, `- '/version': When enabled, provides version number for the application served by this pzsvc-exec instance.`)
	fmt.Fprintln(w, `- '/help': This screen.`)
}
<commit_msg>adding in debug for versionCmd call<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pzse

import (
	"fmt"
	"net/http"
	"os/exec"
	"regexp"
	"strings"

	"github.com/venicegeo/pzsvc-exec/pzsvc"
)

func handleFList(s pzsvc.Session, fList, nameList []string, lFunc rangeFunc, fType, action string, output *OutStruct, fileRec map[string]string) {
	re := regexp.MustCompile(`^[\w\-\.]*$`)
	for i, f := range fList {
		name := ""
		if len(nameList) > i {
			name = nameList[i]
		}
		if !re.Match([]byte(name)) {
			pzsvc.LogAlert(s, `Illegal filename "`+name+`" entered for `+action+`.  Possible attempted security breach.`)
			addOutputError(output, `handleFlist error: Filename "`+name+`" contains illegal characters and is not permitted.`, http.StatusBadRequest)
			continue
		}
		outStr, err := lFunc(f, name, fType)
		if err != nil {
			addOutputError(output, "Error in "+action+" of "+name+".", http.StatusBadRequest)
		} else if outStr == "" {
			pzsvc.LogSimpleErr(s, `handleFlist error: type "`+fType+`", input "`+f+`" blank result.`, nil)
			addOutputError(output, "Blank Result Error in "+action+" of "+name+".", http.StatusBadRequest)
		} else {
			fileRec[f] = outStr
		}
	}
}

func addOutputError(output *OutStruct, errString string, httpStat int) {
	output.Errors = append(output.Errors, errString)
	output.HTTPStatus = httpStat
}

func splitOrNil(inString, knife string) []string {
	if inString == "" {
		return nil
	}
	return strings.Split(inString, knife)
}

// GetVersion spits out the best available version for the
// current software, based on the contents fo the config file
func GetVersion(s pzsvc.Session, configObj *ConfigType) string {
	vCmdSlice := splitOrNil(configObj.VersionCmd, " ")
	if vCmdSlice != nil {
		vCmd := exec.Command(vCmdSlice[0], vCmdSlice[1:]...)
		verB, err := vCmd.Output()
		pzsvc.LogInfo(s, `Called VersionCmd `+configObj.VersionCmd+`.  Results: `+string(verB))
		if err != nil {
			pzsvc.LogSimpleErr(s, "VersionCmd failed: ", err)
		}
		if string(verB) != "" {
			return string(verB)
		}
	}
	return configObj.VersionStr
}

// CheckConfig takes an input config file, checks it over for issues,
// and outputs any issues or concerns to std.out.  It returns whether
// or not the config file permits autoregistration.
func CheckConfig(s pzsvc.Session, configObj *ConfigType) bool {
	canReg := true
	canPzFile := configObj.CanUpload || configObj.CanDownlPz
	if configObj.CliCmd == "" {
		pzsvc.LogAlert(s, `Config: Warning: CliCmd is blank.  This is a major security vulnerability.`)
	}

	if configObj.PzAddr == "" && configObj.PzAddrEnVar == "" {
		errStr := `Config: Did not specify PzAddr or PzAddrEnVar.  Autoregistration disabled.`
		if canPzFile {
			errStr += `  Client will have to provide Piazza Address for uploads and Piazza downloads.`
		}
		pzsvc.LogInfo(s, errStr)
		canReg = false
	} else if configObj.APIKeyEnVar == "" {
		errStr := `Config: APIKeyEnVar was not specified.  Autoregistration disabled.`
		if canPzFile {
			errStr += `  Client will have to provide authKey for uploads and Piazza downloads.`
		}
		pzsvc.LogInfo(s, errStr)
		canReg = false
	} else if configObj.SvcName == "" {
		pzsvc.LogInfo(s, `Config: SvcName not specified.  Autoregistration disabled.`)
		canReg = false
	} else if configObj.URL == "" && configObj.RegForTaskMgr == false {
		pzsvc.LogInfo(s, `Config: URL not specified.  URL required unless registering for task manager.  Autoregistration disabled.`)
		canReg = false
	} else if configObj.RegForTaskMgr && configObj.MaxRunTime == 0 {
		pzsvc.LogInfo(s, `Config: Cannot register for task manager use without MaxRunTime (in seconds).  Registration disabled.`)
		canReg = false
	}

	if !canReg {
		if configObj.VersionCmd != "" {
			pzsvc.LogInfo(s, `Config: VersionCmd was specified, but is much less useful without autoregistration.`)
		}
		if configObj.VersionStr != "" {
			pzsvc.LogInfo(s, `Config: VersionStr was specified, but is much less useful without without autoregistration.`)
		}
		if configObj.APIKeyEnVar != "" {
			if canPzFile {
				pzsvc.LogInfo(s, `Config: APIKeyEnVar was specified, but PzAddr was not.  APIKeyEnVar useless without a Pz instance to authenticate against.`)
			} else {
				pzsvc.LogInfo(s, `Config: APIKeyEnVar was specified, but is meaningless without autoregistration or Pz file interactions.`)
			}
		}
		if configObj.SvcName != "" {
			pzsvc.LogInfo(s, `Config: SvcName was specified, but is meaningless without autoregistration.`)
		}
		if configObj.URL != "" {
			pzsvc.LogInfo(s, `Config: URL was specified, but is meaningless without autoregistration.`)
		}
		if configObj.RegForTaskMgr {
			pzsvc.LogInfo(s, `Config: RegForTaskMgr was specified as true, but is meaningless without autoregistration.`)
		}
		if configObj.MaxRunTime == 0 {
			pzsvc.LogInfo(s, `Config: MaxRunTime was specified, but is meaningless without autoregistration.`)
		}
	} else {
		if configObj.PzAddr == "" && configObj.PzAddrEnVar == "" {
			pzsvc.LogInfo(s, `Config: Both PzAddr and PzAddrEnVar were specified.  Redundant.  Default to PzAddrEnVar.`)
		}
		if configObj.VersionCmd == "" && configObj.VersionStr == "" {
			pzsvc.LogInfo(s, `Config: Neither VersionCmd nor VersionStr were specified.  Version will be left blank.`)
		}
		if configObj.VersionCmd != "" && configObj.VersionStr != "" {
			pzsvc.LogInfo(s, `Config: Both VersionCmd and VersionStr were specified.  Redundant.  Default to VersionCmd.`)
		}
		if configObj.Description == "" {
			pzsvc.LogInfo(s, `Config: Description not specified.  When autoregistering, descriptions are strongly encouraged.`)
		}
		if configObj.MaxRunTime != 0 && !configObj.RegForTaskMgr {
			pzsvc.LogInfo(s, `Config: MaxRunTime not meaningful unless registering for task manager use.`)
		}

	}

	if configObj.Port <= 0 && configObj.PortEnVar == "" {
		pzsvc.LogInfo(s, `Config: Neither Port nor PortEnVar were properly specified.  Default to Port 8080.`)
	}
	if configObj.Port > 0 && configObj.PortEnVar != "" {
		pzsvc.LogInfo(s, `Config: Both Port and PortEnVar were specified.  Redundant.  Default to PortEnVar.`)
	}

	return canReg
}

// PrintHelp prints out a basic helpfile to make things easier on direct users
func PrintHelp(w http.ResponseWriter) {
	fmt.Fprintln(w, `The pzsvc-exec service endpoints are as follows:`)
	fmt.Fprintln(w, `- '/': entry point.  Displays base command if any, and suggests other endpoints.`)
	fmt.Fprintln(w, `- '/execute': The meat of the program.  Downloads files, executes on them, and uploads the results.`)
	fmt.Fprintln(w, `See the Service Request Format section of the Readme for interface details.`)
	fmt.Fprintln(w, `(Readme available at https://github.com/venicegeo/pzsvc-exec).`)
	fmt.Fprintln(w, `- '/description': When enabled, provides a description of this particular pzsvc-exec instance.`)
	fmt.Fprintln(w, `- '/attributes': When enabled, provides a list of key/value attributes for this pzsvc-exec instance.`)
	fmt.Fprintln(w, `- '/version': When enabled, provides version number for the application served by this pzsvc-exec instance.`)
	fmt.Fprintln(w, `- '/help': This screen.`)
}
<|endoftext|>


<commit_before>/*
   Copyright (c) 2014, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package qan

import (
	"fmt"
	"github.com/percona/cloud-protocol/proto"
	"github.com/percona/go-mysql/event"
	"github.com/percona/go-mysql/log"
	parser "github.com/percona/go-mysql/log/slow"
	"github.com/percona/go-mysql/query"
	"github.com/percona/percona-agent/mysql"
	"github.com/percona/percona-agent/pct"
	"os"
	"strings"
	"time"
)

type Job struct {
	Id             string
	SlowLogFile    string
	RunTime        time.Duration
	StartOffset    int64
	EndOffset      int64
	ExampleQueries bool
	// --
	ZeroRunTime bool // testing
}

func (j *Job) String() string {
	return fmt.Sprintf("%s %d-%d", j.SlowLogFile, j.StartOffset, j.EndOffset)
}

type Worker interface {
	Name() string
	Status() string
	Run(job *Job) (*Result, error)
}

type WorkerFactory interface {
	Make(collectFrom, name string, mysqlConn mysql.Connector) Worker
}

// --------------------------------------------------------------------------

type RealWorkerFactory struct {
	logChan chan *proto.LogEntry
}

func NewRealWorkerFactory(logChan chan *proto.LogEntry) *RealWorkerFactory {
	f := &RealWorkerFactory{
		logChan: logChan,
	}
	return f
}

func (f *RealWorkerFactory) Make(collectFrom, name string, mysqlConn mysql.Connector) Worker {
	switch collectFrom {
	case "slowlog":
		return NewSlowLogWorker(pct.NewLogger(f.logChan, "qan-worker"), name)
	case "perfschema":
		return NewPfsWorker(pct.NewLogger(f.logChan, "qan-worker"), name, mysqlConn)
	}
	return nil
}

// --------------------------------------------------------------------------

type SlowLogWorker struct {
	logger *pct.Logger
	name   string
	// --
	status          *pct.Status
	queryChan       chan string
	fingerprintChan chan string
	errChan         chan interface{}
	doneChan        chan bool
}

func NewSlowLogWorker(logger *pct.Logger, name string) *SlowLogWorker {
	w := &SlowLogWorker{
		logger: logger,
		name:   name,
		// --
		status:          pct.NewStatus([]string{name}),
		queryChan:       make(chan string, 1),
		fingerprintChan: make(chan string, 1),
		errChan:         make(chan interface{}, 1),
		doneChan:        make(chan bool, 1),
	}
	return w
}

func (w *SlowLogWorker) Name() string {
	return w.name
}

func (w *SlowLogWorker) Status() string {
	return w.status.Get(w.name)
}

func (w *SlowLogWorker) Run(job *Job) (*Result, error) {
	w.logger.Debug("Run:call")
	defer w.logger.Debug("Run:return")

	w.status.Update(w.name, "Starting job "+job.Id)
	result := &Result{}

	// Open the slow log file.
	file, err := os.Open(job.SlowLogFile)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	// Create a slow log parser and run it.  It sends log.Event via its channel.
	// Be sure to stop it when done, else we'll leak goroutines.
	opts := log.Options{
		StartOffset: uint64(job.StartOffset),
		FilterAdminCommand: map[string]bool{
			"Binlog Dump":      true,
			"Binlog Dump GTID": true,
		},
	}
	p := parser.NewSlowLogParser(file, opts)
	defer p.Stop()
	go func() {
		defer func() {
			if err := recover(); err != nil {
				errMsg := fmt.Sprintf("Slow log parser for %s crashed: %s", job, err)
				w.logger.Error(errMsg)
				result.Error = errMsg
			}
		}()
		if err := p.Start(); err != nil {
			w.logger.Warn(err)
			result.Error = err.Error()
		}
	}()

	// Make an event aggregate to do all the heavy lifting: fingerprint
	// queries, group, and aggregate.
	a := event.NewEventAggregator(job.ExampleQueries)

	// Misc runtime meta data.
	jobSize := job.EndOffset - job.StartOffset
	runtime := time.Duration(0)
	progress := "Not started"
	rateType := ""
	rateLimit := uint(0)

	go w.fingerprinter()
	defer func() { w.doneChan <- true }()

	t0 := time.Now()
EVENT_LOOP:
	for event := range p.EventChan() {
		runtime = time.Now().Sub(t0)
		progress = fmt.Sprintf("%.1f%% %d/%d %d %.1fs",
			float64(event.Offset)/float64(job.EndOffset)*100, event.Offset, job.EndOffset, jobSize, runtime.Seconds())
		w.status.Update(w.name, fmt.Sprintf("Parsing %s: %s", job.SlowLogFile, progress))

		// Check runtime, stop if exceeded.
		if runtime >= job.RunTime {
			errMsg := fmt.Sprintf("Timeout parsing %s: %s", job, progress)
			w.logger.Warn(errMsg)
			result.Error = errMsg
			break EVENT_LOOP
		}

		if int64(event.Offset) >= job.EndOffset {
			result.StopOffset = int64(event.Offset)
			break EVENT_LOOP
		}

		if event.RateType != "" {
			if rateType != "" {
				if rateType != event.RateType || rateLimit != event.RateLimit {
					errMsg := fmt.Sprintf("Slow log has mixed rate limits: %s/%d and %s/%d",
						rateType, rateLimit, event.RateType, event.RateLimit)
					w.logger.Warn(errMsg)
					result.Error = errMsg
					break EVENT_LOOP
				}
			} else {
				rateType = event.RateType
				rateLimit = event.RateLimit
			}
		}

		var fingerprint string
		w.queryChan <- event.Query
		select {
		case fingerprint = <-w.fingerprintChan:
			id := query.Id(fingerprint)
			a.AddEvent(event, id, fingerprint)
		case _ = <-w.errChan:
			w.logger.Warn(fmt.Sprintf("Cannot fingerprint '%s'", event.Query))
			go w.fingerprinter()
		}
	}

	if result.StopOffset == 0 {
		result.StopOffset, _ = file.Seek(0, os.SEEK_CUR)
	}

	// Finalize the global and class metrics, i.e. calculate metric stats.
	w.status.Update(w.name, "Finalizing job "+job.Id)
	r := a.Finalize()

	// The aggregator result is a map, but we need an array of classes for
	// the query report, so convert it.
	n := len(r.Class)
	classes := make([]*event.QueryClass, n)
	for _, class := range r.Class {
		n-- // can't classes[--n] in Go
		classes[n] = class
	}
	result.Global = r.Global
	result.Class = classes

	// Zero the runtime for testing.
	if !job.ZeroRunTime {
		result.RunTime = time.Now().Sub(t0).Seconds()
	}

	w.status.Update(w.name, "Done job "+job.Id)
	w.logger.Info(fmt.Sprintf("Parsed %s: %s", job, progress))
	return result, nil
}

func (w *SlowLogWorker) fingerprinter() {
	w.logger.Debug("fingerprinter:call")
	defer w.logger.Debug("fingerprinter:return")
	defer func() {
		if err := recover(); err != nil {
			w.errChan <- err
		}
	}()
	for {
		select {
		case q := <-w.queryChan:
			f := query.Fingerprint(q)
			w.fingerprintChan <- f
		case <-w.doneChan:
			return
		}
	}
}
<commit_msg>PCT-1085<commit_after>/*
   Copyright (c) 2014, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package qan

import (
	"fmt"
	"github.com/percona/cloud-protocol/proto"
	"github.com/percona/go-mysql/event"
	"github.com/percona/go-mysql/log"
	parser "github.com/percona/go-mysql/log/slow"
	"github.com/percona/go-mysql/query"
	"github.com/percona/percona-agent/mysql"
	"github.com/percona/percona-agent/pct"
	"os"
	"time"
)

type Job struct {
	Id             string
	SlowLogFile    string
	RunTime        time.Duration
	StartOffset    int64
	EndOffset      int64
	ExampleQueries bool
	// --
	ZeroRunTime bool // testing
}

func (j *Job) String() string {
	return fmt.Sprintf("%s %d-%d", j.SlowLogFile, j.StartOffset, j.EndOffset)
}

type Worker interface {
	Name() string
	Status() string
	Run(job *Job) (*Result, error)
}

type WorkerFactory interface {
	Make(collectFrom, name string, mysqlConn mysql.Connector) Worker
}

// --------------------------------------------------------------------------

type RealWorkerFactory struct {
	logChan chan *proto.LogEntry
}

func NewRealWorkerFactory(logChan chan *proto.LogEntry) *RealWorkerFactory {
	f := &RealWorkerFactory{
		logChan: logChan,
	}
	return f
}

func (f *RealWorkerFactory) Make(collectFrom, name string, mysqlConn mysql.Connector) Worker {
	switch collectFrom {
	case "slowlog":
		return NewSlowLogWorker(pct.NewLogger(f.logChan, "qan-worker"), name)
	case "perfschema":
		return NewPfsWorker(pct.NewLogger(f.logChan, "qan-worker"), name, mysqlConn)
	}
	return nil
}

// --------------------------------------------------------------------------

type SlowLogWorker struct {
	logger *pct.Logger
	name   string
	// --
	status          *pct.Status
	queryChan       chan string
	fingerprintChan chan string
	errChan         chan interface{}
	doneChan        chan bool
}

func NewSlowLogWorker(logger *pct.Logger, name string) *SlowLogWorker {
	w := &SlowLogWorker{
		logger: logger,
		name:   name,
		// --
		status:          pct.NewStatus([]string{name}),
		queryChan:       make(chan string, 1),
		fingerprintChan: make(chan string, 1),
		errChan:         make(chan interface{}, 1),
		doneChan:        make(chan bool, 1),
	}
	return w
}

func (w *SlowLogWorker) Name() string {
	return w.name
}

func (w *SlowLogWorker) Status() string {
	return w.status.Get(w.name)
}

func (w *SlowLogWorker) Run(job *Job) (*Result, error) {
	w.logger.Debug("Run:call")
	defer w.logger.Debug("Run:return")

	w.status.Update(w.name, "Starting job "+job.Id)
	result := &Result{}

	// Open the slow log file.
	file, err := os.Open(job.SlowLogFile)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	// Create a slow log parser and run it.  It sends log.Event via its channel.
	// Be sure to stop it when done, else we'll leak goroutines.
	opts := log.Options{
		StartOffset: uint64(job.StartOffset),
		FilterAdminCommand: map[string]bool{
			"Binlog Dump":      true,
			"Binlog Dump GTID": true,
		},
	}
	p := parser.NewSlowLogParser(file, opts)
	defer p.Stop()
	go func() {
		defer func() {
			if err := recover(); err != nil {
				errMsg := fmt.Sprintf("Slow log parser for %s crashed: %s", job, err)
				w.logger.Error(errMsg)
				result.Error = errMsg
			}
		}()
		if err := p.Start(); err != nil {
			w.logger.Warn(err)
			result.Error = err.Error()
		}
	}()

	// Make an event aggregate to do all the heavy lifting: fingerprint
	// queries, group, and aggregate.
	a := event.NewEventAggregator(job.ExampleQueries)

	// Misc runtime meta data.
	jobSize := job.EndOffset - job.StartOffset
	runtime := time.Duration(0)
	progress := "Not started"
	rateType := ""
	rateLimit := uint(0)

	go w.fingerprinter()
	defer func() { w.doneChan <- true }()

	t0 := time.Now()
EVENT_LOOP:
	for event := range p.EventChan() {
		runtime = time.Now().Sub(t0)
		progress = fmt.Sprintf("%.1f%% %d/%d %d %.1fs",
			float64(event.Offset)/float64(job.EndOffset)*100, event.Offset, job.EndOffset, jobSize, runtime.Seconds())
		w.status.Update(w.name, fmt.Sprintf("Parsing %s: %s", job.SlowLogFile, progress))

		// Check runtime, stop if exceeded.
		if runtime >= job.RunTime {
			errMsg := fmt.Sprintf("Timeout parsing %s: %s", job, progress)
			w.logger.Warn(errMsg)
			result.Error = errMsg
			break EVENT_LOOP
		}

		if int64(event.Offset) >= job.EndOffset {
			result.StopOffset = int64(event.Offset)
			break EVENT_LOOP
		}

		if event.RateType != "" {
			if rateType != "" {
				if rateType != event.RateType || rateLimit != event.RateLimit {
					errMsg := fmt.Sprintf("Slow log has mixed rate limits: %s/%d and %s/%d",
						rateType, rateLimit, event.RateType, event.RateLimit)
					w.logger.Warn(errMsg)
					result.Error = errMsg
					break EVENT_LOOP
				}
			} else {
				rateType = event.RateType
				rateLimit = event.RateLimit
			}
		}

		var fingerprint string
		w.queryChan <- event.Query
		select {
		case fingerprint = <-w.fingerprintChan:
			id := query.Id(fingerprint)
			a.AddEvent(event, id, fingerprint)
		case _ = <-w.errChan:
			w.logger.Warn(fmt.Sprintf("Cannot fingerprint '%s'", event.Query))
			go w.fingerprinter()
		}
	}

	if result.StopOffset == 0 {
		result.StopOffset, _ = file.Seek(0, os.SEEK_CUR)
	}

	// Finalize the global and class metrics, i.e. calculate metric stats.
	w.status.Update(w.name, "Finalizing job "+job.Id)
	r := a.Finalize()

	// The aggregator result is a map, but we need an array of classes for
	// the query report, so convert it.
	n := len(r.Class)
	classes := make([]*event.QueryClass, n)
	for _, class := range r.Class {
		n-- // can't classes[--n] in Go
		classes[n] = class
	}
	result.Global = r.Global
	result.Class = classes

	// Zero the runtime for testing.
	if !job.ZeroRunTime {
		result.RunTime = time.Now().Sub(t0).Seconds()
	}

	w.status.Update(w.name, "Done job "+job.Id)
	w.logger.Info(fmt.Sprintf("Parsed %s: %s", job, progress))
	return result, nil
}

func (w *SlowLogWorker) fingerprinter() {
	w.logger.Debug("fingerprinter:call")
	defer w.logger.Debug("fingerprinter:return")
	defer func() {
		if err := recover(); err != nil {
			w.errChan <- err
		}
	}()
	for {
		select {
		case q := <-w.queryChan:
			f := query.Fingerprint(q)
			w.fingerprintChan <- f
		case <-w.doneChan:
			return
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"fmt"

	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
	"github.com/coreos/etcd/client"
)

func NewLsCommand() cli.Command {
	return cli.Command{
		Name:      "ls",
		Usage:     "retrieve a directory",
		ArgsUsage: "[key]",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "sort", Usage: "returns result in sorted order"},
			cli.BoolFlag{Name: "recursive", Usage: "returns all key names recursively for the given path"},
			cli.BoolFlag{Name: "p", Usage: "append slash (/) to directories"},
		},
		Action: func(c *cli.Context) {
			lsCommandFunc(c, mustNewKeyAPI(c))
		},
	}
}

// lsCommandFunc executes the "ls" command.
func lsCommandFunc(c *cli.Context, ki client.KeysAPI) {
	key := "/"
	if len(c.Args()) != 0 {
		key = c.Args()[0]
	}

	sort := c.Bool("sort")
	recursive := c.Bool("recursive")

	ctx, cancel := contextWithTotalTimeout(c)
	resp, err := ki.Get(ctx, key, &client.GetOptions{Sort: sort, Recursive: recursive})
	cancel()
	if err != nil {
		handleError(ExitServerError, err)
	}

	printLs(c, resp)
}

// printLs writes a response out in a manner similar to the `ls` command in unix.
// Non-empty directories list their contents and files list their name.
func printLs(c *cli.Context, resp *client.Response) {
	if !resp.Node.Dir {
		fmt.Println(resp.Node.Key)
	}
	for _, node := range resp.Node.Nodes {
		rPrint(c, node)
	}
}

// rPrint recursively prints out the nodes in the node structure.
func rPrint(c *cli.Context, n *client.Node) {
	if n.Dir && c.Bool("p") {
		fmt.Println(fmt.Sprintf("%v/", n.Key))
	} else {
		fmt.Println(n.Key)
	}

	for _, node := range n.Nodes {
		rPrint(c, node)
	}
}
<commit_msg>etcdctl: add an option to ls for consistent result<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"fmt"

	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
	"github.com/coreos/etcd/client"
)

func NewLsCommand() cli.Command {
	return cli.Command{
		Name:      "ls",
		Usage:     "retrieve a directory",
		ArgsUsage: "[key]",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "sort", Usage: "returns result in sorted order"},
			cli.BoolFlag{Name: "recursive", Usage: "returns all key names recursively for the given path"},
			cli.BoolFlag{Name: "p", Usage: "append slash (/) to directories"},
			cli.BoolFlag{Name: "quorum", Usage: "require quorum for get request"},
		},
		Action: func(c *cli.Context) {
			lsCommandFunc(c, mustNewKeyAPI(c))
		},
	}
}

// lsCommandFunc executes the "ls" command.
func lsCommandFunc(c *cli.Context, ki client.KeysAPI) {
	key := "/"
	if len(c.Args()) != 0 {
		key = c.Args()[0]
	}

	sort := c.Bool("sort")
	recursive := c.Bool("recursive")
	quorum := c.Bool("quorum")

	ctx, cancel := contextWithTotalTimeout(c)
	resp, err := ki.Get(ctx, key, &client.GetOptions{Sort: sort, Recursive: recursive, Quorum: quorum})
	cancel()
	if err != nil {
		handleError(ExitServerError, err)
	}

	printLs(c, resp)
}

// printLs writes a response out in a manner similar to the `ls` command in unix.
// Non-empty directories list their contents and files list their name.
func printLs(c *cli.Context, resp *client.Response) {
	if !resp.Node.Dir {
		fmt.Println(resp.Node.Key)
	}
	for _, node := range resp.Node.Nodes {
		rPrint(c, node)
	}
}

// rPrint recursively prints out the nodes in the node structure.
func rPrint(c *cli.Context, n *client.Node) {
	if n.Dir && c.Bool("p") {
		fmt.Println(fmt.Sprintf("%v/", n.Key))
	} else {
		fmt.Println(n.Key)
	}

	for _, node := range n.Nodes {
		rPrint(c, node)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package retrieval

import (
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/extraction"
	"github.com/prometheus/client_golang/prometheus"

	clientmodel "github.com/prometheus/client_golang/model"

	"github.com/prometheus/prometheus/utility"
)

const (
	InstanceLabel clientmodel.LabelName = "instance"
	// The metric name for the synthetic health variable.
	ScrapeHealthMetricName clientmodel.LabelValue = "up"

	// Constants for instrumentation.
	namespace = "prometheus"
	job       = "target_job"
	instance  = "target_instance"
	failure   = "failure"
	outcome   = "outcome"
	success   = "success"
	interval  = "interval"
)

var (
	localhostRepresentations = []string{"http://127.0.0.1", "http://localhost"}

	targetOperationLatencies = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace:  namespace,
			Name:       "target_operation_latency_milliseconds",
			Help:       "The latencies for target operations.",
			Objectives: []float64{0.01, 0.05, 0.5, 0.90, 0.99},
		},
		[]string{job, instance, outcome},
	)
	targetIntervalLength = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace:  namespace,
			Name:       "target_interval_length_seconds",
			Help:       "Actual intervals between scrapes.",
			Objectives: []float64{0.01, 0.05, 0.5, 0.90, 0.99},
		},
		[]string{interval},
	)
)

func init() {
	prometheus.MustRegister(targetOperationLatencies)
	prometheus.MustRegister(targetIntervalLength)
}

// The state of the given Target.
type TargetState int

func (t TargetState) String() string {
	switch t {
	case UNKNOWN:
		return "UNKNOWN"
	case ALIVE:
		return "ALIVE"
	case UNREACHABLE:
		return "UNREACHABLE"
	}

	panic("unknown state")
}

const (
	// The Target has not been seen; we know nothing about it, except that it is
	// on our docket for examination.
	UNKNOWN TargetState = iota
	// The Target has been found and successfully queried.
	ALIVE
	// The Target was either historically found or not found and then determined
	// to be unhealthy by either not responding or disappearing.
	UNREACHABLE
)

// A Target represents an endpoint that should be interrogated for metrics.
//
// The protocol described by this type will likely change in future iterations,
// as it offers no good support for aggregated targets and fan out.  Thusly,
// it is likely that the current Target and target uses will be
// wrapped with some resolver type.
//
// For the future, the Target protocol will abstract away the exact means that
// metrics are retrieved and deserialized from the given instance to which it
// refers.
type Target interface {
	// Return the last encountered scrape error, if any.
	LastError() error
	// Return the health of the target.
	State() TargetState
	// Return the last time a scrape was attempted.
	LastScrape() time.Time
	// The address to which the Target corresponds.  Out of all of the available
	// points in this interface, this one is the best candidate to change given
	// the ways to express the endpoint.
	Address() string
	// The address as seen from other hosts. References to localhost are resolved
	// to the address of the prometheus server.
	GlobalAddress() string
	// Return the target's base labels.
	BaseLabels() clientmodel.LabelSet
	// SetBaseLabelsFrom queues a replacement of the current base labels by
	// the labels of the given target. The method returns immediately after
	// queuing. The actual replacement of the base labels happens
	// asynchronously (but most likely before the next scrape for the target
	// begins).
	SetBaseLabelsFrom(Target)
	// Scrape target at the specified interval.
	RunScraper(extraction.Ingester, time.Duration)
	// Stop scraping, synchronous.
	StopScraper()
}

// target is a Target that refers to a singular HTTP or HTTPS endpoint.
type target struct {
	// The current health state of the target.
	state TargetState
	// The last encountered scrape error, if any.
	lastError error
	// The last time a scrape was attempted.
	lastScrape time.Time
	// Closing scraperStopping signals that scraping should stop.
	scraperStopping chan struct{}
	// Closing scraperStopped signals that scraping has been stopped.
	scraperStopped chan struct{}
	// Channel to queue base labels to be replaced.
	newBaseLabels chan clientmodel.LabelSet

	address string
	// What is the deadline for the HTTP or HTTPS against this endpoint.
	Deadline time.Duration
	// Any base labels that are added to this target and its metrics.
	baseLabels clientmodel.LabelSet
	// The HTTP client used to scrape the target's endpoint.
	httpClient *http.Client

	// Mutex protects lastError, lastScrape, state, and baseLabels.  Writing
	// the above must only happen in the goroutine running the RunScraper
	// loop, and it must happen under the lock. In that way, no mutex lock
	// is required for reading the above in the goroutine running the
	// RunScraper loop, but only for reading in other goroutines.
	sync.Mutex
}

// Furnish a reasonably configured target for querying.
func NewTarget(address string, deadline time.Duration, baseLabels clientmodel.LabelSet) Target {
	target := &target{
		address:         address,
		Deadline:        deadline,
		baseLabels:      baseLabels,
		httpClient:      utility.NewDeadlineClient(deadline),
		scraperStopping: make(chan struct{}),
		scraperStopped:  make(chan struct{}),
		newBaseLabels:   make(chan clientmodel.LabelSet, 1),
	}

	return target
}

func (t *target) recordScrapeHealth(ingester extraction.Ingester, timestamp clientmodel.Timestamp, healthy bool) {
	metric := clientmodel.Metric{}
	for label, value := range t.baseLabels {
		metric[label] = value
	}
	metric[clientmodel.MetricNameLabel] = clientmodel.LabelValue(ScrapeHealthMetricName)
	metric[InstanceLabel] = clientmodel.LabelValue(t.Address())

	healthValue := clientmodel.SampleValue(0)
	if healthy {
		healthValue = clientmodel.SampleValue(1)
	}

	sample := &clientmodel.Sample{
		Metric:    metric,
		Timestamp: timestamp,
		Value:     healthValue,
	}

	ingester.Ingest(&extraction.Result{
		Err:     nil,
		Samples: clientmodel.Samples{sample},
	})
}

// RunScraper implements Target.
func (t *target) RunScraper(ingester extraction.Ingester, interval time.Duration) {
	defer func() {
		// Need to drain t.newBaseLabels to not make senders block during shutdown.
		for {
			select {
			case <-t.newBaseLabels:
				// Do nothing.
			default:
				close(t.scraperStopped)
				return
			}
		}
	}()

	jitterTimer := time.NewTimer(time.Duration(float64(interval) * rand.Float64()))
	select {
	case <-jitterTimer.C:
	case <-t.scraperStopping:
		jitterTimer.Stop()
		return
	}
	jitterTimer.Stop()

	ticker := time.NewTicker(interval)
	defer ticker.Stop()

	t.Lock() // Writing t.lastScrape requires the lock.
	t.lastScrape = time.Now()
	t.Unlock()
	t.scrape(ingester)

	// Explanation of the contraption below:
	//
	// In case t.newBaseLabels or t.scraperStopping have something to receive,
	// we want to read from those channels rather than starting a new scrape
	// (which might take very long). That's why the outer select has no
	// ticker.C. Should neither t.newBaseLabels nor t.scraperStopping have
	// anything to receive, we go into the inner select, where ticker.C is
	// in the mix.
	for {
		select {
		case newBaseLabels := <-t.newBaseLabels:
			t.Lock() // Writing t.baseLabels requires the lock.
			t.baseLabels = newBaseLabels
			t.Unlock()
		case <-t.scraperStopping:
			return
		default:
			select {
			case newBaseLabels := <-t.newBaseLabels:
				t.Lock() // Writing t.baseLabels requires the lock.
				t.baseLabels = newBaseLabels
				t.Unlock()
			case <-t.scraperStopping:
				return
			case <-ticker.C:
				targetIntervalLength.WithLabelValues(interval.String()).Observe(float64(time.Since(t.lastScrape) / time.Second))
				t.Lock() // Write t.lastScrape requires locking.
				t.lastScrape = time.Now()
				t.Unlock()
				t.scrape(ingester)
			}
		}
	}
}

// StopScraper implements Target.
func (t *target) StopScraper() {
	close(t.scraperStopping)
	<-t.scraperStopped
}

const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3,application/json;schema="prometheus/telemetry";version=0.0.2;q=0.2,*/*;q=0.1`

func (t *target) scrape(ingester extraction.Ingester) (err error) {
	timestamp := clientmodel.Now()
	defer func(start time.Time) {
		ms := float64(time.Since(start)) / float64(time.Millisecond)
		labels := prometheus.Labels{
			job:      string(t.baseLabels[clientmodel.JobLabel]),
			instance: t.Address(),
			outcome:  success,
		}
		t.Lock() // Writing t.state and t.lastError requires the lock.
		if err == nil {
			t.state = ALIVE
			t.recordScrapeHealth(ingester, timestamp, true)
			labels[outcome] = failure
		} else {
			t.state = UNREACHABLE
			t.recordScrapeHealth(ingester, timestamp, false)
		}
		targetOperationLatencies.With(labels).Observe(ms)
		t.lastError = err
		t.Unlock()
	}(time.Now())

	req, err := http.NewRequest("GET", t.Address(), nil)
	if err != nil {
		panic(err)
	}
	req.Header.Add("Accept", acceptHeader)

	resp, err := t.httpClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("server returned HTTP status %s", resp.Status)
	}

	processor, err := extraction.ProcessorForRequestHeader(resp.Header)
	if err != nil {
		return err
	}

	baseLabels := clientmodel.LabelSet{InstanceLabel: clientmodel.LabelValue(t.Address())}
	for baseLabel, baseValue := range t.baseLabels {
		baseLabels[baseLabel] = baseValue
	}

	i := &MergeLabelsIngester{
		Labels:          baseLabels,
		CollisionPrefix: clientmodel.ExporterLabelPrefix,

		Ingester: ingester,
	}
	processOptions := &extraction.ProcessOptions{
		Timestamp: timestamp,
	}
	return processor.ProcessSingle(resp.Body, i, processOptions)
}

// LastError implements Target.
func (t *target) LastError() error {
	t.Lock()
	defer t.Unlock()
	return t.lastError
}

// State implements Target.
func (t *target) State() TargetState {
	t.Lock()
	defer t.Unlock()
	return t.state
}

// LastScrape implements Target.
func (t *target) LastScrape() time.Time {
	t.Lock()
	defer t.Unlock()
	return t.lastScrape
}

// Address implements Target.
func (t *target) Address() string {
	return t.address
}

// GlobalAddress implements Target.
func (t *target) GlobalAddress() string {
	address := t.address
	hostname, err := os.Hostname()
	if err != nil {
		glog.Warningf("Couldn't get hostname: %s, returning target.Address()", err)
		return address
	}
	for _, localhostRepresentation := range localhostRepresentations {
		address = strings.Replace(address, localhostRepresentation, fmt.Sprintf("http://%s", hostname), -1)
	}
	return address
}

// BaseLabels implements Target.
func (t *target) BaseLabels() clientmodel.LabelSet {
	t.Lock()
	defer t.Unlock()
	return t.baseLabels
}

// SetBaseLabelsFrom implements Target.
func (t *target) SetBaseLabelsFrom(newTarget Target) {
	if t.Address() != newTarget.Address() {
		panic("targets don't refer to the same endpoint")
	}
	t.newBaseLabels <- newTarget.BaseLabels()
}
<commit_msg>Reduce lock-protected area during scrape.<commit_after>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package retrieval

import (
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/extraction"
	"github.com/prometheus/client_golang/prometheus"

	clientmodel "github.com/prometheus/client_golang/model"

	"github.com/prometheus/prometheus/utility"
)

const (
	InstanceLabel clientmodel.LabelName = "instance"
	// The metric name for the synthetic health variable.
	ScrapeHealthMetricName clientmodel.LabelValue = "up"

	// Constants for instrumentation.
	namespace = "prometheus"
	job       = "target_job"
	instance  = "target_instance"
	failure   = "failure"
	outcome   = "outcome"
	success   = "success"
	interval  = "interval"
)

var (
	localhostRepresentations = []string{"http://127.0.0.1", "http://localhost"}

	targetOperationLatencies = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace:  namespace,
			Name:       "target_operation_latency_milliseconds",
			Help:       "The latencies for target operations.",
			Objectives: []float64{0.01, 0.05, 0.5, 0.90, 0.99},
		},
		[]string{job, instance, outcome},
	)
	targetIntervalLength = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace:  namespace,
			Name:       "target_interval_length_seconds",
			Help:       "Actual intervals between scrapes.",
			Objectives: []float64{0.01, 0.05, 0.5, 0.90, 0.99},
		},
		[]string{interval},
	)
)

func init() {
	prometheus.MustRegister(targetOperationLatencies)
	prometheus.MustRegister(targetIntervalLength)
}

// The state of the given Target.
type TargetState int

func (t TargetState) String() string {
	switch t {
	case UNKNOWN:
		return "UNKNOWN"
	case ALIVE:
		return "ALIVE"
	case UNREACHABLE:
		return "UNREACHABLE"
	}

	panic("unknown state")
}

const (
	// The Target has not been seen; we know nothing about it, except that it is
	// on our docket for examination.
	UNKNOWN TargetState = iota
	// The Target has been found and successfully queried.
	ALIVE
	// The Target was either historically found or not found and then determined
	// to be unhealthy by either not responding or disappearing.
	UNREACHABLE
)

// A Target represents an endpoint that should be interrogated for metrics.
//
// The protocol described by this type will likely change in future iterations,
// as it offers no good support for aggregated targets and fan out.  Thusly,
// it is likely that the current Target and target uses will be
// wrapped with some resolver type.
//
// For the future, the Target protocol will abstract away the exact means that
// metrics are retrieved and deserialized from the given instance to which it
// refers.
type Target interface {
	// Return the last encountered scrape error, if any.
	LastError() error
	// Return the health of the target.
	State() TargetState
	// Return the last time a scrape was attempted.
	LastScrape() time.Time
	// The address to which the Target corresponds.  Out of all of the available
	// points in this interface, this one is the best candidate to change given
	// the ways to express the endpoint.
	Address() string
	// The address as seen from other hosts. References to localhost are resolved
	// to the address of the prometheus server.
	GlobalAddress() string
	// Return the target's base labels.
	BaseLabels() clientmodel.LabelSet
	// SetBaseLabelsFrom queues a replacement of the current base labels by
	// the labels of the given target. The method returns immediately after
	// queuing. The actual replacement of the base labels happens
	// asynchronously (but most likely before the next scrape for the target
	// begins).
	SetBaseLabelsFrom(Target)
	// Scrape target at the specified interval.
	RunScraper(extraction.Ingester, time.Duration)
	// Stop scraping, synchronous.
	StopScraper()
}

// target is a Target that refers to a singular HTTP or HTTPS endpoint.
type target struct {
	// The current health state of the target.
	state TargetState
	// The last encountered scrape error, if any.
	lastError error
	// The last time a scrape was attempted.
	lastScrape time.Time
	// Closing scraperStopping signals that scraping should stop.
	scraperStopping chan struct{}
	// Closing scraperStopped signals that scraping has been stopped.
	scraperStopped chan struct{}
	// Channel to queue base labels to be replaced.
	newBaseLabels chan clientmodel.LabelSet

	address string
	// What is the deadline for the HTTP or HTTPS against this endpoint.
	Deadline time.Duration
	// Any base labels that are added to this target and its metrics.
	baseLabels clientmodel.LabelSet
	// The HTTP client used to scrape the target's endpoint.
	httpClient *http.Client

	// Mutex protects lastError, lastScrape, state, and baseLabels.  Writing
	// the above must only happen in the goroutine running the RunScraper
	// loop, and it must happen under the lock. In that way, no mutex lock
	// is required for reading the above in the goroutine running the
	// RunScraper loop, but only for reading in other goroutines.
	sync.Mutex
}

// Furnish a reasonably configured target for querying.
func NewTarget(address string, deadline time.Duration, baseLabels clientmodel.LabelSet) Target {
	target := &target{
		address:         address,
		Deadline:        deadline,
		baseLabels:      baseLabels,
		httpClient:      utility.NewDeadlineClient(deadline),
		scraperStopping: make(chan struct{}),
		scraperStopped:  make(chan struct{}),
		newBaseLabels:   make(chan clientmodel.LabelSet, 1),
	}

	return target
}

func (t *target) recordScrapeHealth(ingester extraction.Ingester, timestamp clientmodel.Timestamp, healthy bool) {
	metric := clientmodel.Metric{}
	for label, value := range t.baseLabels {
		metric[label] = value
	}
	metric[clientmodel.MetricNameLabel] = clientmodel.LabelValue(ScrapeHealthMetricName)
	metric[InstanceLabel] = clientmodel.LabelValue(t.Address())

	healthValue := clientmodel.SampleValue(0)
	if healthy {
		healthValue = clientmodel.SampleValue(1)
	}

	sample := &clientmodel.Sample{
		Metric:    metric,
		Timestamp: timestamp,
		Value:     healthValue,
	}

	ingester.Ingest(&extraction.Result{
		Err:     nil,
		Samples: clientmodel.Samples{sample},
	})
}

// RunScraper implements Target.
func (t *target) RunScraper(ingester extraction.Ingester, interval time.Duration) {
	defer func() {
		// Need to drain t.newBaseLabels to not make senders block during shutdown.
		for {
			select {
			case <-t.newBaseLabels:
				// Do nothing.
			default:
				close(t.scraperStopped)
				return
			}
		}
	}()

	jitterTimer := time.NewTimer(time.Duration(float64(interval) * rand.Float64()))
	select {
	case <-jitterTimer.C:
	case <-t.scraperStopping:
		jitterTimer.Stop()
		return
	}
	jitterTimer.Stop()

	ticker := time.NewTicker(interval)
	defer ticker.Stop()

	t.Lock() // Writing t.lastScrape requires the lock.
	t.lastScrape = time.Now()
	t.Unlock()
	t.scrape(ingester)

	// Explanation of the contraption below:
	//
	// In case t.newBaseLabels or t.scraperStopping have something to receive,
	// we want to read from those channels rather than starting a new scrape
	// (which might take very long). That's why the outer select has no
	// ticker.C. Should neither t.newBaseLabels nor t.scraperStopping have
	// anything to receive, we go into the inner select, where ticker.C is
	// in the mix.
	for {
		select {
		case newBaseLabels := <-t.newBaseLabels:
			t.Lock() // Writing t.baseLabels requires the lock.
			t.baseLabels = newBaseLabels
			t.Unlock()
		case <-t.scraperStopping:
			return
		default:
			select {
			case newBaseLabels := <-t.newBaseLabels:
				t.Lock() // Writing t.baseLabels requires the lock.
				t.baseLabels = newBaseLabels
				t.Unlock()
			case <-t.scraperStopping:
				return
			case <-ticker.C:
				targetIntervalLength.WithLabelValues(interval.String()).Observe(float64(time.Since(t.lastScrape) / time.Second))
				t.Lock() // Write t.lastScrape requires locking.
				t.lastScrape = time.Now()
				t.Unlock()
				t.scrape(ingester)
			}
		}
	}
}

// StopScraper implements Target.
func (t *target) StopScraper() {
	close(t.scraperStopping)
	<-t.scraperStopped
}

const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3,application/json;schema="prometheus/telemetry";version=0.0.2;q=0.2,*/*;q=0.1`

func (t *target) scrape(ingester extraction.Ingester) (err error) {
	timestamp := clientmodel.Now()
	defer func(start time.Time) {
		ms := float64(time.Since(start)) / float64(time.Millisecond)
		labels := prometheus.Labels{
			job:      string(t.baseLabels[clientmodel.JobLabel]),
			instance: t.Address(),
			outcome:  success,
		}
		t.Lock() // Writing t.state and t.lastError requires the lock.
		if err == nil {
			t.state = ALIVE
			labels[outcome] = failure
		} else {
			t.state = UNREACHABLE
		}
		t.lastError = err
		t.Unlock()
		targetOperationLatencies.With(labels).Observe(ms)
		t.recordScrapeHealth(ingester, timestamp, err == nil)
	}(time.Now())

	req, err := http.NewRequest("GET", t.Address(), nil)
	if err != nil {
		panic(err)
	}
	req.Header.Add("Accept", acceptHeader)

	resp, err := t.httpClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("server returned HTTP status %s", resp.Status)
	}

	processor, err := extraction.ProcessorForRequestHeader(resp.Header)
	if err != nil {
		return err
	}

	baseLabels := clientmodel.LabelSet{InstanceLabel: clientmodel.LabelValue(t.Address())}
	for baseLabel, baseValue := range t.baseLabels {
		baseLabels[baseLabel] = baseValue
	}

	i := &MergeLabelsIngester{
		Labels:          baseLabels,
		CollisionPrefix: clientmodel.ExporterLabelPrefix,

		Ingester: ingester,
	}
	processOptions := &extraction.ProcessOptions{
		Timestamp: timestamp,
	}
	return processor.ProcessSingle(resp.Body, i, processOptions)
}

// LastError implements Target.
func (t *target) LastError() error {
	t.Lock()
	defer t.Unlock()
	return t.lastError
}

// State implements Target.
func (t *target) State() TargetState {
	t.Lock()
	defer t.Unlock()
	return t.state
}

// LastScrape implements Target.
func (t *target) LastScrape() time.Time {
	t.Lock()
	defer t.Unlock()
	return t.lastScrape
}

// Address implements Target.
func (t *target) Address() string {
	return t.address
}

// GlobalAddress implements Target.
func (t *target) GlobalAddress() string {
	address := t.address
	hostname, err := os.Hostname()
	if err != nil {
		glog.Warningf("Couldn't get hostname: %s, returning target.Address()", err)
		return address
	}
	for _, localhostRepresentation := range localhostRepresentations {
		address = strings.Replace(address, localhostRepresentation, fmt.Sprintf("http://%s", hostname), -1)
	}
	return address
}

// BaseLabels implements Target.
func (t *target) BaseLabels() clientmodel.LabelSet {
	t.Lock()
	defer t.Unlock()
	return t.baseLabels
}

// SetBaseLabelsFrom implements Target.
func (t *target) SetBaseLabelsFrom(newTarget Target) {
	if t.Address() != newTarget.Address() {
		panic("targets don't refer to the same endpoint")
	}
	t.newBaseLabels <- newTarget.BaseLabels()
}
<|endoftext|>