<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package settings

import (
	"context"
	"fmt"
	"github.com/parnurzeal/gorequest"
	"net"
	"strings"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"k8s.io/ingress-nginx/test/e2e/framework"
)

var _ = framework.IngressNginxDescribe("TCP Feature", func() {
	f := framework.NewDefaultFramework("tcp")

	BeforeEach(func() {
	})

	AfterEach(func() {
	})

	It("should expose a TCP service", func() {
		f.NewEchoDeploymentWithReplicas(1)

		config, err := f.KubeClientSet.
			CoreV1().
			ConfigMaps(f.IngressController.Namespace).
			Get("tcp-services", metav1.GetOptions{})
		Expect(err).To(BeNil(), "unexpected error obtaining tcp-services configmap")
		Expect(config).NotTo(BeNil(), "expected a configmap but none returned")

		if config.Data == nil {
			config.Data = map[string]string{}
		}

		config.Data["8080"] = fmt.Sprintf("%v/http-svc:80", f.IngressController.Namespace)

		_, err = f.KubeClientSet.
			CoreV1().
			ConfigMaps(f.IngressController.Namespace).
			Update(config)
		Expect(err).NotTo(HaveOccurred(), "unexpected error updating configmap")

		svc, err := f.KubeClientSet.
			CoreV1().
			Services(f.IngressController.Namespace).
			Get("ingress-nginx", metav1.GetOptions{})
		Expect(err).To(BeNil(), "unexpected error obtaining ingress-nginx service")
		Expect(svc).NotTo(BeNil(), "expected a service but none returned")

		svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
			Name:       "http-svc",
			Port:       8080,
			TargetPort: intstr.FromInt(8080),
		})
		_, err = f.KubeClientSet.
			CoreV1().
			Services(f.IngressController.Namespace).
			Update(svc)
		Expect(err).NotTo(HaveOccurred(), "unexpected error updating service")

		f.WaitForNginxConfiguration(
			func(cfg string) bool {
				return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-http-svc-80"`, f.IngressController.Namespace))
			})

		ip := f.GetNginxIP()
		port, err := f.GetNginxPort("http-svc")
		Expect(err).NotTo(HaveOccurred(), "unexpected error obtaining service port")

		resp, _, errs := gorequest.New().
			Get(fmt.Sprintf("http://%v:%v", ip, port)).
			End()
		Expect(errs).Should(BeEmpty())
		Expect(resp.StatusCode).Should(Equal(200))
	})

	It("should expose an ExternalName TCP service", func() {
		// Setup:
		// - Create an external name service for DNS lookups on port 5353. Point it to google's DNS server
		// - Expose port 5353 on the nginx ingress NodePort service to open a hole for this test
		// - Update the `tcp-services` configmap to proxy traffic to the configured external name service

		// Create an external service for DNS
		externalService := &corev1.Service{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "dns-external-name-svc",
				Namespace: f.IngressController.Namespace,
			},

			Spec: corev1.ServiceSpec{
				ExternalName: "google-public-dns-a.google.com",
				Ports: []corev1.ServicePort{
					{
						Name:       "dns-external-name-svc",
						Port:       5353,
						TargetPort: intstr.FromInt(53),
					},
				},
				Type: corev1.ServiceTypeExternalName,
			},
		}
		f.EnsureService(externalService)

		// Expose the `external name` port on the `ingress-nginx` service
		svc, err := f.KubeClientSet.
			CoreV1().
			Services(f.IngressController.Namespace).
			Get("ingress-nginx", metav1.GetOptions{})
		Expect(err).To(BeNil(), "unexpected error obtaining ingress-nginx service")
		Expect(svc).NotTo(BeNil(), "expected a service but none returned")

		svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
			Name:       "dns-svc",
			Port:       5353,
			TargetPort: intstr.FromInt(5353),
		})
		_, err = f.KubeClientSet.
			CoreV1().
			Services(f.IngressController.Namespace).
			Update(svc)
		Expect(err).NotTo(HaveOccurred(), "unexpected error updating service")

		// Update the TCP configmap to link port 5353 to the DNS external name service
		config, err := f.KubeClientSet.
			CoreV1().
			ConfigMaps(f.IngressController.Namespace).
			Get("tcp-services", metav1.GetOptions{})
		Expect(err).To(BeNil(), "unexpected error obtaining tcp-services configmap")
		Expect(config).NotTo(BeNil(), "expected a configmap but none returned")

		if config.Data == nil {
			config.Data = map[string]string{}
		}

		config.Data["5353"] = fmt.Sprintf("%v/dns-external-name-svc:5353", f.IngressController.Namespace)

		_, err = f.KubeClientSet.
			CoreV1().
			ConfigMaps(f.IngressController.Namespace).
			Update(config)
		Expect(err).NotTo(HaveOccurred(), "unexpected error updating configmap")

		// Validate that the generated nginx config contains the expected `proxy_upstream_name` value
		f.WaitForNginxConfiguration(
			func(cfg string) bool {
				return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-dns-external-name-svc-5353"`, f.IngressController.Namespace))
			})

		// Execute the test. Use the `external name` service to resolve a domain name.
		ip := f.GetNginxIP()
		port, err := f.GetNginxPort("dns-svc")
		Expect(err).NotTo(HaveOccurred(), "unexpected error obtaining service port")

		resolver := net.Resolver{
			PreferGo: true,
			Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
				d := net.Dialer{}
				return d.DialContext(ctx, "tcp", fmt.Sprintf("%v:%v", ip, port))
			},
		}
		ips, err := resolver.LookupHost(context.Background(), "google-public-dns-b.google.com")
		Expect(ips).Should(ContainElement("8.8.4.4"))

	})
})
<commit_msg>Add 'waitForLuaSync' sleep in external name TCP test<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package settings

import (
	"context"
	"fmt"
	"github.com/parnurzeal/gorequest"
	"net"
	"strings"
	"time"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"k8s.io/ingress-nginx/test/e2e/framework"
)

const (
	waitForLuaSync = 5 * time.Second
)

var _ = framework.IngressNginxDescribe("TCP Feature", func() {
	f := framework.NewDefaultFramework("tcp")

	BeforeEach(func() {
	})

	AfterEach(func() {
	})

	It("should expose a TCP service", func() {
		f.NewEchoDeploymentWithReplicas(1)

		config, err := f.KubeClientSet.
			CoreV1().
			ConfigMaps(f.IngressController.Namespace).
			Get("tcp-services", metav1.GetOptions{})
		Expect(err).To(BeNil(), "unexpected error obtaining tcp-services configmap")
		Expect(config).NotTo(BeNil(), "expected a configmap but none returned")

		if config.Data == nil {
			config.Data = map[string]string{}
		}

		config.Data["8080"] = fmt.Sprintf("%v/http-svc:80", f.IngressController.Namespace)

		_, err = f.KubeClientSet.
			CoreV1().
			ConfigMaps(f.IngressController.Namespace).
			Update(config)
		Expect(err).NotTo(HaveOccurred(), "unexpected error updating configmap")

		svc, err := f.KubeClientSet.
			CoreV1().
			Services(f.IngressController.Namespace).
			Get("ingress-nginx", metav1.GetOptions{})
		Expect(err).To(BeNil(), "unexpected error obtaining ingress-nginx service")
		Expect(svc).NotTo(BeNil(), "expected a service but none returned")

		svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
			Name:       "http-svc",
			Port:       8080,
			TargetPort: intstr.FromInt(8080),
		})
		_, err = f.KubeClientSet.
			CoreV1().
			Services(f.IngressController.Namespace).
			Update(svc)
		Expect(err).NotTo(HaveOccurred(), "unexpected error updating service")

		f.WaitForNginxConfiguration(
			func(cfg string) bool {
				return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-http-svc-80"`, f.IngressController.Namespace))
			})

		ip := f.GetNginxIP()
		port, err := f.GetNginxPort("http-svc")
		Expect(err).NotTo(HaveOccurred(), "unexpected error obtaining service port")

		resp, _, errs := gorequest.New().
			Get(fmt.Sprintf("http://%v:%v", ip, port)).
			End()
		Expect(errs).Should(BeEmpty())
		Expect(resp.StatusCode).Should(Equal(200))
	})

	It("should expose an ExternalName TCP service", func() {
		// Setup:
		// - Create an external name service for DNS lookups on port 5353. Point it to google's DNS server
		// - Expose port 5353 on the nginx ingress NodePort service to open a hole for this test
		// - Update the `tcp-services` configmap to proxy traffic to the configured external name service

		// Create an external service for DNS
		externalService := &corev1.Service{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "dns-external-name-svc",
				Namespace: f.IngressController.Namespace,
			},

			Spec: corev1.ServiceSpec{
				ExternalName: "google-public-dns-a.google.com",
				Ports: []corev1.ServicePort{
					{
						Name:       "dns-external-name-svc",
						Port:       5353,
						TargetPort: intstr.FromInt(53),
					},
				},
				Type: corev1.ServiceTypeExternalName,
			},
		}
		f.EnsureService(externalService)

		// Expose the `external name` port on the `ingress-nginx` service
		svc, err := f.KubeClientSet.
			CoreV1().
			Services(f.IngressController.Namespace).
			Get("ingress-nginx", metav1.GetOptions{})
		Expect(err).To(BeNil(), "unexpected error obtaining ingress-nginx service")
		Expect(svc).NotTo(BeNil(), "expected a service but none returned")

		svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
			Name:       "dns-svc",
			Port:       5353,
			TargetPort: intstr.FromInt(5353),
		})
		_, err = f.KubeClientSet.
			CoreV1().
			Services(f.IngressController.Namespace).
			Update(svc)
		Expect(err).NotTo(HaveOccurred(), "unexpected error updating service")

		// Update the TCP configmap to link port 5353 to the DNS external name service
		config, err := f.KubeClientSet.
			CoreV1().
			ConfigMaps(f.IngressController.Namespace).
			Get("tcp-services", metav1.GetOptions{})
		Expect(err).To(BeNil(), "unexpected error obtaining tcp-services configmap")
		Expect(config).NotTo(BeNil(), "expected a configmap but none returned")

		if config.Data == nil {
			config.Data = map[string]string{}
		}

		config.Data["5353"] = fmt.Sprintf("%v/dns-external-name-svc:5353", f.IngressController.Namespace)

		_, err = f.KubeClientSet.
			CoreV1().
			ConfigMaps(f.IngressController.Namespace).
			Update(config)
		Expect(err).NotTo(HaveOccurred(), "unexpected error updating configmap")

		time.Sleep(waitForLuaSync)

		// Validate that the generated nginx config contains the expected `proxy_upstream_name` value
		f.WaitForNginxConfiguration(
			func(cfg string) bool {
				return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-dns-external-name-svc-5353"`, f.IngressController.Namespace))
			})

		// Execute the test. Use the `external name` service to resolve a domain name.
		ip := f.GetNginxIP()
		port, err := f.GetNginxPort("dns-svc")
		Expect(err).NotTo(HaveOccurred(), "unexpected error obtaining service port")

		resolver := net.Resolver{
			PreferGo: true,
			Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
				d := net.Dialer{}
				return d.DialContext(ctx, "tcp", fmt.Sprintf("%v:%v", ip, port))
			},
		}
		ips, err := resolver.LookupHost(context.Background(), "google-public-dns-b.google.com")
		Expect(err).NotTo(HaveOccurred(), "unexpected error from DNS resolver")
		Expect(ips).Should(ContainElement("8.8.4.4"))

	})
})
<|endoftext|>


<commit_before>package jiffy

import (
	"sync"
	"time"
)

// A topic coordinates the publishing and distribution
// of messages.
type Topic struct {
	Name          string
	Subscriptions map[string]*Subscription
	Data          map[string]*Message
	// Mutex for creating and destroying subscriptions.
	subscriptionMutex sync.Mutex
	// Mutex for creating and destroying messages.
	messageMutex sync.Mutex
}

func CreateTopic(name string) *Topic {
	return &Topic{
		name,
		make(map[string]*Subscription),
		make(map[string]*Message),
		sync.Mutex{},
		sync.Mutex{},
	}
}

// Publishes a message to all subscribers.
func (topic *Topic) Publish(message *Message) {
	for _, subscription := range topic.Subscriptions {
		go func(s *Subscription) {
			s.ResponseChannel <- message
		}(subscription)
	}
}

// Publishes a message and stores it in the topic.
func (topic *Topic) RecordAndPublish(message *Message, ttl time.Duration) {
	topic.messageMutex.Lock()
	defer topic.messageMutex.Unlock()
	topic.Data[message.Name] = message
	go topic.Publish(message)
	go message.QueueExpiration(topic, ttl)
}

// Creates a subscription on the topic if it doesn't exist
// and returns it.
func (topic *Topic) GetSubscription(name string, ttl time.Duration) *Subscription {
	topic.subscriptionMutex.Lock()
	defer topic.subscriptionMutex.Unlock()

	if subscription, ok := topic.Subscriptions[name]; ok {
		select {
		case subscription.expireChan <- cancelTTL:
			// If we were able to cancel successfully,
			// just restart the expiration.
			go subscription.QueueExpiration(ttl)
			return subscription
		default:
			// We're too late
		}
	}

	subscription := NewSubscription(name, topic, ttl)
	topic.Subscriptions[name] = subscription

	// Fetch the current state for the topic if this is the
	// first session.
	go subscription.FetchData()

	return subscription
}
<commit_msg>Fetch data optionally<commit_after>package jiffy

import (
	"sync"
	"time"
)

// A topic coordinates the publishing and distribution
// of messages.
type Topic struct {
	Name          string
	Subscriptions map[string]*Subscription
	Data          map[string]*Message
	// Mutex for creating and destroying subscriptions.
	subscriptionMutex sync.Mutex
	// Mutex for creating and destroying messages.
	messageMutex sync.Mutex
}

func CreateTopic(name string) *Topic {
	return &Topic{
		name,
		make(map[string]*Subscription),
		make(map[string]*Message),
		sync.Mutex{},
		sync.Mutex{},
	}
}

// Publishes a message to all subscribers.
func (topic *Topic) Publish(message *Message) {
	for _, subscription := range topic.Subscriptions {
		go func(s *Subscription) {
			s.ResponseChannel <- message
		}(subscription)
	}
}

// Publishes a message and stores it in the topic.
func (topic *Topic) RecordAndPublish(message *Message, ttl time.Duration) {
	topic.messageMutex.Lock()
	defer topic.messageMutex.Unlock()
	topic.Data[message.Name] = message
	go topic.Publish(message)
	go message.QueueExpiration(topic, ttl)
}

// Creates a subscription on the topic if it doesn't exist
// and returns it.
func (topic *Topic) GetSubscription(name string, ttl time.Duration) *Subscription {
	topic.subscriptionMutex.Lock()
	defer topic.subscriptionMutex.Unlock()

	if subscription, ok := topic.Subscriptions[name]; ok {
		select {
		case subscription.expireChan <- cancelTTL:
			// If we were able to cancel successfully,
			// just restart the expiration.
			go subscription.QueueExpiration(ttl)
			return subscription
		default:
			// We're too late
		}
	}

	subscription := NewSubscription(name, topic, ttl)
	topic.Subscriptions[name] = subscription
	return subscription
}
<|endoftext|>


<commit_before>package riak

type NodeManager interface {
	ExecuteOnNode(nodes []*Node, command Command) (executing bool, err error)
}

type defaultNodeManager struct {
}

func (nm *defaultNodeManager) ExecuteOnNode(nodes []*Node, command Command) (executing bool, err error) {
	return true, nil
}
<commit_msg>Implement defaultNodeManager<commit_after>package riak

type NodeManager interface {
	ExecuteOnNode(nodes []*Node, command Command, previous *Node) (executed bool, err error)
}

type defaultNodeManager struct {
	nodeIndex uint16
}

func (nm *defaultNodeManager) ExecuteOnNode(nodes []*Node, command Command, previous *Node) (executed bool, err error) {
	executed = false
	startingIndex := nm.nodeIndex

	for {
		node := nodes[nm.nodeIndex]

		nm.nodeIndex++

		if int(nm.nodeIndex) == len(nodes) {
			nm.nodeIndex = 0
		}

		// don't try the same node twice in a row if we have multiple nodes
		if len(nodes) > 1 && previous != nil && previous == node {
			continue
		}

		if executed, err = node.Execute(command); err != nil {
			executed = false
			logErr(err)
		} else {
			executed = true
			break
		}

		if nm.nodeIndex == startingIndex {
			break
		}
	}

	return
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Nodetemple <hostmaster@nodetemple.com>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"strings"

	"github.com/codegangsta/cli"
	"github.com/nodetemple/nodetemple/common"
	"github.com/nodetemple/nodetemple/nodectl/util"
	"github.com/nodetemple/nodetemple/nodectl/command"
)

func main() {
	app := cli.NewApp()
	app.Name = "nodectl"
	app.Usage = "CLI for an orchestration of CoreOS and Kubernetes cluster"
	app.Version = common.Version
	app.Flags = []cli.Flag{
		cli.StringFlag{Name: "providers, p", Usage: "A comma-separated list of IaaS providers ("+strings.Join(common.AvailableProviders, ",")+") and API keys, format: 'provider:api-key,...'", EnvVar: util.EnvVarConv(app.Name, "providers"),},
		cli.BoolFlag{Name: "debug", Usage: "Print out more debug information to stderr"},
	}
	app.Before = func(c *cli.Context) {
		if c.String("providers") == "" && !c.Bool("help") && !c.Bool("version") {
			util.Err("set at least one provider with a valid API key")
		}
	}
	app.Commands = []cli.Command{
		command.DemoCmd,
	}
	app.CommandNotFound = func(c *cli.Context, command string) {
		util.Err("unknown command '%v'\nRun '%v help [command]' for usage information", command, c.App.Name)
	}

	app.RunAndExitOnError()
}
<commit_msg>App before run to func<commit_after>/*
Copyright 2015 Nodetemple <hostmaster@nodetemple.com>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"strings"

	"github.com/codegangsta/cli"
	"github.com/nodetemple/nodetemple/common"
	"github.com/nodetemple/nodetemple/nodectl/util"
	"github.com/nodetemple/nodetemple/nodectl/command"
)

func main() {
	app := cli.NewApp()
	app.Name = "nodectl"
	app.Usage = "CLI for an orchestration of CoreOS and Kubernetes cluster"
	app.Version = common.Version
	app.Flags = []cli.Flag{
		cli.StringFlag{Name: "providers, p", Usage: "A comma-separated list of IaaS providers ("+strings.Join(common.AvailableProviders, ",")+") and API keys, format: 'provider:api-key,...'", EnvVar: util.EnvVarConv(app.Name, "providers"),},
		cli.BoolFlag{Name: "debug", Usage: "Print out more debug information to stderr"},
	}
	app.Before = appBefore
	app.Commands = []cli.Command{
		command.DemoCmd,
	}
	app.CommandNotFound = func(c *cli.Context, command string) {
		util.Err("unknown command '%v'\nRun '%v help [command]' for usage information", command, c.App.Name)
	}

	app.RunAndExitOnError()
}

func appBefore(c *cli.Context) {
	if c.String("providers") == "" && !c.Bool("help") && !c.Bool("version") {
		util.Err("set at least one provider with a valid API key")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Nodetemple <hostmaster@nodetemple.com>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"strings"
)

/* TODO
func stdout(format string, a ...interface{}) {
	out := fmt.Sprintf(format, a...)
	fmt.Fprintln(os.Stdout, strings.TrimSuffix(out, "\n"))
}

func stderr(format string, a ...interface{}) {
	out := fmt.Sprintf("Error: " + format, a...)
	fmt.Fprintln(os.Stderr, strings.TrimSuffix(out, "\n"))
}
*/

func stringSliceContains(slice []string, item string) bool {
	set := make(map[string]struct{}, len(slice))
    for _, s := range slice {
        set[s] = struct{}{}
    }

    _, ret := set[item]
    return ret
}
<commit_msg>Imported and not used<commit_after>/*
Copyright 2015 Nodetemple <hostmaster@nodetemple.com>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

/* TODO
func stdout(format string, a ...interface{}) {
	out := fmt.Sprintf(format, a...)
	fmt.Fprintln(os.Stdout, strings.TrimSuffix(out, "\n"))
}

func stderr(format string, a ...interface{}) {
	out := fmt.Sprintf("Error: " + format, a...)
	fmt.Fprintln(os.Stderr, strings.TrimSuffix(out, "\n"))
}
*/

func stringSliceContains(slice []string, item string) bool {
	set := make(map[string]struct{}, len(slice))
    for _, s := range slice {
        set[s] = struct{}{}
    }

    _, ret := set[item]
    return ret
}
<|endoftext|>


<commit_before>package client

import (
	"fmt"

	"github.com/sourcegraph/vcsstore/vcsclient"

	"sourcegraph.com/sourcegraph/api_router"
)

// RepositoryTreeService communicates with the Sourcegraph API endpoints that
// fetch file and directory entries in repositories.
type RepositoryTreeService interface {
	Get(entry TreeEntrySpec, opt *RepositoryTreeGetOptions) (*TreeEntry, Response, error)
}

type repositoryTreeService struct {
	client *Client
}

var _ RepositoryTreeService = &repositoryTreeService{}

type TreeEntrySpec struct {
	Repo RepositorySpec
	Path string
}

func (s *TreeEntrySpec) RouteVars() map[string]string {
	return map[string]string{"RepoURI": s.Repo.URI, "Rev": s.Repo.CommitID, "Path": s.Path}
}

func (s TreeEntrySpec) String() string {
	return fmt.Sprintf("%s: %s (rev %q)", s.Repo.URI, s.Path, s.Repo.CommitID)
}

// TreeEntry is a file or directory in a repository, with additional feedback
// from the formatting operation (if Formatted is true in the options).
type TreeEntry struct {
	*vcsclient.TreeEntry

	// FormatResult is only set if this TreeEntry is a file.
	FormatResult *FormatResult `json:",omitempty"`

	// EntryDefinitions is a list of defined symbols for each entry in this
	// directory. It is only populated if DirEntryDefinitions is true.
	EntryDefinitions map[string]interface{}
}

// FormatResult contains information about and warnings from the formatting
// operation (if Formatted is true in the options).
type FormatResult struct {
	// TooManyRefs indicates that the file being formatted exceeded the maximum
	// number of refs that are linked. Only the first NumRefs refs are linked.
	TooManyRefs bool `json:",omitempty"`

	// NumRefs is the number of refs that were linked in this file. If the total
	// number of refs in the file exceeds the (server-defined) limit, NumRefs is
	// capped at the limit.
	NumRefs int
}

// RepositoryTreeGetOptions specifies options for (RepositoryTreeService).Get.
type RepositoryTreeGetOptions struct {
	// Formatted is whether the specified entry, if it's a file, should have its
	// contents code-formatted.
	Formatted bool

	// DirEntryDefinitions is whether the specified entry, if it's a directory,
	// should include a list of defined symbols for each of its entries (in
	// EntryDefinitions). For example, if the specified entry has a file "a" and
	// a dir "b/", the result would include a list of symbols defined in "a" and
	// in any file underneath "b/". Not all symbols defined in the entries are
	// returned; only the top few are.
	DirEntryDefinitions bool `url:",omitempty"`
}

func (s *repositoryTreeService) Get(entry TreeEntrySpec, opt *RepositoryTreeGetOptions) (*TreeEntry, Response, error) {
	url, err := s.client.url(api_router.RepositoryTreeEntry, entry.RouteVars(), opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, nil, err
	}

	var entry_ *TreeEntry
	resp, err := s.client.Do(req, &entry_)
	if err != nil {
		return nil, resp, err
	}

	return entry_, resp, nil
}

type MockRepositoryTreeService struct {
	Get_ func(entry TreeEntrySpec, opt *RepositoryTreeGetOptions) (*TreeEntry, Response, error)
}

var _ RepositoryTreeService = MockRepositoryTreeService{}

func (s MockRepositoryTreeService) Get(entry TreeEntrySpec, opt *RepositoryTreeGetOptions) (*TreeEntry, Response, error) {
	if s.Get_ == nil {
		return nil, &HTTPResponse{}, nil
	}
	return s.Get_(entry, opt)
}
<commit_msg>add contents-as-string option to file annotation api<commit_after>package client

import (
	"fmt"

	"github.com/sourcegraph/vcsstore/vcsclient"

	"sourcegraph.com/sourcegraph/api_router"
)

// RepositoryTreeService communicates with the Sourcegraph API endpoints that
// fetch file and directory entries in repositories.
type RepositoryTreeService interface {
	Get(entry TreeEntrySpec, opt *RepositoryTreeGetOptions) (*TreeEntry, Response, error)
}

type repositoryTreeService struct {
	client *Client
}

var _ RepositoryTreeService = &repositoryTreeService{}

type TreeEntrySpec struct {
	Repo RepositorySpec
	Path string
}

func (s *TreeEntrySpec) RouteVars() map[string]string {
	return map[string]string{"RepoURI": s.Repo.URI, "Rev": s.Repo.CommitID, "Path": s.Path}
}

func (s TreeEntrySpec) String() string {
	return fmt.Sprintf("%s: %s (rev %q)", s.Repo.URI, s.Path, s.Repo.CommitID)
}

// TreeEntry is a file or directory in a repository, with additional feedback
// from the formatting operation (if Formatted is true in the options).
type TreeEntry struct {
	*vcsclient.TreeEntry

	ContentsString string

	// FormatResult is only set if this TreeEntry is a file.
	FormatResult *FormatResult `json:",omitempty"`

	// EntryDefinitions is a list of defined symbols for each entry in this
	// directory. It is only populated if DirEntryDefinitions is true.
	EntryDefinitions map[string]interface{}
}

// FormatResult contains information about and warnings from the formatting
// operation (if Formatted is true in the options).
type FormatResult struct {
	// TooManyRefs indicates that the file being formatted exceeded the maximum
	// number of refs that are linked. Only the first NumRefs refs are linked.
	TooManyRefs bool `json:",omitempty"`

	// NumRefs is the number of refs that were linked in this file. If the total
	// number of refs in the file exceeds the (server-defined) limit, NumRefs is
	// capped at the limit.
	NumRefs int
}

// RepositoryTreeGetOptions specifies options for (RepositoryTreeService).Get.
type RepositoryTreeGetOptions struct {
	// Formatted is whether the specified entry, if it's a file, should have its
	// contents code-formatted.
	Formatted bool

	// DirEntryDefinitions is whether the specified entry, if it's a directory,
	// should include a list of defined symbols for each of its entries (in
	// EntryDefinitions). For example, if the specified entry has a file "a" and
	// a dir "b/", the result would include a list of symbols defined in "a" and
	// in any file underneath "b/". Not all symbols defined in the entries are
	// returned; only the top few are.
	DirEntryDefinitions bool `url:",omitempty"`

	ContentsAsString bool `url:",omitempty"`
}

func (s *repositoryTreeService) Get(entry TreeEntrySpec, opt *RepositoryTreeGetOptions) (*TreeEntry, Response, error) {
	url, err := s.client.url(api_router.RepositoryTreeEntry, entry.RouteVars(), opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, nil, err
	}

	var entry_ *TreeEntry
	resp, err := s.client.Do(req, &entry_)
	if err != nil {
		return nil, resp, err
	}

	return entry_, resp, nil
}

type MockRepositoryTreeService struct {
	Get_ func(entry TreeEntrySpec, opt *RepositoryTreeGetOptions) (*TreeEntry, Response, error)
}

var _ RepositoryTreeService = MockRepositoryTreeService{}

func (s MockRepositoryTreeService) Get(entry TreeEntrySpec, opt *RepositoryTreeGetOptions) (*TreeEntry, Response, error) {
	if s.Get_ == nil {
		return nil, &HTTPResponse{}, nil
	}
	return s.Get_(entry, opt)
}
<|endoftext|>


<commit_before>package updater

import (
	"bytes"
	"compress/gzip"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/inconshreveable/go-update"
	"github.com/kr/binarydist"

	"github.com/kardianos/osext"
)

// Update protocol:
//
//   GET hk.heroku.com/hk/linux-amd64.json
//
//   200 ok
//   {
//       "Version": "2",
//       "Sha256": "..." // base64
//   }
//
// then
//
//   GET hkpatch.s3.amazonaws.com/hk/1/2/linux-amd64
//
//   200 ok
//   [bsdiff data]
//
// or
//
//   GET hkdist.s3.amazonaws.com/hk/2/linux-amd64.gz
//
//   200 ok
//   [gzipped executable data]
//
//

const (
	plat = runtime.GOOS + "-" + runtime.GOARCH
)

const devValidTime = 7 * 24 * time.Hour

var errHashMismatch = errors.New("new file hash mismatch after patch")
var up = update.New()

// Updater is the configuration and runtime data for doing an update.
//
// Note that ApiURL, BinURL and DiffURL should have the same value if all files are available at the same location.
//
// Example:
//
//  updater := &selfupdate.Updater{
//  	CurrentVersion: version,
//  	ApiURL:         "http://updates.yourdomain.com/",
//  	BinURL:         "http://updates.yourdownmain.com/",
//  	DiffURL:        "http://updates.yourdomain.com/",
//  	Dir:            "update/",
//  	CmdName:        "myapp", // app name
//  }
//  if updater != nil {
//  	go updater.BackgroundRun()
//  }
type Updater struct {
	CurrentVersion string // Currently running version.
	APIURL         string // Base URL for API requests (json files).
	CmdName        string // Command name is appended to the ApiURL like http://apiurl/CmdName/. This represents one binary.
	BinURL         string // Base URL for full binary downloads.
	DiffURL        string // Base URL for diff downloads.
	Dir            string // Directory to store selfupdate state.
	Info           struct {
		Version string
		Sha256  []byte
	}
}

// BackgroundRun starts the update check and apply cycle.
func (u *Updater) BackgroundRun() error {
	os.MkdirAll(u.getExecRelativeDir(u.Dir), 0777)
	if u.wantUpdate() {
		if err := up.CanUpdate(); err != nil {
			log.Println(err)
			return err
		}
		//self, err := osext.Executable()
		//if err != nil {
		// fail update, couldn't figure out path to self
		//return
		//}
		// TODO(bgentry): logger isn't on Windows. Replace w/ proper error reports.
		if err := u.update(); err != nil {
			return err
		}
	}
	return errors.New("Won't update because it's a development daemon. Change the version in main.go")
}

func fetch(url string) (io.ReadCloser, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		log.Errorf("bad http status from %s: %v", url, resp.Status)
		return nil, fmt.Errorf("bad http status from %s: %v", url, resp.Status)
	}
	return resp.Body, nil
}

func verifySha(bin []byte, sha []byte) bool {
	h := sha256.New()
	h.Write(bin)
	return bytes.Equal(h.Sum(nil), sha)
}

func (u *Updater) fetchAndApplyPatch(old io.Reader) ([]byte, error) {
	r, err := fetch(u.DiffURL + u.CmdName + "/" + u.CurrentVersion + "/" + u.Info.Version + "/" + plat)
	if err != nil {
		return nil, err
	}
	defer r.Close()
	var buf bytes.Buffer
	err = binarydist.Patch(old, &buf, r)
	return buf.Bytes(), err
}

func (u *Updater) fetchAndVerifyPatch(old io.Reader) ([]byte, error) {
	bin, err := u.fetchAndApplyPatch(old)
	if err != nil {
		return nil, err
	}
	if !verifySha(bin, u.Info.Sha256) {
		return nil, errHashMismatch
	}
	return bin, nil
}

func (u *Updater) fetchAndVerifyFullBin() ([]byte, error) {
	bin, err := u.fetchBin()
	if err != nil {
		return nil, err
	}
	verified := verifySha(bin, u.Info.Sha256)
	if !verified {
		return nil, errHashMismatch
	}
	return bin, nil
}

func (u *Updater) fetchBin() ([]byte, error) {
	r, err := fetch(u.BinURL + u.CmdName + "/" + u.Info.Version + "/" + plat + ".gz")
	if err != nil {
		return nil, err
	}
	defer r.Close()
	buf := new(bytes.Buffer)
	gz, err := gzip.NewReader(r)
	if err != nil {
		return nil, err
	}
	if _, err = io.Copy(buf, gz); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

func (u *Updater) fetchInfo() error {
	r, err := fetch(u.APIURL + u.CmdName + "/" + plat + ".json")
	if err != nil {
		return err
	}
	defer r.Close()
	err = json.NewDecoder(r).Decode(&u.Info)
	if err != nil {
		return err
	}
	if len(u.Info.Sha256) != sha256.Size {
		return errors.New("bad cmd hash in info")
	}
	return nil
}

func (u *Updater) getExecRelativeDir(dir string) string {
	filename, _ := osext.Executable()
	path := filepath.Join(filepath.Dir(filename), dir)
	return path
}

func (u *Updater) update() error {
	path, err := osext.Executable()
	if err != nil {
		return err
	}
	old, err := os.Open(path)
	if err != nil {
		return err
	}
	defer old.Close()

	err = u.fetchInfo()
	if err != nil {
		log.Println(err)
		return err
	}
	if u.Info.Version == u.CurrentVersion {
		return nil
	}
	bin, err := u.fetchAndVerifyPatch(old)
	if err != nil {
		if err == errHashMismatch {
			log.Println("update: hash mismatch from patched binary")
		} else {
			if u.DiffURL != "" {
				log.Println("update: patching binary,", err)
			}
		}

		bin, err = u.fetchAndVerifyFullBin()
		if err != nil {
			if err == errHashMismatch {
				log.Println("update: hash mismatch from full binary")
			} else {
				log.Println("update: fetching full binary,", err)
			}
			return err
		}
	}

	// close the old binary before installing because on windows
	// it can't be renamed if a handle to the file is still open
	old.Close()

	err, errRecover := up.FromStream(bytes.NewBuffer(bin))
	if errRecover != nil {
		log.Errorf("update and recovery errors: %q %q", err, errRecover)
		return fmt.Errorf("update and recovery errors: %q %q", err, errRecover)
	}
	if err != nil {
		return err
	}

	return nil
}

func (u *Updater) wantUpdate() bool {
	// if strings.Contains(u.CurrentVersion, "dev") {
	// 	return false
	// }
	return true
}
<commit_msg>Remove useless function<commit_after>package updater

import (
	"bytes"
	"compress/gzip"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/inconshreveable/go-update"
	"github.com/kr/binarydist"

	"github.com/kardianos/osext"
)

// Update protocol:
//
//   GET hk.heroku.com/hk/linux-amd64.json
//
//   200 ok
//   {
//       "Version": "2",
//       "Sha256": "..." // base64
//   }
//
// then
//
//   GET hkpatch.s3.amazonaws.com/hk/1/2/linux-amd64
//
//   200 ok
//   [bsdiff data]
//
// or
//
//   GET hkdist.s3.amazonaws.com/hk/2/linux-amd64.gz
//
//   200 ok
//   [gzipped executable data]
//
//

const (
	plat = runtime.GOOS + "-" + runtime.GOARCH
)

const devValidTime = 7 * 24 * time.Hour

var errHashMismatch = errors.New("new file hash mismatch after patch")
var up = update.New()

// Updater is the configuration and runtime data for doing an update.
//
// Note that ApiURL, BinURL and DiffURL should have the same value if all files are available at the same location.
//
// Example:
//
//  updater := &selfupdate.Updater{
//  	CurrentVersion: version,
//  	ApiURL:         "http://updates.yourdomain.com/",
//  	BinURL:         "http://updates.yourdownmain.com/",
//  	DiffURL:        "http://updates.yourdomain.com/",
//  	Dir:            "update/",
//  	CmdName:        "myapp", // app name
//  }
//  if updater != nil {
//  	go updater.BackgroundRun()
//  }
type Updater struct {
	CurrentVersion string // Currently running version.
	APIURL         string // Base URL for API requests (json files).
	CmdName        string // Command name is appended to the ApiURL like http://apiurl/CmdName/. This represents one binary.
	BinURL         string // Base URL for full binary downloads.
	DiffURL        string // Base URL for diff downloads.
	Dir            string // Directory to store selfupdate state.
	Info           struct {
		Version string
		Sha256  []byte
	}
}

// BackgroundRun starts the update check and apply cycle.
func (u *Updater) BackgroundRun() error {
	os.MkdirAll(u.getExecRelativeDir(u.Dir), 0777)
	if err := up.CanUpdate(); err != nil {
		log.Println(err)
		return err
	}
	//self, err := osext.Executable()
	//if err != nil {
	// fail update, couldn't figure out path to self
	//return
	//}
	// TODO(bgentry): logger isn't on Windows. Replace w/ proper error reports.
	if err := u.update(); err != nil {
		return err
	}
	return nil
}

func fetch(url string) (io.ReadCloser, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		log.Errorf("bad http status from %s: %v", url, resp.Status)
		return nil, fmt.Errorf("bad http status from %s: %v", url, resp.Status)
	}
	return resp.Body, nil
}

func verifySha(bin []byte, sha []byte) bool {
	h := sha256.New()
	h.Write(bin)
	return bytes.Equal(h.Sum(nil), sha)
}

func (u *Updater) fetchAndApplyPatch(old io.Reader) ([]byte, error) {
	r, err := fetch(u.DiffURL + u.CmdName + "/" + u.CurrentVersion + "/" + u.Info.Version + "/" + plat)
	if err != nil {
		return nil, err
	}
	defer r.Close()
	var buf bytes.Buffer
	err = binarydist.Patch(old, &buf, r)
	return buf.Bytes(), err
}

func (u *Updater) fetchAndVerifyPatch(old io.Reader) ([]byte, error) {
	bin, err := u.fetchAndApplyPatch(old)
	if err != nil {
		return nil, err
	}
	if !verifySha(bin, u.Info.Sha256) {
		return nil, errHashMismatch
	}
	return bin, nil
}

func (u *Updater) fetchAndVerifyFullBin() ([]byte, error) {
	bin, err := u.fetchBin()
	if err != nil {
		return nil, err
	}
	verified := verifySha(bin, u.Info.Sha256)
	if !verified {
		return nil, errHashMismatch
	}
	return bin, nil
}

func (u *Updater) fetchBin() ([]byte, error) {
	r, err := fetch(u.BinURL + u.CmdName + "/" + u.Info.Version + "/" + plat + ".gz")
	if err != nil {
		return nil, err
	}
	defer r.Close()
	buf := new(bytes.Buffer)
	gz, err := gzip.NewReader(r)
	if err != nil {
		return nil, err
	}
	if _, err = io.Copy(buf, gz); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

func (u *Updater) fetchInfo() error {
	r, err := fetch(u.APIURL + u.CmdName + "/" + plat + ".json")
	if err != nil {
		return err
	}
	defer r.Close()
	err = json.NewDecoder(r).Decode(&u.Info)
	if err != nil {
		return err
	}
	if len(u.Info.Sha256) != sha256.Size {
		return errors.New("bad cmd hash in info")
	}
	return nil
}

func (u *Updater) getExecRelativeDir(dir string) string {
	filename, _ := osext.Executable()
	path := filepath.Join(filepath.Dir(filename), dir)
	return path
}

func (u *Updater) update() error {
	path, err := osext.Executable()
	if err != nil {
		return err
	}
	old, err := os.Open(path)
	if err != nil {
		return err
	}
	defer old.Close()

	err = u.fetchInfo()
	if err != nil {
		log.Println(err)
		return err
	}
	if u.Info.Version == u.CurrentVersion {
		return nil
	}
	bin, err := u.fetchAndVerifyPatch(old)
	if err != nil {
		if err == errHashMismatch {
			log.Println("update: hash mismatch from patched binary")
		} else {
			if u.DiffURL != "" {
				log.Println("update: patching binary,", err)
			}
		}

		bin, err = u.fetchAndVerifyFullBin()
		if err != nil {
			if err == errHashMismatch {
				log.Println("update: hash mismatch from full binary")
			} else {
				log.Println("update: fetching full binary,", err)
			}
			return err
		}
	}

	// close the old binary before installing because on windows
	// it can't be renamed if a handle to the file is still open
	old.Close()

	err, errRecover := up.FromStream(bytes.NewBuffer(bin))
	if errRecover != nil {
		log.Errorf("update and recovery errors: %q %q", err, errRecover)
		return fmt.Errorf("update and recovery errors: %q %q", err, errRecover)
	}
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package uploader

import (
	"bufio"
	"fmt"
	"io"
	"time"

	"go.uber.org/zap"
)

type cached struct {
	*Base
	existsCache CMap // store known keys and don't load it to clickhouse tree
	parser      func(filename string, out io.Writer) (map[string]bool, error)
	insertQuery string
}

func newCached(base *Base) *cached {
	u := &cached{Base: base}
	u.Base.handler = u.upload
	u.existsCache = NewCMap()
	u.insertQuery = fmt.Sprintf("%s (Date, Level, Path, Version)", u.config.TableName)
	return u
}

func (u *cached) Stat(send func(metric string, value float64)) {
	u.Base.Stat(send)

	send("cacheSize", float64(u.existsCache.Count()))
}

func (u *cached) Start() error {
	err := u.Base.Start()
	if err != nil {
		return err
	}

	if u.config.CacheTTL.Value() != 0 {
		u.Go(func(exit chan struct{}) {
			u.existsCache.ExpireWorker(exit, u.config.CacheTTL.Value())
		})
	}

	return nil
}

func (u *cached) Reset() {
	u.existsCache.Clear()
}

func (u *cached) upload(exit chan struct{}, logger *zap.Logger, filename string) error {
	pipeReader, pipeWriter := io.Pipe()
	writer := bufio.NewWriter(pipeWriter)
	startTime := time.Now()

	uploadResult := make(chan error, 1)

	u.Go(func(exit chan struct{}) {
		uploadResult <- u.insertRowBinary(
			u.insertQuery,
			pipeReader,
		)
	})

	newSeries, err := u.parser(filename, writer)
	if err == nil {
		err = writer.Flush()
	}
	pipeWriter.CloseWithError(err)

	var uploadErr error

	select {
	case uploadErr = <-uploadResult:
		// pass
	case <-exit:
		return fmt.Errorf("upload aborted")
	}

	if err != nil {
		return err
	}

	if uploadErr != nil {
		return uploadErr
	}

	// commit new series
	u.existsCache.Merge(newSeries, startTime.Unix())

	return nil
}
<commit_msg>bugfix: series table stops uploading into clickhouse after error<commit_after>package uploader

import (
	"bufio"
	"fmt"
	"io"
	"time"

	"go.uber.org/zap"
)

type cached struct {
	*Base
	existsCache CMap // store known keys and don't load it to clickhouse tree
	parser      func(filename string, out io.Writer) (map[string]bool, error)
	insertQuery string
}

func newCached(base *Base) *cached {
	u := &cached{Base: base}
	u.Base.handler = u.upload
	u.existsCache = NewCMap()
	u.insertQuery = fmt.Sprintf("%s (Date, Level, Path, Version)", u.config.TableName)
	return u
}

func (u *cached) Stat(send func(metric string, value float64)) {
	u.Base.Stat(send)

	send("cacheSize", float64(u.existsCache.Count()))
}

func (u *cached) Start() error {
	err := u.Base.Start()
	if err != nil {
		return err
	}

	if u.config.CacheTTL.Value() != 0 {
		u.Go(func(exit chan struct{}) {
			u.existsCache.ExpireWorker(exit, u.config.CacheTTL.Value())
		})
	}

	return nil
}

func (u *cached) Reset() {
	u.existsCache.Clear()
}

func (u *cached) upload(exit chan struct{}, logger *zap.Logger, filename string) error {
	pipeReader, pipeWriter := io.Pipe()
	writer := bufio.NewWriter(pipeWriter)
	startTime := time.Now()

	uploadResult := make(chan error, 1)

	u.Go(func(exit chan struct{}) {
		err := u.insertRowBinary(
			u.insertQuery,
			pipeReader,
		)
		uploadResult <- err
		if err != nil {
			pipeReader.CloseWithError(err)
		}
	})

	newSeries, err := u.parser(filename, writer)
	if err == nil {
		err = writer.Flush()
	}
	pipeWriter.CloseWithError(err)

	var uploadErr error

	select {
	case uploadErr = <-uploadResult:
		// pass
	case <-exit:
		return fmt.Errorf("upload aborted")
	}

	if err != nil {
		return err
	}

	if uploadErr != nil {
		return uploadErr
	}

	// commit new series
	u.existsCache.Merge(newSeries, startTime.Unix())

	return nil
}
<|endoftext|>


<commit_before>package gaeutil

import (
	"fmt"
	"golang.org/x/net/context"
	"google.golang.org/appengine/memcache"
)

const Chunksize = 950000  // A single memcache item can't be bigger than 1000000 bytes
const MaxChunks = 32      // We only shard this much.

func singletonMCKey(name string) string { return "singleton:"+name }

func DeleteSingletonFromMemcache(c context.Context, name string) (error) {
	return memcache.Delete(c, name)
}

func LoadSingletonFromMemcache(c context.Context, name string) ([]byte, error) {
	item,err := memcache.Get(c, singletonMCKey(name))
	if err != nil {
    return nil, err  // might be memcache.ErrCacheMiss
	}
	return item.Value, nil
}

func SaveSingletonToMemcache(c context.Context, name string, data []byte) error {
	if len(data) > 950000 {
		return fmt.Errorf("singleton too large (name=%s, size=%d)", name, len(data))
	}
	item := &memcache.Item{Key:singletonMCKey(name), Value:data}
	return memcache.Set(c, item)
}

func LoadShardedSingletonFromMemcache(c context.Context, name string) ([]byte, error) {
	return bytesFromMemcacheShards(c, name)
}
func SaveShardedSingletonToMemcache(c context.Context, name string, data []byte) error {
	return bytesToMemcacheShards(c, name, data)
}

func bytesToMemcacheShards(c context.Context, key string, b []byte) error {
	if len(b) > MaxChunks * Chunksize {
		return fmt.Errorf("obj '%s' was too large; %d > %d", key, len(b), MaxChunks * Chunksize)
	}

	items := []*memcache.Item{}
	for i:=0; i<len(b); i+=Chunksize {
		k := fmt.Sprintf("=%d=%s",i,key)
		s,e := i, i+Chunksize-1
		if e>=len(b) { e = len(b)-1 }
		items = append(items, &memcache.Item{ Key:k , Value:b[s:e+1] }) // slice sytax is [s,e)
	}

	return memcache.SetMulti(c, items)
}

// err might be memcache.ErrCacheMiss
func bytesFromMemcacheShards(c context.Context, key string) ([]byte, error) {
	keys := []string{}
	for i:=0; i<32; i++ { keys = append(keys, fmt.Sprintf("=%d=%s",i*Chunksize,key)) }

	if items,err := memcache.GetMulti(c, keys); err != nil {
		return nil, fmt.Errorf("MCShards/GetMulti/'%s' err: %v\n", key, err)

	} else {
		b := []byte{}
		for i:=0; i<32; i++ {
			if item,exists := items[keys[i]]; exists==false {
				break
			} else {
				b = append(b, item.Value...)
			}
		}

		if len(b) > 0 {
			return b, nil
		} else {
			return nil, memcache.ErrCacheMiss
		}
	}
}

<commit_msg>Memcache API with a transparent TTL.<commit_after>package gaeutil

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"reflect"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/appengine/memcache"
	"google.golang.org/appengine/log"
)

const Chunksize = 950000  // A single memcache item can't be bigger than 1000000 bytes
const MaxChunks = 32      // We only shard this much.

func singletonMCKey(name string) string { return "singleton:"+name }

func DeleteSingletonFromMemcache(c context.Context, name string) (error) {
	return memcache.Delete(c, name)
}

func LoadSingletonFromMemcache(c context.Context, name string) ([]byte, error) {
	item,err := memcache.Get(c, singletonMCKey(name))
	if err != nil {
    return nil, err  // might be memcache.ErrCacheMiss
	}
	return item.Value, nil
}

func SaveSingletonToMemcache(c context.Context, name string, data []byte) error {
	if len(data) > 950000 {
		return fmt.Errorf("singleton too large (name=%s, size=%d)", name, len(data))
	}
	item := &memcache.Item{Key:singletonMCKey(name), Value:data}
	return memcache.Set(c, item)
}

func LoadShardedSingletonFromMemcache(c context.Context, name string) ([]byte, error) {
	return bytesFromMemcacheShards(c, name)
}
func SaveShardedSingletonToMemcache(c context.Context, name string, data []byte) error {
	return bytesToMemcacheShards(c, name, data)
}

func bytesToMemcacheShards(c context.Context, key string, b []byte) error {
	if len(b) > MaxChunks * Chunksize {
		return fmt.Errorf("obj '%s' was too large; %d > %d", key, len(b), MaxChunks * Chunksize)
	}

	items := []*memcache.Item{}
	for i:=0; i<len(b); i+=Chunksize {
		k := fmt.Sprintf("=%d=%s",i,key)
		s,e := i, i+Chunksize-1
		if e>=len(b) { e = len(b)-1 }
		items = append(items, &memcache.Item{ Key:k , Value:b[s:e+1] }) // slice sytax is [s,e)
	}

	return memcache.SetMulti(c, items)
}

// err might be memcache.ErrCacheMiss
func bytesFromMemcacheShards(c context.Context, key string) ([]byte, error) {
	keys := []string{}
	for i:=0; i<32; i++ { keys = append(keys, fmt.Sprintf("=%d=%s",i*Chunksize,key)) }

	if items,err := memcache.GetMulti(c, keys); err != nil {
		return nil, fmt.Errorf("MCShards/GetMulti/'%s' err: %v\n", key, err)

	} else {
		b := []byte{}
		for i:=0; i<32; i++ {
			if item,exists := items[keys[i]]; exists==false {
				break
			} else {
				b = append(b, item.Value...)
			}
		}

		if len(b) > 0 {
			return b, nil
		} else {
			return nil, memcache.ErrCacheMiss
		}
	}
}

/*

 object := SomeThing{}

 gaeutil.SaveToMemcacheShards(ctx, "mything", &object)

 if err := gaeutil.LoadFromMemcacheShards(ctx, "mything", &object); err == nil {
    // use object
 }

 */

func SaveToMemcacheShards(ctx context.Context, name string, ptr interface{}) error {
	var buf bytes.Buffer
	if err := gob.NewEncoder(&buf).Encode(ptr); err != nil {
		return err
	}
	// log.Infof(ctx, "SAVE<%T>, %d bytes", ptr, buf.Len())

	return bytesToMemcacheShards(ctx, name, buf.Bytes())
}

func LoadFromMemcacheShards(ctx context.Context, name string, ptr interface{}) error {
	myBytes,err := bytesFromMemcacheShards(ctx, name)
	if err != nil { return err }

	buf := bytes.NewBuffer(myBytes)
	if err := gob.NewDecoder(buf).Decode(ptr); err != nil {
		log.Errorf(ctx, "LoadFromMemcacheShards err %v", err)
		return err
	}

	//log.Infof(ctx, "LOAD<%T>, %d/%d bytes", ptr, len(myBytes), buf.Len())

	return nil
}

/* TTL versions; if memcache obj older than 'duration', returns memcache.ErrCacheMiss

 object := SomeThing{}
 duration := time.Minute

 err := gaeutil.SaveToMemcacheShardsTTL(ctx, "mything", &object, duration)
 err := gaeutil.LoadFromMemcacheShardsTTL(ctx, "mything", &object)

 */

// Note that the pointer gets flattened in this roundtrip through encoding/gob.
// When submitted to Save*, the interface is a pointer to an object; but when retrived via
// Load*, the interface contains the actual object originally pointed to.
// There is some reflection nonsense needed to copy results between interfaces.
// https://github.com/mohae/deepcopy/blob/master/deepcopy.go
type explodingObj struct {
	Obj interface{}
	Expires time.Time
}

func SaveToMemcacheShardsTTL(ctx context.Context, name string, ptr interface{}, d time.Duration) error {
	return SaveToMemcacheShards(ctx, name, explodingObj{ptr, time.Now().Add(d)})
}

func LoadFromMemcacheShardsTTL(ctx context.Context, name string, ptr interface{}) error {
	eObj := explodingObj{}
	if err := LoadFromMemcacheShards(ctx, name, &eObj); err != nil {
		return err
	} else if time.Now().After(eObj.Expires) {
		return memcache.ErrCacheMiss
	}

	// Assert that 'ptr' is indeed a pointer to something (and is not nil)
	if reflect.TypeOf(ptr).Kind() != reflect.Ptr {
		return fmt.Errorf("interface arg was '%s', expected pointer", reflect.TypeOf(ptr))

	}
	if reflect.ValueOf(ptr).IsNil() {
		return fmt.Errorf("interface arg was nil pointer")
	}

	// Assert that ptr points to the same type of thing that we have in the explodey obj
	if reflect.ValueOf(ptr).Elem().Type() != reflect.TypeOf(eObj.Obj) {
		return fmt.Errorf("type mismatch; asked to load '%s' into '%s'",
			reflect.TypeOf(eObj.Obj), reflect.ValueOf(ptr).Elem().Type())
	}

	// Now copy whatever is inside of eObj.Obj into whatever the pointer points to
	srcValue := reflect.ValueOf(eObj.Obj)
	dstValue := reflect.ValueOf(ptr).Elem() // Follow the pointer
	dstValue.Set(srcValue)

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Stratumn SAS. All rights reserved.
// Use of this source code is governed by an Apache License 2.0
// that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcutil"
	"github.com/stratumn/go/fossilizer/fossilizerhttp"
	"github.com/stratumn/go/jsonhttp"
	"github.com/stratumn/goprivate/batchfossilizer"
	"github.com/stratumn/goprivate/bcbatchfossilizer"
	"github.com/stratumn/goprivate/blockchain/btc"
	"github.com/stratumn/goprivate/blockchain/btc/blockcypher"
	"github.com/stratumn/goprivate/blockchain/btc/btctimestamper"
	"github.com/stratumn/goprivate/merkle"
)

var (
	port             = flag.String("port", fossilizerhttp.DefaultPort, "server port")
	certFile         = flag.String("tlscert", "", "TLS certificate file")
	keyFile          = flag.String("tlskey", "", "TLS private key file")
	numResultWorkers = flag.Int("workers", fossilizerhttp.DefaultNumResultWorkers, "number of result workers")
	callbackTimeout  = flag.Duration("callbacktimeout", fossilizerhttp.DefaultCallbackTimeout, "callback requests timeout")
	verbose          = flag.Bool("verbose", fossilizerhttp.DefaultVerbose, "verbose output")
	interval         = flag.Duration("interval", batchfossilizer.DefaultInterval, "batch interval")
	maxLeaves        = flag.Int("maxleaves", batchfossilizer.DefaultMaxLeaves, "maximum number of leaves in a Merkle tree")
	path             = flag.String("path", "", "an optional path to store files")
	archive          = flag.Bool("archive", batchfossilizer.DefaultArchive, "whether to archive completed batches (requires path)")
	exitBatch        = flag.Bool("exitbatch", batchfossilizer.DefaultStopBatch, "whether to do a batch on exit")
	fsync            = flag.Bool("fsync", batchfossilizer.DefaultFSync, "whether to fsync after saving a pending hash (requires path)")
	key              = flag.String("wif", "", "wallet import format key")
	fee              = flag.Int64("fee", btctimestamper.DefaultFee, "transaction fee (satoshis)")
	bcyAPIKey        = flag.String("bcyapikey", "", "BlockCypher API key")
	version          = "0.1.0"
	commit           = "00000000000000000000000000000000"
)

func main() {

	flag.Parse()

	if *key == "" {
		log.Fatal("Fatal: a WIF encoded private key is required")
	}

	WIF, err := btcutil.DecodeWIF(*key)
	if err != nil {
		log.Fatalf("Fatal: %s", err)
	}

	var network btc.Network
	if WIF.IsForNet(&chaincfg.TestNet3Params) {
		network = btc.NetworkTest3
	} else if WIF.IsForNet(&chaincfg.MainNetParams) {
		network = btc.NetworkMain
	} else {
		log.Fatal("Fatal: unknown Bitcoin network")
	}

	log.SetPrefix(fmt.Sprintf("btcfossilizer:%s ", network))

	log.Printf("%s v%s@%s", bcbatchfossilizer.Description, version, commit[:7])
	log.Print("Copyright (c) 2016 Stratumn SAS")
	log.Print("All Rights Reserved")
	log.Printf("Runtime %s %s %s", runtime.Version(), runtime.GOOS, runtime.GOARCH)

	bcy := blockcypher.New(network, *bcyAPIKey)
	ts, err := btctimestamper.New(&btctimestamper.Config{
		UnspentFinder: bcy,
		Broadcaster:   bcy,
		WIF:           *key,
		Fee:           *fee,
	})
	if err != nil {
		log.Fatalf("Fatal: %s", err)
	}

	a, err := bcbatchfossilizer.New(&bcbatchfossilizer.Config{
		HashTimestamper: ts,
	}, &batchfossilizer.Config{
		Version:   version,
		Commit:    commit,
		Interval:  *interval,
		MaxLeaves: *maxLeaves,
		Path:      *path,
		Archive:   *archive,
		StopBatch: *exitBatch,
		FSync:     *fsync,
	})
	if err != nil {
		log.Fatalf("Fatal: %s", err)
	}

	go func() {
		if err := a.Start(); err != nil {
			log.Fatalf("Fatal: %s", err)
		}
	}()

	go func() {
		sigc := make(chan os.Signal)
		signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM)
		sig := <-sigc
		log.Printf("Got signal %q", sig)
		log.Print("Cleaning up")
		if err := a.Stop(); err != nil {
			log.Printf("Error: %s", err)
			os.Exit(1)
		}
		log.Print("Stopped")
		os.Exit(0)
	}()

	config := &fossilizerhttp.Config{
		NumResultWorkers: *numResultWorkers,
		CallbackTimeout:  *callbackTimeout,
		MinDataLen:       merkle.HashByteSize * 2,
		MaxDataLen:       merkle.HashByteSize * 2,
	}
	httpConfig := &jsonhttp.Config{
		Port:     *port,
		CertFile: *certFile,
		KeyFile:  *keyFile,
		Verbose:  *verbose,
	}
	h := fossilizerhttp.New(a, config, httpConfig)

	log.Printf("Listening on %q", *port)
	if err := h.ListenAndServe(); err != nil {
		log.Fatalf("Fatal: %s", err)
	}
}
<commit_msg>cmd: Rate limit bcy<commit_after>// Copyright 2016 Stratumn SAS. All rights reserved.
// Use of this source code is governed by an Apache License 2.0
// that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcutil"
	"github.com/stratumn/go/fossilizer/fossilizerhttp"
	"github.com/stratumn/go/jsonhttp"
	"github.com/stratumn/goprivate/batchfossilizer"
	"github.com/stratumn/goprivate/bcbatchfossilizer"
	"github.com/stratumn/goprivate/blockchain/btc"
	"github.com/stratumn/goprivate/blockchain/btc/blockcypher"
	"github.com/stratumn/goprivate/blockchain/btc/btctimestamper"
	"github.com/stratumn/goprivate/merkle"
)

var (
	port             = flag.String("port", fossilizerhttp.DefaultPort, "server port")
	certFile         = flag.String("tlscert", "", "TLS certificate file")
	keyFile          = flag.String("tlskey", "", "TLS private key file")
	numResultWorkers = flag.Int("workers", fossilizerhttp.DefaultNumResultWorkers, "number of result workers")
	callbackTimeout  = flag.Duration("callbacktimeout", fossilizerhttp.DefaultCallbackTimeout, "callback requests timeout")
	verbose          = flag.Bool("verbose", fossilizerhttp.DefaultVerbose, "verbose output")
	interval         = flag.Duration("interval", batchfossilizer.DefaultInterval, "batch interval")
	maxLeaves        = flag.Int("maxleaves", batchfossilizer.DefaultMaxLeaves, "maximum number of leaves in a Merkle tree")
	path             = flag.String("path", "", "an optional path to store files")
	archive          = flag.Bool("archive", batchfossilizer.DefaultArchive, "whether to archive completed batches (requires path)")
	exitBatch        = flag.Bool("exitbatch", batchfossilizer.DefaultStopBatch, "whether to do a batch on exit")
	fsync            = flag.Bool("fsync", batchfossilizer.DefaultFSync, "whether to fsync after saving a pending hash (requires path)")
	key              = flag.String("wif", "", "wallet import format key")
	fee              = flag.Int64("fee", btctimestamper.DefaultFee, "transaction fee (satoshis)")
	bcyAPIKey        = flag.String("bcyapikey", "", "BlockCypher API key")
	limiterInterval  = flag.Duration("limiterinterval", blockcypher.DefaultLimiterInterval, "BlockCypher API limiter interval")
	limiterSize      = flag.Int("limitersize", blockcypher.DefaultLimiterSize, "BlockCypher API limiter size")
	version          = "0.1.0"
	commit           = "00000000000000000000000000000000"
)

func main() {

	flag.Parse()

	if *key == "" {
		log.Fatal("Fatal: a WIF encoded private key is required")
	}

	WIF, err := btcutil.DecodeWIF(*key)
	if err != nil {
		log.Fatalf("Fatal: %s", err)
	}

	var network btc.Network
	if WIF.IsForNet(&chaincfg.TestNet3Params) {
		network = btc.NetworkTest3
	} else if WIF.IsForNet(&chaincfg.MainNetParams) {
		network = btc.NetworkMain
	} else {
		log.Fatal("Fatal: unknown Bitcoin network")
	}

	log.SetPrefix(fmt.Sprintf("btcfossilizer:%s ", network))

	log.Printf("%s v%s@%s", bcbatchfossilizer.Description, version, commit[:7])
	log.Print("Copyright (c) 2016 Stratumn SAS")
	log.Print("All Rights Reserved")
	log.Printf("Runtime %s %s %s", runtime.Version(), runtime.GOOS, runtime.GOARCH)

	bcy := blockcypher.New(&blockcypher.Config{
		Network:         network,
		APIKey:          *bcyAPIKey,
		LimiterInterval: *limiterInterval,
		LimiterSize:     *limiterSize,
	})
	ts, err := btctimestamper.New(&btctimestamper.Config{
		UnspentFinder: bcy,
		Broadcaster:   bcy,
		WIF:           *key,
		Fee:           *fee,
	})
	if err != nil {
		log.Fatalf("Fatal: %s", err)
	}

	a, err := bcbatchfossilizer.New(&bcbatchfossilizer.Config{
		HashTimestamper: ts,
	}, &batchfossilizer.Config{
		Version:   version,
		Commit:    commit,
		Interval:  *interval,
		MaxLeaves: *maxLeaves,
		Path:      *path,
		Archive:   *archive,
		StopBatch: *exitBatch,
		FSync:     *fsync,
	})
	if err != nil {
		log.Fatalf("Fatal: %s", err)
	}

	go func() {
		if err := a.Start(); err != nil {
			log.Fatalf("Fatal: %s", err)
		}
	}()

	go bcy.Start()

	go func() {
		sigc := make(chan os.Signal)
		signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM)
		sig := <-sigc
		log.Printf("Got signal %q", sig)
		log.Print("Cleaning up")
		if err := a.Stop(); err != nil {
			bcy.Stop()
			log.Printf("Error: %s", err)
			os.Exit(1)
		}
		bcy.Stop()
		log.Print("Stopped")
		os.Exit(0)
	}()

	config := &fossilizerhttp.Config{
		NumResultWorkers: *numResultWorkers,
		CallbackTimeout:  *callbackTimeout,
		MinDataLen:       merkle.HashByteSize * 2,
		MaxDataLen:       merkle.HashByteSize * 2,
	}
	httpConfig := &jsonhttp.Config{
		Port:     *port,
		CertFile: *certFile,
		KeyFile:  *keyFile,
		Verbose:  *verbose,
	}
	h := fossilizerhttp.New(a, config, httpConfig)

	log.Printf("Listening on %q", *port)
	if err := h.ListenAndServe(); err != nil {
		log.Fatalf("Fatal: %s", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"html/template"
	"io"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"sort"
	"strings"
	"sync/atomic"
	"time"
)

func handleStatus(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.NotFound(w, r)
		return
	}
	round := func(t time.Duration) time.Duration {
		return t / time.Second * time.Second
	}
	df := diskFree()

	statusMu.Lock()
	data := statusData{
		Total:        len(status),
		Uptime:       round(time.Now().Sub(processStartTime)),
		Recent:       append([]*buildStatus{}, statusDone...),
		DiskFree:     df,
		Version:      Version,
		NumFD:        fdCount(),
		NumGoroutine: runtime.NumGoroutine(),
	}
	for _, st := range status {
		if atomic.LoadInt32(&st.hasBuildlet) != 0 {
			data.ActiveBuilds++
			data.Active = append(data.Active, st)
		} else {
			data.Pending = append(data.Pending, st)
		}
	}
	// TODO: make this prettier.
	var buf bytes.Buffer
	for _, key := range tryList {
		if ts := tries[key]; ts != nil {
			state := ts.state()
			fmt.Fprintf(&buf, "Change-ID: %v Commit: %v (status)\n",
				key.ChangeTriple(), key.Commit, key.Commit[:8])
			fmt.Fprintf(&buf, "   Remain: %d, fails: %v\n", state.remain, state.failed)
			for _, bs := range ts.builds {
				fmt.Fprintf(&buf, "  %s: running=%v\n", bs.Name, bs.isRunning())
			}
		}
	}
	statusMu.Unlock()

	data.RemoteBuildlets = template.HTML(remoteBuildletStatus())

	sort.Sort(byAge(data.Active))
	sort.Sort(byAge(data.Pending))
	sort.Sort(sort.Reverse(byAge(data.Recent)))
	if errTryDeps != nil {
		data.TrybotsErr = errTryDeps.Error()
	} else {
		if buf.Len() == 0 {
			data.Trybots = template.HTML("(none)")
		} else {
			data.Trybots = template.HTML("
" + buf.String() + "
")
		}
	}

	buf.Reset()
	gcePool.WriteHTMLStatus(&buf)
	data.GCEPoolStatus = template.HTML(buf.String())
	buf.Reset()

	kubePool.WriteHTMLStatus(&buf)
	data.KubePoolStatus = template.HTML(buf.String())
	buf.Reset()

	reversePool.WriteHTMLStatus(&buf)
	data.ReversePoolStatus = template.HTML(buf.String())

	buf.Reset()
	if err := statusTmpl.Execute(&buf, data); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	buf.WriteTo(w)
}

func fdCount() int {
	f, err := os.Open("/proc/self/fd")
	if err != nil {
		return -1
	}
	defer f.Close()
	n := 0
	for {
		names, err := f.Readdirnames(1000)
		n += len(names)
		if err == io.EOF {
			return n
		}
		if err != nil {
			return -1
		}
	}
}

func friendlyDuration(d time.Duration) string {
	if d > 10*time.Second {
		d2 := ((d + 50*time.Millisecond) / (100 * time.Millisecond)) * (100 * time.Millisecond)
		return d2.String()
	}
	if d > time.Second {
		d2 := ((d + 5*time.Millisecond) / (10 * time.Millisecond)) * (10 * time.Millisecond)
		return d2.String()
	}
	d2 := ((d + 50*time.Microsecond) / (100 * time.Microsecond)) * (100 * time.Microsecond)
	return d2.String()
}

func diskFree() string {
	out, _ := exec.Command("df", "-h").Output()
	return string(out)
}

// statusData is the data that fills out statusTmpl.
type statusData struct {
	Total             int // number of total builds (including those waiting for a buildlet)
	ActiveBuilds      int // number of running builds (subset of Total with a buildlet)
	NumFD             int
	NumGoroutine      int
	Uptime            time.Duration
	Active            []*buildStatus // have a buildlet
	Pending           []*buildStatus // waiting on a buildlet
	Recent            []*buildStatus
	TrybotsErr        string
	Trybots           template.HTML
	GCEPoolStatus     template.HTML // TODO: embed template
	KubePoolStatus    template.HTML // TODO: embed template
	ReversePoolStatus template.HTML // TODO: embed template
	RemoteBuildlets   template.HTML
	DiskFree          string
	Version           string
}

var statusTmpl = template.Must(template.New("status").Parse(`
<!DOCTYPE html>

<link rel="stylesheet" href="/style.css"/><title>Go Farmer</title>

Go Build Coordinator

		Dashboard
		Builders

Running

{{printf "%d" .Total}} total builds; {{printf "%d" .ActiveBuilds}} active. Uptime {{printf "%s" .Uptime}}. Version {{.Version}}.

🔗 Active Trybot Runs

{{- if .TrybotsErr}}
trybots disabled:: {{.TrybotsErr}}
{{else}}
{{.Trybots}}
{{end}}

🔗 Remote buildlets

{{.RemoteBuildlets}}

🔗 Buildlet pools

{{.GCEPoolStatus}}

{{.KubePoolStatus}}

{{.ReversePoolStatus}}

🔗 Active builds

{{range .Active}}

{{.HTMLStatusLine}}

{{end}}

🔗 Pending builds

{{range .Pending}}

{{.HTMLStatusLine}}

{{end}}

🔗 Recently completed

{{range .Recent}}

{{.HTMLStatusLine_done}}

{{end}}

🔗 Disk Space

{{.DiskFree}}

🔗 File Descriptors

{{.NumFD}}

🔗 Goroutines

{{.NumGoroutine}} goroutines

`))

func handleStyleCSS(w http.ResponseWriter, r *http.Request) {
	src := strings.NewReader(styleCSS)
	http.ServeContent(w, r, "style.css", processStartTime, src)
}

const styleCSS = `
body {
	font-family: sans-serif;
	color: #222;
	padding: 10px;
	margin: 0;
}

h1, h2 { color: #375EAB; }
h1 { font-size: 24px; }
h2 { font-size: 20px; }

pre {
	font-family: monospace;
	font-size: 9pt;
}

header {
	margin: -10px -10px 0 -10px;
	padding: 10px 10px;
	background: #E0EBF5;
}
header a { color: #222; }
header h1 {
	display: inline;
	margin: 0;
	padding-top: 5px;
}
header nav {
	display: inline-block;
	margin-left: 20px;
}
header nav a {
	display: inline-block;
	padding: 10px;
	margin: 0;
	margin-right: 5px;
	color: white;
	background: #375EAB;
	text-decoration: none;
	font-size: 16px;
	border: 1px solid #375EAB;
	border-radius: 5px;
}

table {
	border-collapse: collapse;
	font-size: 9pt;
}

table td, table th, table td, table th {
	text-align: left;
	vertical-align: top;
	padding: 2px 6px;
}

table thead tr {
	background: #fff !important;
}
`
<commit_msg>cmd/coordinator: make farmer styles slightly less hideous<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"html/template"
	"io"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"sort"
	"strings"
	"sync/atomic"
	"time"
)

func handleStatus(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.NotFound(w, r)
		return
	}
	round := func(t time.Duration) time.Duration {
		return t / time.Second * time.Second
	}
	df := diskFree()

	statusMu.Lock()
	data := statusData{
		Total:        len(status),
		Uptime:       round(time.Now().Sub(processStartTime)),
		Recent:       append([]*buildStatus{}, statusDone...),
		DiskFree:     df,
		Version:      Version,
		NumFD:        fdCount(),
		NumGoroutine: runtime.NumGoroutine(),
	}
	for _, st := range status {
		if atomic.LoadInt32(&st.hasBuildlet) != 0 {
			data.ActiveBuilds++
			data.Active = append(data.Active, st)
		} else {
			data.Pending = append(data.Pending, st)
		}
	}
	// TODO: make this prettier.
	var buf bytes.Buffer
	for _, key := range tryList {
		if ts := tries[key]; ts != nil {
			state := ts.state()
			fmt.Fprintf(&buf, "Change-ID: %v Commit: %v (status)\n",
				key.ChangeTriple(), key.Commit, key.Commit[:8])
			fmt.Fprintf(&buf, "   Remain: %d, fails: %v\n", state.remain, state.failed)
			for _, bs := range ts.builds {
				fmt.Fprintf(&buf, "  %s: running=%v\n", bs.Name, bs.isRunning())
			}
		}
	}
	statusMu.Unlock()

	data.RemoteBuildlets = template.HTML(remoteBuildletStatus())

	sort.Sort(byAge(data.Active))
	sort.Sort(byAge(data.Pending))
	sort.Sort(sort.Reverse(byAge(data.Recent)))
	if errTryDeps != nil {
		data.TrybotsErr = errTryDeps.Error()
	} else {
		if buf.Len() == 0 {
			data.Trybots = template.HTML("(none)")
		} else {
			data.Trybots = template.HTML("
" + buf.String() + "
")
		}
	}

	buf.Reset()
	gcePool.WriteHTMLStatus(&buf)
	data.GCEPoolStatus = template.HTML(buf.String())
	buf.Reset()

	kubePool.WriteHTMLStatus(&buf)
	data.KubePoolStatus = template.HTML(buf.String())
	buf.Reset()

	reversePool.WriteHTMLStatus(&buf)
	data.ReversePoolStatus = template.HTML(buf.String())

	buf.Reset()
	if err := statusTmpl.Execute(&buf, data); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	buf.WriteTo(w)
}

func fdCount() int {
	f, err := os.Open("/proc/self/fd")
	if err != nil {
		return -1
	}
	defer f.Close()
	n := 0
	for {
		names, err := f.Readdirnames(1000)
		n += len(names)
		if err == io.EOF {
			return n
		}
		if err != nil {
			return -1
		}
	}
}

func friendlyDuration(d time.Duration) string {
	if d > 10*time.Second {
		d2 := ((d + 50*time.Millisecond) / (100 * time.Millisecond)) * (100 * time.Millisecond)
		return d2.String()
	}
	if d > time.Second {
		d2 := ((d + 5*time.Millisecond) / (10 * time.Millisecond)) * (10 * time.Millisecond)
		return d2.String()
	}
	d2 := ((d + 50*time.Microsecond) / (100 * time.Microsecond)) * (100 * time.Microsecond)
	return d2.String()
}

func diskFree() string {
	out, _ := exec.Command("df", "-h").Output()
	return string(out)
}

// statusData is the data that fills out statusTmpl.
type statusData struct {
	Total             int // number of total builds (including those waiting for a buildlet)
	ActiveBuilds      int // number of running builds (subset of Total with a buildlet)
	NumFD             int
	NumGoroutine      int
	Uptime            time.Duration
	Active            []*buildStatus // have a buildlet
	Pending           []*buildStatus // waiting on a buildlet
	Recent            []*buildStatus
	TrybotsErr        string
	Trybots           template.HTML
	GCEPoolStatus     template.HTML // TODO: embed template
	KubePoolStatus    template.HTML // TODO: embed template
	ReversePoolStatus template.HTML // TODO: embed template
	RemoteBuildlets   template.HTML
	DiskFree          string
	Version           string
}

var statusTmpl = template.Must(template.New("status").Parse(`
<!DOCTYPE html>

<link rel="stylesheet" href="/style.css"/><title>Go Farmer</title>

Go Build Coordinator

		Dashboard
		Builders

Running

{{printf "%d" .Total}} total builds; {{printf "%d" .ActiveBuilds}} active. Uptime {{printf "%s" .Uptime}}. Version {{.Version}}.

Active Trybot Runs ¶

{{- if .TrybotsErr}}
trybots disabled:: {{.TrybotsErr}}
{{else}}
{{.Trybots}}
{{end}}

Remote buildlets ¶

{{.RemoteBuildlets}}

Buildlet pools ¶

{{.GCEPoolStatus}}

{{.KubePoolStatus}}

{{.ReversePoolStatus}}

Active builds ¶

	{{range .Active}}

{{.HTMLStatusLine}}

	{{end}}

Pending builds ¶

	{{range .Pending}}

{{.HTMLStatusLine}}

	{{end}}

Recently completed ¶

	{{range .Recent}}

{{.HTMLStatusLine_done}}

	{{end}}

Disk Space ¶

{{.DiskFree}}

File Descriptors ¶

{{.NumFD}}

Goroutines ¶

{{.NumGoroutine}} goroutines

`))

func handleStyleCSS(w http.ResponseWriter, r *http.Request) {
	src := strings.NewReader(styleCSS)
	http.ServeContent(w, r, "style.css", processStartTime, src)
}

const styleCSS = `
body {
	font-family: sans-serif;
	color: #222;
	padding: 10px;
	margin: 0;
}

h1, h2, h1 > a, h2 > a, h1 > a:visited, h2 > a:visited {
	color: #375EAB;
}
h1 { font-size: 24px; }
h2 { font-size: 20px; }

h1 > a, h2 > a {
	display: none;
	text-decoration: none;
}

h1:hover > a, h2:hover > a {
	display: inline;
}

h1 > a:hover, h2 > a:hover {
	text-decoration: underline;
}

pre {
	font-family: monospace;
	font-size: 9pt;
}

header {
	margin: -10px -10px 0 -10px;
	padding: 10px 10px;
	background: #E0EBF5;
}
header a { color: #222; }
header h1 {
	display: inline;
	margin: 0;
	padding-top: 5px;
}
header nav {
	display: inline-block;
	margin-left: 20px;
}
header nav a {
	display: inline-block;
	padding: 10px;
	margin: 0;
	margin-right: 5px;
	color: white;
	background: #375EAB;
	text-decoration: none;
	font-size: 16px;
	border: 1px solid #375EAB;
	border-radius: 5px;
}

table {
	border-collapse: collapse;
	font-size: 9pt;
}

table td, table th, table td, table th {
	text-align: left;
	vertical-align: top;
	padding: 2px 6px;
}

table thead tr {
	background: #fff !important;
}
`
<|endoftext|>


<commit_before>package command

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"os/user"
	"path/filepath"
	"strings"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/pipestream"
)

type Upgrade struct {
	Ui  cli.Ui
	Cmd string

	uploadDir      string
	mode           string
	upgradeKateway bool
	upgradeZk      bool
	upgradeKguard  bool
	upgradeActord  bool
	upgradeConfig  bool
}

func (this *Upgrade) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("upgrade", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.uploadDir, "upload", "/var/www/html", "")
	cmdFlags.StringVar(&this.mode, "m", "d", "")
	cmdFlags.BoolVar(&this.upgradeConfig, "c", false, "")
	cmdFlags.BoolVar(&this.upgradeKateway, "k", false, "")
	cmdFlags.BoolVar(&this.upgradeActord, "at", false, "")
	cmdFlags.BoolVar(&this.upgradeZk, "zk", false, "")
	cmdFlags.BoolVar(&this.upgradeKguard, "kg", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	gopath := os.Getenv("GOPATH")
	usr, _ := user.Current()

	if this.upgradeConfig {
		switch this.mode {
		case "d":
			this.runCmd("wget", []string{this.storeUrl(".gafka.cf"), "-O", ".gafka.cf"})
			this.runCmd("mv", []string{"-f", ".gafka.cf", filepath.Join(usr.HomeDir, ".gafka.cf")})

		case "u":
			this.runCmd("cp", []string{"-f", filepath.Join(usr.HomeDir, ".gafka.cf"), this.uploadDir})
		}

		return
	}

	if this.upgradeActord {
		switch this.mode {
		case "d":
			this.runCmd("wget", []string{this.storeUrl("actord"), "-O", "actord"})
			this.runCmd("chmod", []string{"a+x", "actord"})
			this.runCmd("mv", []string{"-f", "actord", "/var/wd/actord/actord"})

		case "u":
			this.runCmd("cp", []string{"-f", fmt.Sprintf("%s/bin/actord", gopath), this.uploadDir})
		}

		return
	}

	if this.upgradeKateway {
		switch this.mode {
		case "d":
			this.runCmd("wget", []string{this.storeUrl("kateway"), "-O", "kateway"})
			this.runCmd("chmod", []string{"a+x", "kateway"})
			this.runCmd("mv", []string{"-f", "kateway", "/var/wd/kateway/kateway"})

		case "u":
			this.Ui.Warn("you must run './build.sh -it kateway' first.")
			this.runCmd("cp", []string{"-f", fmt.Sprintf("%s/bin/kateway", gopath), this.uploadDir})
		}

		return
	}

	if this.upgradeZk {
		switch this.mode {
		case "d":
			this.runCmd("wget", []string{this.storeUrl("zk"), "-O", "zk"})
			this.runCmd("chmod", []string{"a+x", "zk"})
			this.runCmd("mv", []string{"-f", "zk", "/usr/bin/zk"})

		case "u":
			this.runCmd("cp", []string{"-f", fmt.Sprintf("%s/bin/zk", gopath), this.uploadDir})
		}

		return
	}

	if this.upgradeKguard {
		switch this.mode {
		case "d":
			this.runCmd("wget", []string{this.storeUrl("kguard"), "-O", "kguard"})
			this.runCmd("chmod", []string{"a+x", "kguard"})
			this.runCmd("mv", []string{"-f", "kguard", "/var/wd/kguard/kguard"})

		case "u":
			this.runCmd("cp", []string{"-f", fmt.Sprintf("%s/bin/kguard", gopath), this.uploadDir})
		}

		return
	}

	// work on gk
	switch this.mode {
	case "d":
		u, err := user.Current()
		swallow(err)
		this.runCmd("/usr/bin/gk", []string{"-v"})
		this.runCmd("rm", []string{"-f", "gk"})
		if this.upgradeConfig {
			this.runCmd("rm", []string{"-f", fmt.Sprintf("%s/.gafka.cf", u.HomeDir)})
		}
		this.runCmd("wget", []string{this.storeUrl("gk"), "-O", "gk"})
		this.runCmd("chmod", []string{"a+x", "gk"})
		this.runCmd("mv", []string{"-f", "gk", "/usr/bin/gk"})
		this.runCmd("/usr/bin/gk", []string{"-v"})

	case "u":
		this.runCmd("cp", []string{"-f", fmt.Sprintf("%s/bin/gk", gopath), this.uploadDir})

	default:
		this.Ui.Error("invalid mode")
		return 2
	}

	this.Ui.Info("ok")
	return
}

func (this *Upgrade) storeUrl(fn string) string {
	return fmt.Sprintf("%s/%s", ctx.UpgradeCenter(), fn)
}

func (this *Upgrade) runCmd(c string, args []string) {
	this.Ui.Output(fmt.Sprintf("  %s %+v", c, args))

	cmd := pipestream.New(c, args...)
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		this.Ui.Output(fmt.Sprintf("    %s", scanner.Text()))
	}
	err = scanner.Err()
	if err != nil {
		this.Ui.Error(err.Error())
	}

}

func (*Upgrade) Synopsis() string {
	return "Upgrade local gk to latest version"
}

func (this *Upgrade) Help() string {
	help := fmt.Sprintf(`
Usage: %s upgrade [options]

    Upgrade local gk to latest version

Options:

    -c
      Upgrade local $HOME/.gafka.cf

    -k
      Upgrade kateway instead of gk

    -zk
      Upgrade zk instead of gk

    -kg
      Upgrade kguard instead of gk

    -at
      Upgrade actord instead of gk

    -m <d|u>
      Download or upload mode
      Defaults download mode

    -upload dir
      Upload the gk file to target dir, only run on gk file server
      Defaults /var/www/html

`, this.Cmd)
	return strings.TrimSpace(help)
}
<commit_msg>upgrade gk conf by default<commit_after>package command

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"os/user"
	"path/filepath"
	"strings"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/pipestream"
)

type Upgrade struct {
	Ui  cli.Ui
	Cmd string

	uploadDir      string
	mode           string
	upgradeKateway bool
	upgradeZk      bool
	upgradeKguard  bool
	upgradeActord  bool
	upgradeConfig  bool
}

func (this *Upgrade) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("upgrade", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.uploadDir, "upload", "/var/www/html", "")
	cmdFlags.StringVar(&this.mode, "m", "d", "")
	cmdFlags.BoolVar(&this.upgradeConfig, "c", false, "")
	cmdFlags.BoolVar(&this.upgradeKateway, "k", false, "")
	cmdFlags.BoolVar(&this.upgradeActord, "at", false, "")
	cmdFlags.BoolVar(&this.upgradeZk, "zk", false, "")
	cmdFlags.BoolVar(&this.upgradeKguard, "kg", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	gopath := os.Getenv("GOPATH")
	usr, _ := user.Current()

	if this.upgradeConfig {
		switch this.mode {
		case "d":
			this.runCmd("wget", []string{this.storeUrl(".gafka.cf"), "-O", ".gafka.cf"})
			this.runCmd("mv", []string{"-f", ".gafka.cf", filepath.Join(usr.HomeDir, ".gafka.cf")})

		case "u":
			this.runCmd("cp", []string{"-f", filepath.Join(usr.HomeDir, ".gafka.cf"), this.uploadDir})
		}

		return
	}

	if this.upgradeActord {
		switch this.mode {
		case "d":
			this.runCmd("wget", []string{this.storeUrl("actord"), "-O", "actord"})
			this.runCmd("chmod", []string{"a+x", "actord"})
			this.runCmd("mv", []string{"-f", "actord", "/var/wd/actord/actord"})

		case "u":
			this.runCmd("cp", []string{"-f", fmt.Sprintf("%s/bin/actord", gopath), this.uploadDir})
		}

		return
	}

	if this.upgradeKateway {
		switch this.mode {
		case "d":
			this.runCmd("wget", []string{this.storeUrl("kateway"), "-O", "kateway"})
			this.runCmd("chmod", []string{"a+x", "kateway"})
			this.runCmd("mv", []string{"-f", "kateway", "/var/wd/kateway/kateway"})

		case "u":
			this.Ui.Warn("you must run './build.sh -it kateway' first.")
			this.runCmd("cp", []string{"-f", fmt.Sprintf("%s/bin/kateway", gopath), this.uploadDir})
		}

		return
	}

	if this.upgradeZk {
		switch this.mode {
		case "d":
			this.runCmd("wget", []string{this.storeUrl("zk"), "-O", "zk"})
			this.runCmd("chmod", []string{"a+x", "zk"})
			this.runCmd("mv", []string{"-f", "zk", "/usr/bin/zk"})

		case "u":
			this.runCmd("cp", []string{"-f", fmt.Sprintf("%s/bin/zk", gopath), this.uploadDir})
		}

		return
	}

	if this.upgradeKguard {
		switch this.mode {
		case "d":
			this.runCmd("wget", []string{this.storeUrl("kguard"), "-O", "kguard"})
			this.runCmd("chmod", []string{"a+x", "kguard"})
			this.runCmd("mv", []string{"-f", "kguard", "/var/wd/kguard/kguard"})

		case "u":
			this.runCmd("cp", []string{"-f", fmt.Sprintf("%s/bin/kguard", gopath), this.uploadDir})
		}

		return
	}

	// work on gk
	switch this.mode {
	case "d":
		// upgrade gk
		u, err := user.Current()
		swallow(err)
		this.runCmd("/usr/bin/gk", []string{"-v"})
		this.runCmd("rm", []string{"-f", "gk"})
		if this.upgradeConfig {
			this.runCmd("rm", []string{"-f", fmt.Sprintf("%s/.gafka.cf", u.HomeDir)})
		}
		this.runCmd("wget", []string{this.storeUrl("gk"), "-O", "gk"})
		this.runCmd("chmod", []string{"a+x", "gk"})
		this.runCmd("mv", []string{"-f", "gk", "/usr/bin/gk"})
		this.runCmd("/usr/bin/gk", []string{"-v"})

		// upgrade conf
		this.runCmd("wget", []string{this.storeUrl(".gafka.cf"), "-O", ".gafka.cf"})
		this.runCmd("mv", []string{"-f", ".gafka.cf", filepath.Join(usr.HomeDir, ".gafka.cf")})

	case "u":
		this.runCmd("cp", []string{"-f", fmt.Sprintf("%s/bin/gk", gopath), this.uploadDir})

	default:
		this.Ui.Error("invalid mode")
		return 2
	}

	this.Ui.Info("ok")
	return
}

func (this *Upgrade) storeUrl(fn string) string {
	return fmt.Sprintf("%s/%s", ctx.UpgradeCenter(), fn)
}

func (this *Upgrade) runCmd(c string, args []string) {
	this.Ui.Output(fmt.Sprintf("  %s %+v", c, args))

	cmd := pipestream.New(c, args...)
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		this.Ui.Output(fmt.Sprintf("    %s", scanner.Text()))
	}
	err = scanner.Err()
	if err != nil {
		this.Ui.Error(err.Error())
	}

}

func (*Upgrade) Synopsis() string {
	return "Upgrade local gk to latest version"
}

func (this *Upgrade) Help() string {
	help := fmt.Sprintf(`
Usage: %s upgrade [options]

    Upgrade local gk to latest version

Options:

    -c
      Upgrade local $HOME/.gafka.cf

    -k
      Upgrade kateway instead of gk

    -zk
      Upgrade zk instead of gk

    -kg
      Upgrade kguard instead of gk

    -at
      Upgrade actord instead of gk

    -m <d|u>
      Download or upload mode
      Defaults download mode

    -upload dir
      Upload the gk file to target dir, only run on gk file server
      Defaults /var/www/html

`, this.Cmd)
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"syscall"
	"time"

	"github.com/datawire/teleproxy/pkg/supervisor"
)

// Resource represents one thing managed by playpen daemon. Examples include
// network intercepts (via teleproxy intercept) and cluster connectivity.
type Resource interface {
	Name() string
	IsOkay() bool
	Close() error
}

// crCmd is a handle to a checked retrying command
type crCmd struct {
	name     string
	args     []string
	rai      *RunAsInfo
	check    func() error
	tasks    chan func() error
	callerP  *supervisor.Process // processor's Process
	cmd      *supervisor.Cmd     // (run loop) tracks the cmd for killing it
	quitting bool                // (Close) to get everything to quit
	okay     bool                // (monitor) cmd is running and check passes
}

// CheckedRetryingCommand launches a command, restarting it repeatedly if it
// quits, and killing and restarting it if it fails the given check.
func CheckedRetryingCommand(
	p *supervisor.Process, name string, args []string, rai *RunAsInfo, check func() error,
) (Resource, error) {
	if check == nil {
		check = func() error { return nil }
	}
	crc := &crCmd{
		name:    name,
		args:    args,
		rai:     rai,
		check:   check,
		tasks:   make(chan func() error, 1),
		callerP: p,
	}
	p.Supervisor().Supervise(&supervisor.Worker{
		Name: "crc/" + crc.name,
		Work: crc.processor,
	})
	if err := crc.launch(); err != nil {
		return nil, err
	}
	return crc, nil
}

// Name implements Resource
func (crc *crCmd) Name() string {
	res := make(chan string)
	crc.tasks <- func() error {
		res <- crc.name
		return nil
	}
	return <-res
}

// IsOkay returns whether the resource is okay as far as monitoring is aware
func (crc *crCmd) IsOkay() bool {
	res := make(chan bool)
	crc.tasks <- func() error {
		res <- crc.okay
		return nil
	}
	return <-res
}

// Close shuts down this resource
func (crc *crCmd) Close() error {
	done := make(chan struct{})
	crc.tasks <- func() error {
		defer close(done)
		return crc.quit()
	}
	<-done

	// FIXME: Wait until things have closed?
	return nil
}

func (crc *crCmd) processor(p *supervisor.Process) error {
	ticker := time.NewTicker(3 * time.Second)
	defer ticker.Stop()
	p.Ready()
	for {
		var task func() error
		select {
		case fn := <-crc.tasks: // There is work to do
			task = fn
		case <-ticker.C: // Ticker says it's time to monitor
			task = func() error { return crc.monitor(p) }
		case <-p.Shutdown(): // Supervisor told us to quit
			task = crc.quit
		}
		if err := task(); err != nil {
			return err
		}
		if crc.quitting {
			return nil
		}
	}
}

func (crc *crCmd) launch() error {
	if crc.cmd != nil {
		panic(fmt.Errorf("launching %s: already launched", crc.name))
	}
	launchErr := make(chan error)
	crc.callerP.Supervisor().Supervise(&supervisor.Worker{
		Name: "proc/" + crc.name,
		Work: func(p *supervisor.Process) error {
			// Launch the subprocess
			crc.cmd = crc.rai.Command(p, crc.args...)
			if err := crc.cmd.Start(); err != nil {
				launchErr <- err
				return nil
			}
			launchErr <- nil
			p.Ready()

			// Wait for the subprocess to end, log
			p.Logf("subprocess ended: %v", p.DoClean(crc.cmd.Wait, crc.kill))
			crc.cmd = nil

			return nil
		},
	})
	select {
	case err := <-launchErr:
		return err
	case <-crc.callerP.Shutdown():
		return nil
	}
}

func (crc *crCmd) kill() error {
	if crc.cmd != nil {
		return crc.cmd.Process.Signal(syscall.SIGTERM)
	}
	return nil // Or fmt.Errorf("trying to kill non-running subprocess for %s", crc.name)
}

func (crc *crCmd) quit() error {
	crc.quitting = true
	return crc.kill()
}

// monitor determines and records whether the resource is okay
func (crc *crCmd) monitor(p *supervisor.Process) error {
	defer func(old bool) { MaybeNotify(p, crc.name, old, crc.okay) }(crc.okay)
	if crc.cmd == nil {
		crc.okay = false // Not running is not okay
		crc.tasks <- crc.launch
		return nil
	}
	if err := crc.check(); err != nil {
		crc.okay = false // Check failed is not okay
		p.Logf("check failed: %v", err)
		// Kill the process because it's in a bad state
		if err := crc.kill(); err != nil {
			// Failure to kill is a fatal error
			// FIXME: This will be a problem if the resource is in the process
			// of dying when we user-check it, but is dead when we get around to
			// killing it.
			p.Logf("failed to kill: %v", err)
			return err
		}
	}
	crc.okay = true
	return nil
}
<commit_msg>Add ResourceBase and KCluster Resource<commit_after>package main

import (
	"fmt"
	"strings"
	"syscall"
	"time"

	"github.com/datawire/teleproxy/pkg/supervisor"
)

// Resource represents one thing managed by playpen daemon. Examples include
// network intercepts (via teleproxy intercept) and cluster connectivity.
type Resource interface {
	Name() string
	IsOkay() bool
	Close() error
}

// ResourceBase has helpers to create a monitored resource
type ResourceBase struct {
	name     string
	doCheck  func(*supervisor.Process) error
	doQuit   func() error
	tasks    chan func() error
	quitting bool // (Close) to get everything to quit
	okay     bool // (monitor) cmd is running and check passes
}

// Name implements Resource
func (rb *ResourceBase) Name() string {
	res := make(chan string)
	rb.tasks <- func() error {
		res <- rb.name
		return nil
	}
	return <-res
}

// IsOkay returns whether the resource is okay as far as monitoring is aware
func (rb *ResourceBase) IsOkay() bool {
	res := make(chan bool)
	rb.tasks <- func() error {
		res <- rb.okay
		return nil
	}
	return <-res
}

// Close shuts down this resource
func (rb *ResourceBase) Close() error {
	done := make(chan struct{})
	rb.tasks <- func() error {
		defer close(done)
		return rb.quit()
	}
	<-done

	// FIXME: Wait until things have closed?
	return nil
}

func (rb *ResourceBase) quit() error {
	rb.quitting = true
	return rb.doQuit()
}

func (rb *ResourceBase) monitor(p *supervisor.Process) error {
	old := rb.okay
	if err := rb.doCheck(p); err != nil {
		rb.okay = false // Check failed is not okay
		p.Logf("check failed: %v", err)
	} else {
		rb.okay = true
	}
	MaybeNotify(p, rb.name, old, rb.okay)
	return nil
}

func (rb *ResourceBase) processor(p *supervisor.Process) error {
	ticker := time.NewTicker(3 * time.Second)
	defer ticker.Stop()
	p.Ready()
	for {
		var task func() error
		select {
		case fn := <-rb.tasks: // There is work to do
			task = fn
		case <-ticker.C: // Ticker says it's time to monitor
			task = func() error { return rb.monitor(p) }
		case <-p.Shutdown(): // Supervisor told us to quit
			task = rb.quit
		}
		if err := task(); err != nil {
			return err
		}
		if rb.quitting {
			return nil
		}
	}
}

// cluster is a Kubernetes cluster reference
type cluster struct {
	context string
	server  string
	rai     *RunAsInfo
	kargs   []string
	ResourceBase
}

func (c *cluster) getKubectlCmd(p *supervisor.Process, args ...string) *supervisor.Cmd {
	cmdArgs := make([]string, 0, 1+len(c.kargs)+len(args))
	cmdArgs = append(cmdArgs, "kubectl")
	cmdArgs = append(cmdArgs, c.kargs...)
	cmdArgs = append(cmdArgs, args...)
	return c.rai.Command(p, cmdArgs...)
}

// check for cluster connectivity
func (c *cluster) check(p *supervisor.Process) error {
	cmd := c.getKubectlCmd(p, "get", "po", "ohai", "--ignore-not-found")
	return cmd.Run()
}

// KCluster tracks connectivity to a cluster
func KCluster(p *supervisor.Process, args *ConnectArgs) (Resource, error) {
	c := &cluster{
		rai:   args.RAI,
		kargs: args.KArgs,
		ResourceBase: ResourceBase{
			name:  "cluster",
			tasks: make(chan func() error, 1),
		},
	}
	c.doCheck = c.check
	c.doQuit = func() error { return nil }

	if err := c.check(p); err != nil {
		return nil, err
	}

	cmd := c.getKubectlCmd(p, "config", "current-context")
	output, err := cmd.CombinedOutput()
	if err != nil {
		return nil, err
	}
	c.context = strings.TrimSpace(string(output))

	cmd = c.getKubectlCmd(p, "config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}")
	output, err = cmd.CombinedOutput()
	if err != nil {
		return nil, err
	}
	c.server = strings.TrimSpace(string(output))

	p.Supervisor().Supervise(&supervisor.Worker{
		Name: "cluster",
		Work: c.processor,
	})

	return c, nil
}

// crCmd is a handle to a checked retrying command
type crCmd struct {
	name     string
	args     []string
	rai      *RunAsInfo
	check    func() error
	tasks    chan func() error
	callerP  *supervisor.Process // processor's Process
	cmd      *supervisor.Cmd     // (run loop) tracks the cmd for killing it
	quitting bool                // (Close) to get everything to quit
	okay     bool                // (monitor) cmd is running and check passes
}

// CheckedRetryingCommand launches a command, restarting it repeatedly if it
// quits, and killing and restarting it if it fails the given check.
func CheckedRetryingCommand(
	p *supervisor.Process, name string, args []string, rai *RunAsInfo, check func() error,
) (Resource, error) {
	if check == nil {
		check = func() error { return nil }
	}
	crc := &crCmd{
		name:    name,
		args:    args,
		rai:     rai,
		check:   check,
		tasks:   make(chan func() error, 1),
		callerP: p,
	}
	p.Supervisor().Supervise(&supervisor.Worker{
		Name: "crc/" + crc.name,
		Work: crc.processor,
	})
	if err := crc.launch(); err != nil {
		return nil, err
	}
	return crc, nil
}

// Name implements Resource
func (crc *crCmd) Name() string {
	res := make(chan string)
	crc.tasks <- func() error {
		res <- crc.name
		return nil
	}
	return <-res
}

// IsOkay returns whether the resource is okay as far as monitoring is aware
func (crc *crCmd) IsOkay() bool {
	res := make(chan bool)
	crc.tasks <- func() error {
		res <- crc.okay
		return nil
	}
	return <-res
}

// Close shuts down this resource
func (crc *crCmd) Close() error {
	done := make(chan struct{})
	crc.tasks <- func() error {
		defer close(done)
		return crc.quit()
	}
	<-done

	// FIXME: Wait until things have closed?
	return nil
}

func (crc *crCmd) processor(p *supervisor.Process) error {
	ticker := time.NewTicker(3 * time.Second)
	defer ticker.Stop()
	p.Ready()
	for {
		var task func() error
		select {
		case fn := <-crc.tasks: // There is work to do
			task = fn
		case <-ticker.C: // Ticker says it's time to monitor
			task = func() error { return crc.monitor(p) }
		case <-p.Shutdown(): // Supervisor told us to quit
			task = crc.quit
		}
		if err := task(); err != nil {
			return err
		}
		if crc.quitting {
			return nil
		}
	}
}

func (crc *crCmd) launch() error {
	if crc.cmd != nil {
		panic(fmt.Errorf("launching %s: already launched", crc.name))
	}
	launchErr := make(chan error)
	crc.callerP.Supervisor().Supervise(&supervisor.Worker{
		Name: "proc/" + crc.name,
		Work: func(p *supervisor.Process) error {
			// Launch the subprocess
			crc.cmd = crc.rai.Command(p, crc.args...)
			if err := crc.cmd.Start(); err != nil {
				launchErr <- err
				return nil
			}
			launchErr <- nil
			p.Ready()

			// Wait for the subprocess to end, log
			p.Logf("subprocess ended: %v", p.DoClean(crc.cmd.Wait, crc.kill))
			crc.cmd = nil

			return nil
		},
	})
	select {
	case err := <-launchErr:
		return err
	case <-crc.callerP.Shutdown():
		return nil
	}
}

func (crc *crCmd) kill() error {
	if crc.cmd != nil {
		return crc.cmd.Process.Signal(syscall.SIGTERM)
	}
	return nil // Or fmt.Errorf("trying to kill non-running subprocess for %s", crc.name)
}

func (crc *crCmd) quit() error {
	crc.quitting = true
	return crc.kill()
}

// monitor determines and records whether the resource is okay
func (crc *crCmd) monitor(p *supervisor.Process) error {
	defer func(old bool) { MaybeNotify(p, crc.name, old, crc.okay) }(crc.okay)
	if crc.cmd == nil {
		crc.okay = false // Not running is not okay
		crc.tasks <- crc.launch
		return nil
	}
	if err := crc.check(); err != nil {
		crc.okay = false // Check failed is not okay
		p.Logf("check failed: %v", err)
		// Kill the process because it's in a bad state
		if err := crc.kill(); err != nil {
			// Failure to kill is a fatal error
			// FIXME: This will be a problem if the resource is in the process
			// of dying when we user-check it, but is dead when we get around to
			// killing it.
			p.Logf("failed to kill: %v", err)
			return err
		}
	}
	crc.okay = true
	return nil
}
<|endoftext|>