<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"fmt"
	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/state/apiserver"
	"launchpad.net/juju-core/worker"
	"launchpad.net/juju-core/worker/firewaller"
	"launchpad.net/juju-core/worker/machiner"
	"launchpad.net/juju-core/worker/provisioner"
	"launchpad.net/tomb"
	"path/filepath"
	"time"
)

var retryDelay = 3 * time.Second

// MachineAgent is a cmd.Command responsible for running a machine agent.
type MachineAgent struct {
	cmd.CommandBase
	tomb      tomb.Tomb
	Conf      AgentConf
	MachineId string
	runner    *worker.Runner
}

// Info returns usage information for the command.
func (a *MachineAgent) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "machine",
		Purpose: "run a juju machine agent",
	}
}

func (a *MachineAgent) SetFlags(f *gnuflag.FlagSet) {
	a.Conf.addFlags(f)
	f.StringVar(&a.MachineId, "machine-id", "", "id of the machine to run")
}

// Init initializes the command for running.
func (a *MachineAgent) Init(args []string) error {
	if !state.IsMachineId(a.MachineId) {
		return fmt.Errorf("--machine-id option must be set, and expects a non-negative integer")
	}
	if err := a.Conf.checkArgs(args); err != nil {
		return err
	}
	a.runner = worker.NewRunner(isFatal, moreImportant)
	return nil
}

func (a *MachineAgent) Wait() error {
	return a.tomb.Wait()
}

// Stop stops the machine agent.
func (a *MachineAgent) Stop() error {
	a.runner.Kill()
	return a.tomb.Wait()
}

// Run runs a machine agent.
func (a *MachineAgent) Run(_ *cmd.Context) error {
	defer a.tomb.Done()
	log.Infof("machine agent start; tag %v", a.Tag())
	if err := a.Conf.read(a.Tag()); err != nil {
		return err
	}
	charm.CacheDir = filepath.Join(a.Conf.DataDir, "charmcache")
	defer a.tomb.Done()
	if a.MachineId == "0" {
		a.runner.StartWorker("state", a.StateWorker)
	}
	a.runner.StartWorker("api", a.APIWorker)
	err := agentDone(a.runner.Wait())
	a.tomb.Kill(err)
	return err
}

func allFatal(error) bool {
	return true
}

var stateJobs = map[params.MachineJob]bool{
	params.JobHostUnits:     true,
	params.JobManageEnviron: true,
	params.JobManageState:      true,
}

func (a *MachineAgent) APIWorker() (worker.Worker, error) {
	log.Infof("opening api state with conf %#v", a.Conf.Conf)
	st, entity, err := openAPIState(a.Conf.Conf, a)
	if err != nil {
		log.Infof("open api failure: %v", err)
		return nil, err
	}
	log.Infof("open api success")
	m := entity.(*api.Machine)
	needsStateWorker := false
	for _, job := range m.Jobs() {
		needsStateWorker = needsStateWorker || stateJobs[job]
	}
	if needsStateWorker {
		// Start any workers that require a state connection.
		// Note the idempotency of StartWorker.
		a.runner.StartWorker("state", a.StateWorker)
	}
	runner := worker.NewRunner(allFatal, moreImportant)
	// No agents currently connect to the API, so just
	// return the runner running nothing.
	return newCloseWorker(runner, st), nil
}

// StateJobs returns a worker running all the workers that require
// a *state.State connection.
func (a *MachineAgent) StateWorker() (worker.Worker, error) {
	st, entity, err := openState(a.Conf.Conf, a)
	if err != nil {
		return nil, err
	}
	// TODO(rog) use more discriminating test for errors
	// rather than taking everything down indiscriminately.
	runner := worker.NewRunner(allFatal, moreImportant)
	runner.StartWorker("upgrader", func() (worker.Worker, error) {
		return NewUpgrader(st, m, a.Conf.DataDir), nil
	})
	runner.StartWorker("machiner", func() (worker.Worker, error) {
		return machiner.NewMachiner(st, m.Id()), nil
	})
	for _, job := range m.Jobs() {
		switch job {
 		case state.JobHostUnits:
			runner.StartWorker("deployer", func() (worker.Worker, error) {
				return newDeployer(st, m.WatchPrincipalUnits(), a.Conf.DataDir), nil
			})
		case state.JobManageEnviron:
			runner.StartWorker("provisioner", func() (worker.Worker, error) {
				return provisioner.NewProvisioner(st, a.MachineId), nil
			})
			runner.StartWorker("firewaller", func() (worker.Worker, error) {
				return firewaller.NewFirewaller(st), nil
			})
		case state.JobManageState:
			runner.StartWorker("apiserver", func() (worker.Worker, error) {
				// If the configuration does not have the required information,
				// it is currently not a recoverable error, so we kill the whole
				// agent, potentially enabling human intervention to fix
				// the agent's configuration file. In the future, we may retrieve
				// the state server certificate and key from the state, and
				// this should then change.
				if len(a.Conf.StateServerCert) == 0 || len(a.Conf.StateServerKey) == 0 {
					return nil, &fatalError{"configuration does not have state server cert/key"}
				}
				return apiserver.NewServer(st, fmt.Sprintf(":%d", a.Conf.APIPort), a.Conf.StateServerCert, a.Conf.StateServerKey)
			})
		default:
			log.Warningf("ignoring unknown job %q", job)
		}
	}
	return newCloseWorker(runner, st), nil
}

func (a *MachineAgent) Entity(st *state.State) (AgentState, error) {
	m, err := st.Machine(a.MachineId)
	if err != nil {
		return nil, err
	}
	// Check the machine nonce as provisioned matches the agent.Conf value.
	if !m.CheckProvisioned(a.Conf.MachineNonce) {
		// The agent is running on a different machine to the one it
		// should be according to state. It must stop immediately.
		log.Errorf("running machine %v agent on inappropriate instance", m)
		return nil, worker.ErrTerminateAgent
	}
	return m, nil
}

func (a *MachineAgent) APIEntity(st *api.State) (AgentAPIState, error) {
	m, err := st.Machine(a.MachineId)
	if err != nil {
		return nil, err
	}
	// TODO(rog) move the CheckProvisioned test into
	// this method when it's implemented in the API
	return m, nil
}

func (a *MachineAgent) Tag() string {
	return state.MachineTag(a.MachineId)
}
<commit_msg>cmd/jujud: use machineagent api<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"fmt"
	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api"
	"launchpad.net/juju-core/state/api/machineagent"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/state/apiserver"
	"launchpad.net/juju-core/worker"
	"launchpad.net/juju-core/worker/firewaller"
	"launchpad.net/juju-core/worker/machiner"
	"launchpad.net/juju-core/worker/provisioner"
	"launchpad.net/tomb"
	"path/filepath"
	"time"
)

var retryDelay = 3 * time.Second

// MachineAgent is a cmd.Command responsible for running a machine agent.
type MachineAgent struct {
	cmd.CommandBase
	tomb      tomb.Tomb
	Conf      AgentConf
	MachineId string
	runner    *worker.Runner
}

// Info returns usage information for the command.
func (a *MachineAgent) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "machine",
		Purpose: "run a juju machine agent",
	}
}

func (a *MachineAgent) SetFlags(f *gnuflag.FlagSet) {
	a.Conf.addFlags(f)
	f.StringVar(&a.MachineId, "machine-id", "", "id of the machine to run")
}

// Init initializes the command for running.
func (a *MachineAgent) Init(args []string) error {
	if !state.IsMachineId(a.MachineId) {
		return fmt.Errorf("--machine-id option must be set, and expects a non-negative integer")
	}
	if err := a.Conf.checkArgs(args); err != nil {
		return err
	}
	a.runner = worker.NewRunner(isFatal, moreImportant)
	return nil
}

func (a *MachineAgent) Wait() error {
	return a.tomb.Wait()
}

// Stop stops the machine agent.
func (a *MachineAgent) Stop() error {
	a.runner.Kill()
	return a.tomb.Wait()
}

// Run runs a machine agent.
func (a *MachineAgent) Run(_ *cmd.Context) error {
	defer a.tomb.Done()
	log.Infof("machine agent start; tag %v", a.Tag())
	if err := a.Conf.read(a.Tag()); err != nil {
		return err
	}
	charm.CacheDir = filepath.Join(a.Conf.DataDir, "charmcache")
	defer a.tomb.Done()
	if a.MachineId == "0" {
		a.runner.StartWorker("state", a.StateWorker)
	}
	a.runner.StartWorker("api", a.APIWorker)
	err := agentDone(a.runner.Wait())
	a.tomb.Kill(err)
	return err
}

func allFatal(error) bool {
	return true
}

var stateJobs = map[params.MachineJob]bool{
	params.JobHostUnits:     true,
	params.JobManageEnviron: true,
	params.JobManageState:      true,
}

func (a *MachineAgent) APIWorker() (worker.Worker, error) {
	log.Infof("opening api state with conf %#v", a.Conf.Conf)
	st, entity, err := openAPIState(a.Conf.Conf, a)
	if err != nil {
		log.Infof("open api failure: %v", err)
		return nil, err
	}
	log.Infof("open api success")
	m := entity.(*machineagent.Machine)
	needsStateWorker := false
	for _, job := range m.Jobs() {
		needsStateWorker = needsStateWorker || stateJobs[job]
	}
	if needsStateWorker {
		// Start any workers that require a state connection.
		// Note the idempotency of StartWorker.
		a.runner.StartWorker("state", a.StateWorker)
	}
	runner := worker.NewRunner(allFatal, moreImportant)
	// No agents currently connect to the API, so just
	// return the runner running nothing.
	return newCloseWorker(runner, st), nil
}

// StateJobs returns a worker running all the workers that require
// a *state.State connection.
func (a *MachineAgent) StateWorker() (worker.Worker, error) {
	st, entity, err := openState(a.Conf.Conf, a)
	if err != nil {
		return nil, err
	}
	// TODO(rog) use more discriminating test for errors
	// rather than taking everything down indiscriminately.
	runner := worker.NewRunner(allFatal, moreImportant)
	runner.StartWorker("upgrader", func() (worker.Worker, error) {
		return NewUpgrader(st, m, a.Conf.DataDir), nil
	})
	runner.StartWorker("machiner", func() (worker.Worker, error) {
		return machiner.NewMachiner(st, m.Id()), nil
	})
	for _, job := range m.Jobs() {
		switch job {
 		case state.JobHostUnits:
			runner.StartWorker("deployer", func() (worker.Worker, error) {
				return newDeployer(st, m.WatchPrincipalUnits(), a.Conf.DataDir), nil
			})
		case state.JobManageEnviron:
			runner.StartWorker("provisioner", func() (worker.Worker, error) {
				return provisioner.NewProvisioner(st, a.MachineId), nil
			})
			runner.StartWorker("firewaller", func() (worker.Worker, error) {
				return firewaller.NewFirewaller(st), nil
			})
		case state.JobManageState:
			runner.StartWorker("apiserver", func() (worker.Worker, error) {
				// If the configuration does not have the required information,
				// it is currently not a recoverable error, so we kill the whole
				// agent, potentially enabling human intervention to fix
				// the agent's configuration file. In the future, we may retrieve
				// the state server certificate and key from the state, and
				// this should then change.
				if len(a.Conf.StateServerCert) == 0 || len(a.Conf.StateServerKey) == 0 {
					return nil, &fatalError{"configuration does not have state server cert/key"}
				}
				return apiserver.NewServer(st, fmt.Sprintf(":%d", a.Conf.APIPort), a.Conf.StateServerCert, a.Conf.StateServerKey)
			})
		default:
			log.Warningf("ignoring unknown job %q", job)
		}
	}
	return newCloseWorker(runner, st), nil
}

func (a *MachineAgent) Entity(st *state.State) (AgentState, error) {
	m, err := st.Machine(a.MachineId)
	if err != nil {
		return nil, err
	}
	// Check the machine nonce as provisioned matches the agent.Conf value.
	if !m.CheckProvisioned(a.Conf.MachineNonce) {
		// The agent is running on a different machine to the one it
		// should be according to state. It must stop immediately.
		log.Errorf("running machine %v agent on inappropriate instance", m)
		return nil, worker.ErrTerminateAgent
	}
	return m, nil
}

func (a *MachineAgent) APIEntity(st *api.State) (AgentAPIState, error) {
	m, err := st.MachineAgent().Machine(a.MachineId)
	if err != nil {
		return nil, err
	}
	// TODO(rog) move the CheckProvisioned test into
	// this method when it's implemented in the API
	return m, nil
}

func (a *MachineAgent) Tag() string {
	return state.MachineTag(a.MachineId)
}
<|endoftext|>


<commit_before>// Command metricsd provides a daemon for collecting latencies and other
// metrics from cored and uploading them to librato.
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/codahale/hdrhistogram"

	"chain/env"
	"chain/log"
	"chain/metrics"
	"chain/net/rpc"
)

const (
	reportTimeout  = 15 * time.Second
	libratoPostURL = "https://metrics-api.librato.com/v1/metrics"
	userAgent      = "chain-metricsd/0.1"
)

var (
	latencyMetricAttributes = attributes{
		Units:     "ms",
		Transform: "x/1000000",
		Summarize: "max",
	}
	period = int64(metrics.Period.Seconds())
)

var (
	coredAddr        = env.String("CORED_ADDR", "http://:1999")
	coredAccessToken = env.String("CORED_ACCESS_TOKEN", "")
	libratoUser      = env.String("LIBRATO_USER", "")
	libratoToken     = env.String("LIBRATO_TOKEN", "")
	metricPrefix     = env.String("METRIC_PREFIX", "cored")
)

func main() {
	env.Parse()

	ctx := context.Background()
	client := &rpc.Client{
		BaseURL:     *coredAddr,
		Username:    userAgent,
		AccessToken: *coredAccessToken,
	}

	// Ensure that we can access cored.
	err := client.Call(ctx, "/health", nil, nil)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}
	log.Messagef(ctx, "Successfully pinged cored at %s.", *coredAddr)

	// Periodically, report metrics.
	latestNumRots := make(map[string]int)
	ticker := time.NewTicker(metrics.Period)
	for {
		err := reportMetrics(ctx, client, latestNumRots)
		if err != nil {
			log.Error(ctx, err)
		}
		<-ticker.C
	}
}

func reportMetrics(ctx context.Context, client *rpc.Client, latestNumRots map[string]int) error {
	ctx, cancel := context.WithTimeout(ctx, reportTimeout)
	defer cancel()

	// Query cored for the latest metrics.
	var varsResp debugVarsResponse
	err := client.Call(ctx, "/debug/vars", nil, &varsResp)
	if err != nil {
		return err
	}

	// Convert the histograms into librato gauges.
	var req libratoMetricsRequest
	req.Source = varsResp.ProcessID
	for endpoint, latency := range varsResp.Latency {
		// figure out how many buckets have happened since we last
		// recorded data.
		latestRot := latestNumRots[endpoint]
		if latestRot == 0 {
			latestRot = 1
		}
		bucketCount := latency.NumRot - latestRot
		if bucketCount >= len(latency.Buckets) {
			bucketCount = len(latency.Buckets) - 1
		}
		latestNumRots[endpoint] = latency.NumRot

		for b := 1; b <= bucketCount; b++ {
			bucket := latency.Buckets[len(latency.Buckets)-1-b]
			h := hdrhistogram.Import(bucket.Histogram)

			cleanedEndpoint := strings.Replace(strings.Trim(endpoint, "/"), "/", "_", -1)
			name := *metricPrefix + ".rpc." + cleanedEndpoint

			req.Gauges = append(req.Gauges, gauge{
				Name:        name + ".latency.mean",
				Value:       int64(h.Mean()),
				Attr:        latencyMetricAttributes,
				Period:      period,
				MeasureTime: bucket.Timestamp,
			}, gauge{
				Name:        name + ".latency.p95",
				Value:       h.ValueAtQuantile(95.0),
				Attr:        latencyMetricAttributes,
				Period:      period,
				MeasureTime: bucket.Timestamp,
			}, gauge{
				Name:        name + ".latency.p99",
				Value:       h.ValueAtQuantile(99.0),
				Attr:        latencyMetricAttributes,
				Period:      period,
				MeasureTime: bucket.Timestamp,
			}, gauge{
				Name:        name + ".latency.p999",
				Value:       h.ValueAtQuantile(99.9),
				Attr:        latencyMetricAttributes,
				Period:      period,
				MeasureTime: bucket.Timestamp,
			}, gauge{
				Name:        name + ".latency.max",
				Value:       h.Max(),
				Attr:        latencyMetricAttributes,
				Period:      period,
				MeasureTime: bucket.Timestamp,
			})
		}
	}
	return sendLibratoMetrics(ctx, &req)
}

func sendLibratoMetrics(ctx context.Context, body *libratoMetricsRequest) error {
	if len(body.Gauges) == 0 {
		return nil
	}

	b, err := json.Marshal(body)
	if err != nil {
		return err
	}
	req, err := http.NewRequest("POST", libratoPostURL, bytes.NewReader(b))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", userAgent)
	req.SetBasicAuth(*libratoUser, *libratoToken)

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		// TODO(jackson): Retry automatically?
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("librato responded with %d", resp.StatusCode)
	}
	return nil
}

type libratoMetricsRequest struct {
	Source string  `json:"source,omitempty"`
	Gauges []gauge `json:"gauges,omitempty"`
}

type gauge struct {
	Name        string     `json:"name"`
	Value       int64      `json:"value"`
	Period      int64      `json:"period"`
	MeasureTime int64      `json:"measure_time"`
	Attr        attributes `json:"attributes"`
}

type attributes struct {
	Units     string `json:"display_units_long,omitempty"`
	Transform string `json:"display_transform,omitempty"`
	Min       int    `json:"display_min"`
	Summarize string `json:"summarize_function,omitempty"`
}

type debugVarsResponse struct {
	BuildCommit string               `json:"buildcommit"`
	BuildDate   string               `json:"builddate"`
	BuildTag    string               `json:"buildtag"`
	Latency     map[string]latencies `json:"latency"`
	ProcessID   string               `json:"processID"`
}

type latencies struct {
	NumRot  int             `json:"NumRot"`
	Buckets []latencyBucket `json:"Buckets"`
}

type latencyBucket struct {
	Over      uint64                 `json:"Over"`
	Timestamp int64                  `json:"Timestamp"`
	Histogram *hdrhistogram.Snapshot `json:"Histogram"`
}
<commit_msg>cmd/metricsd: publish memstats metrics<commit_after>// Command metricsd provides a daemon for collecting latencies and other
// metrics from cored and uploading them to librato.
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"runtime"
	"strings"
	"time"

	"github.com/codahale/hdrhistogram"

	"chain/env"
	"chain/log"
	"chain/metrics"
	"chain/net/rpc"
)

const (
	reportTimeout  = 15 * time.Second
	libratoPostURL = "https://metrics-api.librato.com/v1/metrics"
	userAgent      = "chain-metricsd/0.1"
)

var (
	latencyMetricAttributes = attributes{
		Units:     "ms",
		Transform: "x/1000000",
		Summarize: "max",
	}
	period = int64(metrics.Period.Seconds())
)

var (
	coredAddr        = env.String("CORED_ADDR", "http://:1999")
	coredAccessToken = env.String("CORED_ACCESS_TOKEN", "")
	libratoUser      = env.String("LIBRATO_USER", "")
	libratoToken     = env.String("LIBRATO_TOKEN", "")
	metricPrefix     = env.String("METRIC_PREFIX", "cored")
)

func main() {
	env.Parse()

	ctx := context.Background()
	client := &rpc.Client{
		BaseURL:     *coredAddr,
		Username:    userAgent,
		AccessToken: *coredAccessToken,
	}

	// Ensure that we can access cored.
	err := client.Call(ctx, "/health", nil, nil)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}
	log.Messagef(ctx, "Successfully pinged cored at %s.", *coredAddr)

	// Periodically, report metrics.
	latestNumRots := make(map[string]int)
	ticker := time.NewTicker(metrics.Period)
	for {
		err := reportMetrics(ctx, client, latestNumRots)
		if err != nil {
			log.Error(ctx, err)
		}
		<-ticker.C
	}
}

func reportMetrics(ctx context.Context, client *rpc.Client, latestNumRots map[string]int) error {
	ctx, cancel := context.WithTimeout(ctx, reportTimeout)
	defer cancel()

	// Query cored for the latest metrics.
	var varsResp debugVarsResponse
	err := client.Call(ctx, "/debug/vars", nil, &varsResp)
	if err != nil {
		return err
	}

	var req libratoMetricsRequest
	req.Source = varsResp.ProcessID
	req.MeasureTime = time.Now().Unix()

	// Add measurements from the runtime memstats.
	// See https://golang.org/pkg/runtime/#MemStats for full
	// documentation on the meaning of these metrics.
	memoryPrefix := *metricPrefix + ".memory."
	req.Gauges = append(req.Gauges, gauge{
		Name:   memoryPrefix + "total",
		Value:  int64(varsResp.Memstats.Alloc),
		Period: period,
		Attr: attributes{
			Units:     "MB",
			Transform: "x/1000000",
			Summarize: "max",
		},
	}, gauge{
		Name:   memoryPrefix + "heap.total",
		Value:  int64(varsResp.Memstats.HeapAlloc),
		Period: period,
		Attr: attributes{
			Units:     "MB",
			Transform: "x/1000000",
			Summarize: "max",
		},
	})
	req.Counters = append(req.Counters, counter{
		Name:  memoryPrefix + "mallocs",
		Value: int64(varsResp.Memstats.Mallocs),
	}, counter{
		Name:  memoryPrefix + "frees",
		Value: int64(varsResp.Memstats.Frees),
	}, counter{
		Name:  memoryPrefix + "gc.total_pause",
		Value: int64(varsResp.Memstats.PauseTotalNs),
		Attr: attributes{
			Units:     "ms",
			Transform: "x/1000000",
			Summarize: "max",
		},
	})

	// Convert the most recent latency histograms into librato gauges.
	for endpoint, latency := range varsResp.Latency {
		// figure out how many buckets have happened since we last
		// recorded data.
		latestRot := latestNumRots[endpoint]
		if latestRot == 0 {
			latestRot = 1
		}
		bucketCount := latency.NumRot - latestRot
		if bucketCount >= len(latency.Buckets) {
			bucketCount = len(latency.Buckets) - 1
		}
		latestNumRots[endpoint] = latency.NumRot

		for b := 1; b <= bucketCount; b++ {
			bucket := latency.Buckets[len(latency.Buckets)-1-b]
			h := hdrhistogram.Import(bucket.Histogram)

			cleanedEndpoint := strings.Replace(strings.Trim(endpoint, "/"), "/", "_", -1)
			name := *metricPrefix + ".rpc." + cleanedEndpoint

			req.Gauges = append(req.Gauges, gauge{
				Name:        name + ".latency.mean",
				Value:       int64(h.Mean()),
				Attr:        latencyMetricAttributes,
				Period:      period,
				MeasureTime: bucket.Timestamp,
			}, gauge{
				Name:        name + ".latency.p95",
				Value:       h.ValueAtQuantile(95.0),
				Attr:        latencyMetricAttributes,
				Period:      period,
				MeasureTime: bucket.Timestamp,
			}, gauge{
				Name:        name + ".latency.p99",
				Value:       h.ValueAtQuantile(99.0),
				Attr:        latencyMetricAttributes,
				Period:      period,
				MeasureTime: bucket.Timestamp,
			}, gauge{
				Name:        name + ".latency.p999",
				Value:       h.ValueAtQuantile(99.9),
				Attr:        latencyMetricAttributes,
				Period:      period,
				MeasureTime: bucket.Timestamp,
			}, gauge{
				Name:        name + ".latency.max",
				Value:       h.Max(),
				Attr:        latencyMetricAttributes,
				Period:      period,
				MeasureTime: bucket.Timestamp,
			})
		}
	}
	return sendLibratoMetrics(ctx, &req)
}

func sendLibratoMetrics(ctx context.Context, body *libratoMetricsRequest) error {
	b, err := json.Marshal(body)
	if err != nil {
		return err
	}
	req, err := http.NewRequest("POST", libratoPostURL, bytes.NewReader(b))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", userAgent)
	req.SetBasicAuth(*libratoUser, *libratoToken)

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		// TODO(jackson): Retry automatically?
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		errmsg, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		return fmt.Errorf("librato responded with %d:\n%s", resp.StatusCode, errmsg)
	}
	return nil
}

type libratoMetricsRequest struct {
	Source      string    `json:"source,omitempty"`
	MeasureTime int64     `json:"measure_time,omitempty"`
	Counters    []counter `json:"counters,omitempty"`
	Gauges      []gauge   `json:"gauges,omitempty"`
}

type gauge struct {
	Name        string     `json:"name"`
	Value       int64      `json:"value"`
	Period      int64      `json:"period"`
	MeasureTime int64      `json:"measure_time,omitempty"`
	Attr        attributes `json:"attributes"`
}

type counter struct {
	Name        string     `json:"name"`
	Value       int64      `json:"value"`
	MeasureTime int64      `json:"measure_time,omitempty"`
	Attr        attributes `json:"attributes"`
}

type attributes struct {
	Units     string `json:"display_units_long,omitempty"`
	Transform string `json:"display_transform,omitempty"`
	Min       int    `json:"display_min"`
	Summarize string `json:"summarize_function,omitempty"`
}

type debugVarsResponse struct {
	BuildCommit string               `json:"buildcommit"`
	BuildDate   string               `json:"builddate"`
	BuildTag    string               `json:"buildtag"`
	Latency     map[string]latencies `json:"latency"`
	Memstats    runtime.MemStats     `json:"memstats"`
	ProcessID   string               `json:"processID"`
}

type latencies struct {
	NumRot  int             `json:"NumRot"`
	Buckets []latencyBucket `json:"Buckets"`
}

type latencyBucket struct {
	Over      uint64                 `json:"Over"`
	Timestamp int64                  `json:"Timestamp"`
	Histogram *hdrhistogram.Snapshot `json:"Histogram"`
}
<|endoftext|>


<commit_before>package namespace

import (
	"fmt"

	"github.com/containerum/chkit/cmd/util"
	"github.com/containerum/chkit/pkg/model"
	"gopkg.in/urfave/cli.v2"
)

// GetNamespace -- commmand 'get' entity data
var GetNamespace = &cli.Command{
	Name:        "ns",
	Description: `show namespace or namespace list`,
	Usage:       `Shows namespace data or namespace list`,
	Action: func(ctx *cli.Context) error {
		client := util.GetClient(ctx)
		if ctx.NArg() > 0 {
			name := ctx.Args().First()
			ns, err := client.GetNamespace(name)
			if err != nil {
				return err
			}
			fmt.Println(ns.RenderTable())
		} else {
			list, err := client.GetNamespaceList()
			if err != nil {
				return err
			}
			fmt.Println(model.RenderTable(list))
		}
		return nil
	},
}
<commit_msg>add debug log<commit_after>package namespace

import (
	"fmt"

	"github.com/containerum/chkit/cmd/util"
	"github.com/containerum/chkit/pkg/model"
	"gopkg.in/urfave/cli.v2"
)

// GetNamespace -- commmand 'get' entity data
var GetNamespace = &cli.Command{
	Name:        "ns",
	Description: `show namespace or namespace list`,
	Usage:       `Shows namespace data or namespace list`,
	Action: func(ctx *cli.Context) error {
		log := util.GetLog(ctx)
		client := util.GetClient(ctx)
		log.Debugf("get ns from %q", client.APIaddr)
		if ctx.NArg() > 0 {
			name := ctx.Args().First()
			ns, err := client.GetNamespace(name)
			if err != nil {
				return err
			}
			fmt.Println(ns.RenderTable())
		} else {
			list, err := client.GetNamespaceList()
			if err != nil {
				return err
			}
			fmt.Println(model.RenderTable(list))
		}
		return nil
	},
}
<|endoftext|>


<commit_before>// PromHouse
// Copyright (C) 2017 Percona LLC
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/sirupsen/logrus"
	"gopkg.in/alecthomas/kingpin.v2"

	"github.com/Percona-Lab/PromHouse/prompb"
	"github.com/Percona-Lab/PromHouse/utils/duration"
)

type readProgress struct {
	current, max uint
}

type tsReader interface {
	readTS() ([]*prompb.TimeSeries, *readProgress, error)
}

type tsWriter interface {
	writeTS(ts []*prompb.TimeSeries) error
}

func parseArg(arg string) (string, string, error) {
	t := strings.Split(arg, ":")[0]
	switch t {
	case "file":
		f := strings.TrimPrefix(arg, t+":")
		return t, f, nil
	case "remote":
		u := strings.TrimPrefix(arg, t+":")
		if _, err := url.Parse(u); err != nil {
			return "", "", err
		}
		return t, u, nil
	default:
		return "", "", fmt.Errorf("unexpected type")
	}
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("stdlog: ")

	var (
		// remote:http://127.0.0.1:9090/api/v1/read for Prometheus
		sourceArg      = kingpin.Arg("source", "Read data from that source\n\tfile:data.bin for local file\n\tremote:http://127.0.0.1:7781/read for remote storage").Required().String()
		destinationArg = kingpin.Arg("destination", "Write data to that destination\n\tfile:data.bin for local file\n\tremote:http://127.0.0.1:7781/write for remote storage").Required().String()

		lastF = duration.FromFlag(kingpin.Flag("source.remote.last", "Remote source: read from that time ago").Default("30d"))
		stepF = duration.FromFlag(kingpin.Flag("source.remote.step", "Remote source: interval for a single request").Default("1m"))

		logLevelF = kingpin.Flag("log.level", "Log level").Default("info").String()
	)
	kingpin.CommandLine.HelpFlag.Short('h')
	kingpin.Parse()

	level, err := logrus.ParseLevel(*logLevelF)
	if err != nil {
		logrus.Fatal(err)
	}
	logrus.SetLevel(level)

	var reader tsReader
	var writer tsWriter

	{
		sourceType, sourceAddr, err := parseArg(*sourceArg)
		if err != nil {
			logrus.Fatalf("Failed to parse 'source' argument %s: %s.", *sourceArg, err)
		}
		switch sourceType {
		case "file":
			f, err := os.Open(sourceAddr)
			if err != nil {
				logrus.Fatal(err)
			}
			defer func() {
				if err = f.Close(); err != nil {
					logrus.Error(err)
				}
				logrus.Infof("%s closed.", f.Name())
			}()

			logrus.Infof("Reading metrics from %s %s.", sourceType, sourceAddr)
			reader = newFileClient(f)

		case "remote":
			end := time.Now().Truncate(time.Minute)
			start := end.Add(-time.Duration(*lastF))

			logrus.Infof("Reading metrics from %s %s between %s and %s with step %s.", sourceType, sourceAddr, start, end, *stepF)
			reader = newRemoteClient(sourceAddr, start, end, time.Duration(*stepF))

		default:
			panic("not reached")
		}
	}

	{
		destinationType, destinationAddr, err := parseArg(*destinationArg)
		if err != nil {
			logrus.Fatalf("Failed to parse 'destination' argument %s: %s.", *destinationArg, err)
		}
		switch destinationType {
		case "file":
			f, err := os.Create(destinationAddr)
			if err != nil {
				logrus.Fatal(err)
			}
			defer func() {
				if err = f.Close(); err != nil {
					logrus.Error(err)
				}
				logrus.Infof("%s closed.", f.Name())
			}()

			logrus.Infof("Writing metrics to %s %s.", destinationType, destinationAddr)
			writer = newFileClient(f)

		case "remote":
			logrus.Infof("Writing metrics to %s %s.", destinationType, destinationAddr)
			writer = newRemoteClient(destinationAddr, time.Time{}, time.Time{}, 0)

		default:
			panic("not reached")
		}
	}

	ch := make(chan []*prompb.TimeSeries, 100)
	var lastReport time.Time
	go func() {
		for {
			ts, rp, err := reader.readTS()
			if err != nil {
				if err != io.EOF {
					logrus.Errorf("Read error: %+v", err)
				}
				close(ch)
				return
			}

			if rp != nil && rp.max > 0 {
				if time.Since(lastReport) > 10*time.Second {
					lastReport = time.Now()
					logrus.Infof("Read %.2f%% (%d / %d), write buffer: %d / %d.",
						float64(rp.current*100)/float64(rp.max), rp.current, rp.max, len(ch), cap(ch))
				}
			}

			ch <- ts
		}
	}()

	for ts := range ch {
		if err := writer.writeTS(ts); err != nil {
			logrus.Errorf("Write error: %+v", err)
		}
	}
}
<commit_msg>Handle cancelation signals.<commit_after>// PromHouse
// Copyright (C) 2017 Percona LLC
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"context"
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/sirupsen/logrus"
	"golang.org/x/sys/unix"
	"gopkg.in/alecthomas/kingpin.v2"

	"github.com/Percona-Lab/PromHouse/prompb"
	"github.com/Percona-Lab/PromHouse/utils/duration"
)

type readProgress struct {
	current, max uint
}

type tsReader interface {
	readTS() ([]*prompb.TimeSeries, *readProgress, error)
}

type tsWriter interface {
	writeTS(ts []*prompb.TimeSeries) error
}

func parseArg(arg string) (string, string, error) {
	t := strings.Split(arg, ":")[0]
	switch t {
	case "file":
		f := strings.TrimPrefix(arg, t+":")
		return t, f, nil
	case "remote":
		u := strings.TrimPrefix(arg, t+":")
		if _, err := url.Parse(u); err != nil {
			return "", "", err
		}
		return t, u, nil
	default:
		return "", "", fmt.Errorf("unexpected type")
	}
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("stdlog: ")

	var (
		// remote:http://127.0.0.1:9090/api/v1/read for Prometheus
		sourceArg      = kingpin.Arg("source", "Read data from that source\n\tfile:data.bin for local file\n\tremote:http://127.0.0.1:7781/read for remote storage").Required().String()
		destinationArg = kingpin.Arg("destination", "Write data to that destination\n\tfile:data.bin for local file\n\tremote:http://127.0.0.1:7781/write for remote storage").Required().String()

		lastF = duration.FromFlag(kingpin.Flag("source.remote.last", "Remote source: read from that time ago").Default("30d"))
		stepF = duration.FromFlag(kingpin.Flag("source.remote.step", "Remote source: interval for a single request").Default("1m"))

		logLevelF = kingpin.Flag("log.level", "Log level").Default("info").String()
	)
	kingpin.CommandLine.HelpFlag.Short('h')
	kingpin.Parse()

	level, err := logrus.ParseLevel(*logLevelF)
	if err != nil {
		logrus.Fatal(err)
	}
	logrus.SetLevel(level)

	ctx, cancel := context.WithCancel(context.Background())
	defer logrus.Print("Done.")

	// handle termination signals: first one gracefully, force exit on the second one
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
	go func() {
		s := <-signals
		logrus.Warnf("Got %s, shutting down...", unix.SignalName(s.(syscall.Signal)))
		cancel()

		s = <-signals
		logrus.Panicf("Got %s, exiting!", s, unix.SignalName(s.(syscall.Signal)))
	}()

	var reader tsReader
	var writer tsWriter

	{
		sourceType, sourceAddr, err := parseArg(*sourceArg)
		if err != nil {
			logrus.Fatalf("Failed to parse 'source' argument %s: %s.", *sourceArg, err)
		}
		switch sourceType {
		case "file":
			f, err := os.Open(sourceAddr)
			if err != nil {
				logrus.Fatal(err)
			}
			defer func() {
				if err = f.Close(); err != nil {
					logrus.Error(err)
				}
				logrus.Infof("%s closed.", f.Name())
			}()

			logrus.Infof("Reading metrics from %s %s.", sourceType, sourceAddr)
			reader = newFileClient(f)

		case "remote":
			end := time.Now().Truncate(time.Minute)
			start := end.Add(-time.Duration(*lastF))

			logrus.Infof("Reading metrics from %s %s between %s and %s with step %s.", sourceType, sourceAddr, start, end, *stepF)
			reader = newRemoteClient(sourceAddr, start, end, time.Duration(*stepF))

		default:
			panic("not reached")
		}
	}

	{
		destinationType, destinationAddr, err := parseArg(*destinationArg)
		if err != nil {
			logrus.Fatalf("Failed to parse 'destination' argument %s: %s.", *destinationArg, err)
		}
		switch destinationType {
		case "file":
			f, err := os.Create(destinationAddr)
			if err != nil {
				logrus.Fatal(err)
			}
			defer func() {
				if err = f.Close(); err != nil {
					logrus.Error(err)
				}
				logrus.Infof("%s closed.", f.Name())
			}()

			logrus.Infof("Writing metrics to %s %s.", destinationType, destinationAddr)
			writer = newFileClient(f)

		case "remote":
			logrus.Infof("Writing metrics to %s %s.", destinationType, destinationAddr)
			writer = newRemoteClient(destinationAddr, time.Time{}, time.Time{}, 0)

		default:
			panic("not reached")
		}
	}

	ch := make(chan []*prompb.TimeSeries, 100)
	var lastReport time.Time
	go func() {
		for {
			ts, rp, err := reader.readTS()
			if err == nil {
				err = ctx.Err()
			}
			if err != nil {
				if err != io.EOF {
					logrus.Errorf("Read error: %+v", err)
				}
				close(ch)
				return
			}

			if rp != nil && rp.max > 0 {
				if time.Since(lastReport) > 10*time.Second {
					lastReport = time.Now()
					logrus.Infof("Read %.2f%% (%d / %d), write buffer: %d / %d.",
						float64(rp.current*100)/float64(rp.max), rp.current, rp.max, len(ch), cap(ch))
				}
			}

			ch <- ts
		}
	}()

	for ts := range ch {
		if err := writer.writeTS(ts); err != nil {
			logrus.Errorf("Write error: %+v", err)
		}
	}
}
<|endoftext|>


<commit_before>package scanner // import "github.com/amalog/go/scanner"

import (
	"fmt"
	"io"
)

type Class int

const (
	Atom  Class = iota
	Eof   Class = iota
	Num   Class = iota
	Punct Class = iota
	Var   Class = iota

	String Class = iota

	// classes used internally
	nl Class = iota
)

const eof rune = -1

type Position struct {
	Filename string
	Line     int
	Column   int
}

func (pos Position) String() string {
	f := pos.Filename
	if f == "" {
		f = "<input>"
	}
	return fmt.Sprintf("%s:%d:%d", f, pos.Line, pos.Column)
}

type SyntaxError struct {
	Position Position
	Message  string
}

func (err *SyntaxError) Error() string {
	return fmt.Sprintf("%s: %s", err.Position, err.Message)
}

type Token struct {
	Class    Class
	Position Position
	Text     string
}

func (t *Token) String() string {
	switch t.Class {
	case Atom:
		return fmt.Sprintf("atom(%s)", t.Text)
	case Eof:
		return "eof"
	case Num:
		return fmt.Sprintf("num(%s)", t.Text)
	case Punct:
		return fmt.Sprintf("punct(%s)", t.Text)
	case String:
		return fmt.Sprintf("string(%s)", t.Text)
	case Var:
		return fmt.Sprintf("var(%s)", t.Text)
	default:
		return "<unknown token class>"
	}
}

type Scanner struct {
	r io.RuneScanner

	filename string
	err      *SyntaxError

	line       int
	column     int
	prevLine   int // before calling next()
	prevColumn int // before calling next()

	prevToken *Token
}

func New(r io.RuneScanner) *Scanner {
	s := &Scanner{
		r:      r,
		line:   1,
		column: 0,
	}
	return s
}

// Scan returns the next token in the input.  Returns an infinite stream of Eof
// tokens once reaching the end of input.
func (s *Scanner) Scan() (*Token, error) {
	t, err := s.scan()
	if err != nil {
		s.prevToken = nil
		return nil, err
	}

	// handle comma insertion
	s.insertComma(t)
	if t.Class == nl { // newlines are for internal use only. skip them
		return s.Scan()
	}

	s.prevToken = t
	return t, err
}

func (s *Scanner) scan() (*Token, error) {
	var ch rune
	for {
		ch = s.next()
		if ch == eof {
			if s.err == nil {
				return &Token{Class: Eof, Position: s.Pos()}, nil
			}
			return nil, s.err
		}

		if ch >= 'a' && ch <= 'z' { // atom
			return s.scanAtom(ch)
		} else if ch >= 'A' && ch <= 'Z' { // variable
			return s.scanVariable(ch)
		} else if ch >= '0' && ch <= '9' { // number
			return s.scanNumber(ch)
		} else if ch == '"' { // string
			return s.scanString(ch)
		} else if ch == ' ' { // space
			s.skipSpace()
			continue
		} else if ch == '\n' { // newline
			return &Token{Class: nl, Position: s.Pos()}, nil
		} else {
			break
		}
	}

	t := &Token{
		Class:    Punct,
		Position: s.Pos(),
		Text:     string([]rune{ch}),
	}
	return t, nil
}

func (s *Scanner) insertComma(t *Token) {
	// don't insert comma as first token
	if s.prevToken == nil {
		return
	}

	// certain punctuation prohibits comma insertion
	if s.prevToken.Class == Punct {
		switch s.prevToken.Text {
		case ",", "(", "{":
			return
		}
	}

	switch t.Class {
	case Eof:
		if s.prevToken.Class != Eof { // before first EOF token
			t.Class = Punct
			t.Text = ","
		}
	case nl:
		t.Text = ","
		t.Class = Punct
	case Punct:
		if t.Text == ")" || t.Text == "}" { // before closing a seq or db
			t.Text = ","
			s.back()
		}
	}
}

func (s *Scanner) peek() rune {
	ch := s.next()
	if ch == eof {
		return ch
	}
	s.back()
	return ch
}

func (s *Scanner) back() {
	err := s.r.UnreadRune()
	if err != nil {
		panic(err)
	}

	s.line = s.prevLine
	s.column = s.prevColumn
}

// consumes the next character in the stream. stores errors in s.err
func (s *Scanner) next() rune {
	s.prevLine = s.line
	s.prevColumn = s.column

	ch, _, err := s.r.ReadRune()
	if err == io.EOF {
		return eof
	}
	if err != nil {
		panic(err)
	}

	// handle prohibited characters
	if ch == '\t' || ch == '\r' {
		s.err = s.prohibitedCharacter(ch)
		return eof
	}

	// update position information
	if ch == '\n' {
		s.line++
		s.column = 0
	} else {
		s.column++
	}

	return ch
}

func (s *Scanner) prohibitedCharacter(ch rune) *SyntaxError {
	var name string
	switch ch {
	case '\t':
		name = "tab"
	case '\r':
		name = "carriage return"
	}
	return &SyntaxError{
		Position: s.Pos(),
		Message:  fmt.Sprintf("The %s character is prohibited", name),
	}
}

func (s *Scanner) Pos() Position {
	return Position{
		Filename: s.filename,
		Line:     s.line,
		Column:   s.column,
	}
}

func (s *Scanner) skipSpace() {
	for {
		ch := s.next()
		if ch == ' ' {
			continue
		}
		if ch != eof {
			s.back()
		}
		break
	}
}

func (s *Scanner) scanAtom(ch rune) (*Token, error) {
	chars := make([]rune, 0)

	pos := s.Pos()
CH:
	for {
		switch {
		case ch >= 'a' && ch <= 'z', ch == '_':
			chars = append(chars, ch)
		case ch == eof:
			break CH
		default:
			s.back()
			break CH
		}

		ch = s.next()
	}

	t := &Token{
		Class:    Atom,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}

func (s *Scanner) scanVariable(ch rune) (*Token, error) {
	chars := make([]rune, 0)

	pos := s.Pos()
CH:
	for {
		switch {
		case ch >= 'A' && ch <= 'Z':
			if len(chars) > 0 {
				prev := chars[len(chars)-1]
				if prev < 'a' || prev > 'z' {
					err := &SyntaxError{
						Position: s.Pos(),
						Message:  fmt.Sprintf("variable names may not have consecutive uppercase letters, got %c", ch),
					}
					return nil, err
				}
			}
			chars = append(chars, ch)
		case ch >= 'a' && ch <= 'z':
			if len(chars) == 0 {
				panic("called scanVariable without upper case letter next in stream")
			}
			chars = append(chars, ch)
		case ch == eof:
			break CH
		default:
			s.back()
			break CH
		}

		ch = s.next()
	}

	t := &Token{
		Class:    Var,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}

func (s *Scanner) scanNumber(ch rune) (*Token, error) {
	chars := make([]rune, 0)

	pos := s.Pos()
CH:
	for {
		switch {
		case ch >= '0' && ch <= '9', ch == '_', ch == '.':
			chars = append(chars, ch)
		case ch == eof:
			break CH
		default:
			s.back()
			break CH
		}

		ch = s.next()
	}

	t := &Token{
		Class:    Num,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}

func (s *Scanner) scanString(ch rune) (*Token, error) {
	chars := make([]rune, 0)

	// consume opening quote character
	if ch != '"' {
		panic("scanString without a double quote character to start")
	}
	chars = append(chars, ch)

	ch = s.next()
	pos := s.Pos()
	for {
		chars = append(chars, ch)
		if ch == '"' {
			break
		}
		ch = s.next()
	}

	t := &Token{
		Class:    String,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}
<commit_msg>scanner: remove unused peek method<commit_after>package scanner // import "github.com/amalog/go/scanner"

import (
	"fmt"
	"io"
)

type Class int

const (
	Atom  Class = iota
	Eof   Class = iota
	Num   Class = iota
	Punct Class = iota
	Var   Class = iota

	String Class = iota

	// classes used internally
	nl Class = iota
)

const eof rune = -1

type Position struct {
	Filename string
	Line     int
	Column   int
}

func (pos Position) String() string {
	f := pos.Filename
	if f == "" {
		f = "<input>"
	}
	return fmt.Sprintf("%s:%d:%d", f, pos.Line, pos.Column)
}

type SyntaxError struct {
	Position Position
	Message  string
}

func (err *SyntaxError) Error() string {
	return fmt.Sprintf("%s: %s", err.Position, err.Message)
}

type Token struct {
	Class    Class
	Position Position
	Text     string
}

func (t *Token) String() string {
	switch t.Class {
	case Atom:
		return fmt.Sprintf("atom(%s)", t.Text)
	case Eof:
		return "eof"
	case Num:
		return fmt.Sprintf("num(%s)", t.Text)
	case Punct:
		return fmt.Sprintf("punct(%s)", t.Text)
	case String:
		return fmt.Sprintf("string(%s)", t.Text)
	case Var:
		return fmt.Sprintf("var(%s)", t.Text)
	default:
		return "<unknown token class>"
	}
}

type Scanner struct {
	r io.RuneScanner

	filename string
	err      *SyntaxError

	line       int
	column     int
	prevLine   int // before calling next()
	prevColumn int // before calling next()

	prevToken *Token
}

func New(r io.RuneScanner) *Scanner {
	s := &Scanner{
		r:      r,
		line:   1,
		column: 0,
	}
	return s
}

// Scan returns the next token in the input.  Returns an infinite stream of Eof
// tokens once reaching the end of input.
func (s *Scanner) Scan() (*Token, error) {
	t, err := s.scan()
	if err != nil {
		s.prevToken = nil
		return nil, err
	}

	// handle comma insertion
	s.insertComma(t)
	if t.Class == nl { // newlines are for internal use only. skip them
		return s.Scan()
	}

	s.prevToken = t
	return t, err
}

func (s *Scanner) scan() (*Token, error) {
	var ch rune
	for {
		ch = s.next()
		if ch == eof {
			if s.err == nil {
				return &Token{Class: Eof, Position: s.Pos()}, nil
			}
			return nil, s.err
		}

		if ch >= 'a' && ch <= 'z' { // atom
			return s.scanAtom(ch)
		} else if ch >= 'A' && ch <= 'Z' { // variable
			return s.scanVariable(ch)
		} else if ch >= '0' && ch <= '9' { // number
			return s.scanNumber(ch)
		} else if ch == '"' { // string
			return s.scanString(ch)
		} else if ch == ' ' { // space
			s.skipSpace()
			continue
		} else if ch == '\n' { // newline
			return &Token{Class: nl, Position: s.Pos()}, nil
		} else {
			break
		}
	}

	t := &Token{
		Class:    Punct,
		Position: s.Pos(),
		Text:     string([]rune{ch}),
	}
	return t, nil
}

func (s *Scanner) insertComma(t *Token) {
	// don't insert comma as first token
	if s.prevToken == nil {
		return
	}

	// certain punctuation prohibits comma insertion
	if s.prevToken.Class == Punct {
		switch s.prevToken.Text {
		case ",", "(", "{":
			return
		}
	}

	switch t.Class {
	case Eof:
		if s.prevToken.Class != Eof { // before first EOF token
			t.Class = Punct
			t.Text = ","
		}
	case nl:
		t.Text = ","
		t.Class = Punct
	case Punct:
		if t.Text == ")" || t.Text == "}" { // before closing a seq or db
			t.Text = ","
			s.back()
		}
	}
}

func (s *Scanner) back() {
	err := s.r.UnreadRune()
	if err != nil {
		panic(err)
	}

	s.line = s.prevLine
	s.column = s.prevColumn
}

// consumes the next character in the stream. stores errors in s.err
func (s *Scanner) next() rune {
	s.prevLine = s.line
	s.prevColumn = s.column

	ch, _, err := s.r.ReadRune()
	if err == io.EOF {
		return eof
	}
	if err != nil {
		panic(err)
	}

	// handle prohibited characters
	if ch == '\t' || ch == '\r' {
		s.err = s.prohibitedCharacter(ch)
		return eof
	}

	// update position information
	if ch == '\n' {
		s.line++
		s.column = 0
	} else {
		s.column++
	}

	return ch
}

func (s *Scanner) prohibitedCharacter(ch rune) *SyntaxError {
	var name string
	switch ch {
	case '\t':
		name = "tab"
	case '\r':
		name = "carriage return"
	}
	return &SyntaxError{
		Position: s.Pos(),
		Message:  fmt.Sprintf("The %s character is prohibited", name),
	}
}

func (s *Scanner) Pos() Position {
	return Position{
		Filename: s.filename,
		Line:     s.line,
		Column:   s.column,
	}
}

func (s *Scanner) skipSpace() {
	for {
		ch := s.next()
		if ch == ' ' {
			continue
		}
		if ch != eof {
			s.back()
		}
		break
	}
}

func (s *Scanner) scanAtom(ch rune) (*Token, error) {
	chars := make([]rune, 0)

	pos := s.Pos()
CH:
	for {
		switch {
		case ch >= 'a' && ch <= 'z', ch == '_':
			chars = append(chars, ch)
		case ch == eof:
			break CH
		default:
			s.back()
			break CH
		}

		ch = s.next()
	}

	t := &Token{
		Class:    Atom,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}

func (s *Scanner) scanVariable(ch rune) (*Token, error) {
	chars := make([]rune, 0)

	pos := s.Pos()
CH:
	for {
		switch {
		case ch >= 'A' && ch <= 'Z':
			if len(chars) > 0 {
				prev := chars[len(chars)-1]
				if prev < 'a' || prev > 'z' {
					err := &SyntaxError{
						Position: s.Pos(),
						Message:  fmt.Sprintf("variable names may not have consecutive uppercase letters, got %c", ch),
					}
					return nil, err
				}
			}
			chars = append(chars, ch)
		case ch >= 'a' && ch <= 'z':
			if len(chars) == 0 {
				panic("called scanVariable without upper case letter next in stream")
			}
			chars = append(chars, ch)
		case ch == eof:
			break CH
		default:
			s.back()
			break CH
		}

		ch = s.next()
	}

	t := &Token{
		Class:    Var,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}

func (s *Scanner) scanNumber(ch rune) (*Token, error) {
	chars := make([]rune, 0)

	pos := s.Pos()
CH:
	for {
		switch {
		case ch >= '0' && ch <= '9', ch == '_', ch == '.':
			chars = append(chars, ch)
		case ch == eof:
			break CH
		default:
			s.back()
			break CH
		}

		ch = s.next()
	}

	t := &Token{
		Class:    Num,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}

func (s *Scanner) scanString(ch rune) (*Token, error) {
	chars := make([]rune, 0)

	// consume opening quote character
	if ch != '"' {
		panic("scanString without a double quote character to start")
	}
	chars = append(chars, ch)

	ch = s.next()
	pos := s.Pos()
	for {
		chars = append(chars, ch)
		if ch == '"' {
			break
		}
		ch = s.next()
	}

	t := &Token{
		Class:    String,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}
<|endoftext|>


<commit_before>package termite

import (
	"crypto"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sync"
)

// TODO
// - in-mem cache for have of files.
// - readdir of all files on startup.

// Content based addressing cache.
type ContentCache struct {
	dir string

	mutex         sync.Mutex
	cond          *sync.Cond
	hashPathMap   map[string]string
	faulting      map[string]bool
	inMemoryCache *LruCache

	memoryTries int
	memoryHits  int
}

// NewContentCache creates a content cache based in directory d.
// memorySize sets the maximum number of file contents to keep in
// memory.
func NewContentCache(d string) *ContentCache {
	if fi, _ := os.Lstat(d); fi == nil {
		err := os.MkdirAll(d, 0700)
		if err != nil {
			panic(err)
		}
	}

	c := &ContentCache{
		dir:           d,
		hashPathMap:   make(map[string]string),
		inMemoryCache: NewLruCache(1024),
		faulting:      make(map[string]bool),
	}
	c.cond = sync.NewCond(&c.mutex)
	return c
}

// SetMemoryCacheSize readjusts the size of the in-memory content
// cache.  Not thread safe.
func (me *ContentCache) SetMemoryCacheSize(fileCount int) {
	if fileCount == 0 {
		me.inMemoryCache = nil
		return
	}
	if me.inMemoryCache.Size() != fileCount {
		me.inMemoryCache = NewLruCache(fileCount)
	}
}

func (me *ContentCache) MemoryHitRate() float64 {
	if me.memoryTries == 0 {
		return 0.0
	}
	return float64(me.memoryHits) / float64(me.memoryTries)
}

func HashPath(dir string, md5 string) string {
	s := fmt.Sprintf("%x", md5)
	prefix := s[:2]
	name := s[2:]
	dst := filepath.Join(dir, prefix, name)
	prefixDir, _ := filepath.Split(dst)
	if err := os.MkdirAll(prefixDir, 0700); err != nil {
		log.Fatal("MkdirAll error:", err)
	}
	return dst
}

func (me *ContentCache) localPath(hash string) string {
	me.mutex.Lock()
	defer me.mutex.Unlock()

	return me.hashPathMap[hash]
}

func (me *ContentCache) HasHash(hash string) bool {
	me.mutex.Lock()
	defer me.mutex.Unlock()

	_, ok := me.hashPathMap[hash]
	if ok {
		return true
	}

	if me.inMemoryCache != nil {
		ok = me.inMemoryCache.Has(hash)
		if ok {
			return true
		}
	}

	p := HashPath(me.dir, hash)
	_, err := os.Lstat(p)
	return err == nil
}

func (me *ContentCache) ContentsIfLoaded(hash string) []byte {
	me.mutex.Lock()
	defer me.mutex.Unlock()
	for me.faulting[hash] {
		me.cond.Wait()
	}
	me.memoryTries++
	if me.inMemoryCache == nil {
		return nil
	}
	c := me.inMemoryCache.Get(hash)
	if c != nil {
		me.memoryHits++
		return c.([]byte)
	}
	return nil
}

func (me *ContentCache) Path(hash string) string {
	p := me.localPath(hash)
	if p != "" {
		return p
	}
	return HashPath(me.dir, hash)
}

func (me *ContentCache) NewHashWriter() *HashWriter {
	return NewHashWriter(me.dir, crypto.MD5)
}

type HashWriter struct {
	hasher hash.Hash
	dest   *os.File
}

func NewHashWriter(dir string, hashfunc crypto.Hash) *HashWriter {
	me := &HashWriter{}
	tmp, err := ioutil.TempFile(dir, ".md5temp")
	if err != nil {
		log.Panic("NewHashWriter: ", err)
	}

	me.dest = tmp
	me.hasher = hashfunc.New()
	return me
}

func (me *HashWriter) Sum() string {
	return string(me.hasher.Sum())
}

func (me *HashWriter) Write(p []byte) (n int, err error) {
	n, err = me.dest.Write(p)
	me.hasher.Write(p[:n])
	return n, err
}

func (me *HashWriter) WriteClose(p []byte) (err error) {
	_, err = me.Write(p)
	if err != nil {
		return err
	}
	err = me.Close()
	return err
}

func (me *HashWriter) CopyClose(input io.Reader, size int64) error {
	_, err := io.CopyN(me, input, size)
	if err != nil {
		return err
	}
	err = me.Close()
	return err
}

func (me *HashWriter) Close() error {
	me.dest.Chmod(0444)
	err := me.dest.Close()
	if err != nil {
		return err
	}
	src := me.dest.Name()
	dir, _ := filepath.Split(src)
	sum := me.Sum()
	sumpath := HashPath(dir, sum)

	log.Printf("saving hash %x\n", sum)
	err = os.Rename(src, sumpath)
	if err != nil {
		if fi, _ := os.Lstat(sumpath); fi == nil {
			log.Println("already have", sumpath)
			os.Remove(src)
		}
	}
	return err
}

const _BUFSIZE = 32 * 1024

func (me *ContentCache) DestructiveSavePath(path string) (md5 string, err error) {
	var f *os.File
	f, err = os.Open(path)
	if err != nil {
		return "", err
	}
	before, _ := f.Stat()
	defer f.Close()

	h := crypto.MD5.New()
	var content []byte
	if before.Size < _MEMORY_LIMIT {
		content, err = ioutil.ReadAll(f)
		if err != nil {
			return "", err
		}

		h.Write(content)
	} else {
		io.Copy(h, f)
	}

	s := string(h.Sum())
	if me.HasHash(s) {
		os.Remove(path)
		return s, nil
	}

	if content != nil && me.inMemoryCache != nil {
		me.mutex.Lock()
		me.inMemoryCache.Add(s, content)
		me.mutex.Unlock()
	}

	p := me.Path(s)
	err = os.Rename(path, p)
	if err != nil {
		if fi, _ := os.Lstat(p); fi != nil {
			os.Remove(p)
			return s, nil
		}
		return "", err
	}
	f.Chmod(0444)
	after, _ := f.Stat()
	if after.Mtime_ns != before.Mtime_ns || after.Size != before.Size {
		log.Fatal("File changed during save", OsFileInfo(*before), OsFileInfo(*after))
	}
	return s, nil
}

func (me *ContentCache) SavePath(path string) (md5 string) {
	f, err := os.Open(path)
	if err != nil {
		log.Println("SavePath:", err)
		return ""
	}
	defer f.Close()

	fi, _ := f.Stat()
	return me.SaveStream(f, fi.Size)
}

func (me *ContentCache) SaveImmutablePath(path string) (md5 string) {
	hasher := crypto.MD5.New()

	f, err := os.Open(path)
	if err != nil {
		log.Println("SaveImmutablePath:", err)
		return ""
	}
	defer f.Close()

	var content []byte
	fi, _ := f.Stat()
	if fi.Size < _MEMORY_LIMIT {
		content, err = ioutil.ReadAll(f)
		if err != nil {
			log.Println("ReadAll:", err)
			return ""
		}
		hasher.Write(content)
	} else {
		_, err = io.Copy(hasher, f)
	}

	if err != nil && err != io.EOF {
		log.Println("io.Copy:", err)
		return ""
	}

	md5 = string(hasher.Sum())
	me.mutex.Lock()
	defer me.mutex.Unlock()
	me.hashPathMap[md5] = path
	if content != nil && me.inMemoryCache != nil {
		me.inMemoryCache.Add(md5, content)
	}

	log.Printf("hashed %s to %x", path, md5)
	return md5
}

// FaultIn loads the data from disk into the memory cache.
func (me *ContentCache) FaultIn(hash string) {
	me.mutex.Lock()
	defer me.mutex.Unlock()
	for !me.inMemoryCache.Has(hash) && me.faulting[hash] {
		me.cond.Wait()
	}
	if me.inMemoryCache.Has(hash) {
		return
	}

	me.faulting[hash] = true
	me.mutex.Unlock()
	c, err := ioutil.ReadFile(me.Path(hash))
	me.mutex.Lock()
	if err != nil {
		log.Fatal("FaultIn:", err)
	}
	delete(me.faulting, hash)
	me.inMemoryCache.Add(hash, c)
	me.cond.Broadcast()
}

func (me *ContentCache) Save(content []byte) (md5 string) {
	return me.saveViaMemory(content)
}

func (me *ContentCache) saveViaMemory(content []byte) (md5 string) {
	writer := me.NewHashWriter()
	err := writer.WriteClose(content)
	if err != nil {
		log.Println("saveViaMemory:", err)
		return ""
	}
	hash := writer.Sum()

	if me.inMemoryCache != nil {
		me.mutex.Lock()
		defer me.mutex.Unlock()
		me.inMemoryCache.Add(hash, content)
	}
	return hash
}

const _MEMORY_LIMIT = 128 * 1024

func (me *ContentCache) SaveStream(input io.Reader, size int64) (md5 string) {
	if size < _MEMORY_LIMIT {
		b, err := ioutil.ReadAll(input)
		if int64(len(b)) != size {
			log.Panicf("SaveStream: short read: %v %v", len(b), err)
		}

		return me.saveViaMemory(b)
	}

	dup := me.NewHashWriter()
	err := dup.CopyClose(input, size)
	if err != nil {
		return ""
	}

	return dup.Sum()
}
<commit_msg>Make hash switchable.<commit_after>package termite

import (
	"crypto"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sync"
)

// TODO
// - in-mem cache for have of files.
// - readdir of all files on startup.

// Content based addressing cache.
type ContentCache struct {
	dir string

	hashFunc      crypto.Hash
	mutex         sync.Mutex
	cond          *sync.Cond
	hashPathMap   map[string]string
	faulting      map[string]bool
	inMemoryCache *LruCache

	memoryTries int
	memoryHits  int
}

// NewContentCache creates a content cache based in directory d.
// memorySize sets the maximum number of file contents to keep in
// memory.
func NewContentCache(d string) *ContentCache {
	if fi, _ := os.Lstat(d); fi == nil {
		err := os.MkdirAll(d, 0700)
		if err != nil {
			panic(err)
		}
	}

	c := &ContentCache{
		dir:           d,
		hashPathMap:   make(map[string]string),
		inMemoryCache: NewLruCache(1024),
		faulting:      make(map[string]bool),
		hashFunc:      crypto.MD5,
	}
	c.cond = sync.NewCond(&c.mutex)
	return c
}

// SetMemoryCacheSize readjusts the size of the in-memory content
// cache.  Not thread safe.
func (me *ContentCache) SetMemoryCacheSize(fileCount int) {
	if fileCount == 0 {
		me.inMemoryCache = nil
		return
	}
	if me.inMemoryCache.Size() != fileCount {
		me.inMemoryCache = NewLruCache(fileCount)
	}
}

func (me *ContentCache) MemoryHitRate() float64 {
	if me.memoryTries == 0 {
		return 0.0
	}
	return float64(me.memoryHits) / float64(me.memoryTries)
}

func HashPath(dir string, hash string) string {
	s := fmt.Sprintf("%x", hash)
	prefix := s[:2]
	name := s[2:]
	dst := filepath.Join(dir, prefix, name)
	prefixDir, _ := filepath.Split(dst)
	if err := os.MkdirAll(prefixDir, 0700); err != nil {
		log.Fatal("MkdirAll error:", err)
	}
	return dst
}

func (me *ContentCache) localPath(hash string) string {
	me.mutex.Lock()
	defer me.mutex.Unlock()

	return me.hashPathMap[hash]
}

func (me *ContentCache) HasHash(hash string) bool {
	me.mutex.Lock()
	defer me.mutex.Unlock()

	_, ok := me.hashPathMap[hash]
	if ok {
		return true
	}

	if me.inMemoryCache != nil {
		ok = me.inMemoryCache.Has(hash)
		if ok {
			return true
		}
	}

	p := HashPath(me.dir, hash)
	_, err := os.Lstat(p)
	return err == nil
}

func (me *ContentCache) ContentsIfLoaded(hash string) []byte {
	me.mutex.Lock()
	defer me.mutex.Unlock()
	for me.faulting[hash] {
		me.cond.Wait()
	}
	me.memoryTries++
	if me.inMemoryCache == nil {
		return nil
	}
	c := me.inMemoryCache.Get(hash)
	if c != nil {
		me.memoryHits++
		return c.([]byte)
	}
	return nil
}

func (me *ContentCache) Path(hash string) string {
	p := me.localPath(hash)
	if p != "" {
		return p
	}
	return HashPath(me.dir, hash)
}

func (me *ContentCache) NewHashWriter() *HashWriter {
	return NewHashWriter(me.dir, me.hashFunc)
}

type HashWriter struct {
	hasher hash.Hash
	dest   *os.File
}

func NewHashWriter(dir string, hashfunc crypto.Hash) *HashWriter {
	me := &HashWriter{}
	tmp, err := ioutil.TempFile(dir, ".hashtemp")
	if err != nil {
		log.Panic("NewHashWriter: ", err)
	}

	me.dest = tmp
	me.hasher = hashfunc.New()
	return me
}

func (me *HashWriter) Sum() string {
	return string(me.hasher.Sum())
}

func (me *HashWriter) Write(p []byte) (n int, err error) {
	n, err = me.dest.Write(p)
	me.hasher.Write(p[:n])
	return n, err
}

func (me *HashWriter) WriteClose(p []byte) (err error) {
	_, err = me.Write(p)
	if err != nil {
		return err
	}
	err = me.Close()
	return err
}

func (me *HashWriter) CopyClose(input io.Reader, size int64) error {
	_, err := io.CopyN(me, input, size)
	if err != nil {
		return err
	}
	err = me.Close()
	return err
}

func (me *HashWriter) Close() error {
	me.dest.Chmod(0444)
	err := me.dest.Close()
	if err != nil {
		return err
	}
	src := me.dest.Name()
	dir, _ := filepath.Split(src)
	sum := me.Sum()
	sumpath := HashPath(dir, sum)

	log.Printf("saving hash %x\n", sum)
	err = os.Rename(src, sumpath)
	if err != nil {
		if fi, _ := os.Lstat(sumpath); fi == nil {
			log.Println("already have", sumpath)
			os.Remove(src)
		}
	}
	return err
}

const _BUFSIZE = 32 * 1024

func (me *ContentCache) DestructiveSavePath(path string) (hash string, err error) {
	var f *os.File
	f, err = os.Open(path)
	if err != nil {
		return "", err
	}
	before, _ := f.Stat()
	defer f.Close()

	h := me.hashFunc.New()

	var content []byte
	if before.Size < _MEMORY_LIMIT {
		content, err = ioutil.ReadAll(f)
		if err != nil {
			return "", err
		}

		h.Write(content)
	} else {
		io.Copy(h, f)
	}

	s := string(h.Sum())
	if me.HasHash(s) {
		os.Remove(path)
		return s, nil
	}

	if content != nil && me.inMemoryCache != nil {
		me.mutex.Lock()
		me.inMemoryCache.Add(s, content)
		me.mutex.Unlock()
	}

	p := me.Path(s)
	err = os.Rename(path, p)
	if err != nil {
		if fi, _ := os.Lstat(p); fi != nil {
			os.Remove(p)
			return s, nil
		}
		return "", err
	}
	f.Chmod(0444)
	after, _ := f.Stat()
	if after.Mtime_ns != before.Mtime_ns || after.Size != before.Size {
		log.Fatal("File changed during save", OsFileInfo(*before), OsFileInfo(*after))
	}
	return s, nil
}

func (me *ContentCache) SavePath(path string) (hash string) {
	f, err := os.Open(path)
	if err != nil {
		log.Println("SavePath:", err)
		return ""
	}
	defer f.Close()

	fi, _ := f.Stat()
	return me.SaveStream(f, fi.Size)
}

func (me *ContentCache) SaveImmutablePath(path string) (hash string) {
	hasher := me.hashFunc.New()

	f, err := os.Open(path)
	if err != nil {
		log.Println("SaveImmutablePath:", err)
		return ""
	}
	defer f.Close()

	var content []byte
	fi, _ := f.Stat()
	if fi.Size < _MEMORY_LIMIT {
		content, err = ioutil.ReadAll(f)
		if err != nil {
			log.Println("ReadAll:", err)
			return ""
		}
		hasher.Write(content)
	} else {
		_, err = io.Copy(hasher, f)
	}

	if err != nil && err != io.EOF {
		log.Println("io.Copy:", err)
		return ""
	}

	hash = string(hasher.Sum())
	me.mutex.Lock()
	defer me.mutex.Unlock()
	me.hashPathMap[hash] = path
	if content != nil && me.inMemoryCache != nil {
		me.inMemoryCache.Add(hash, content)
	}

	log.Printf("hashed %s to %x", path, hash)
	return hash
}

// FaultIn loads the data from disk into the memory cache.
func (me *ContentCache) FaultIn(hash string) {
	me.mutex.Lock()
	defer me.mutex.Unlock()
	for !me.inMemoryCache.Has(hash) && me.faulting[hash] {
		me.cond.Wait()
	}
	if me.inMemoryCache.Has(hash) {
		return
	}

	me.faulting[hash] = true
	me.mutex.Unlock()
	c, err := ioutil.ReadFile(me.Path(hash))
	me.mutex.Lock()
	if err != nil {
		log.Fatal("FaultIn:", err)
	}
	delete(me.faulting, hash)
	me.inMemoryCache.Add(hash, c)
	me.cond.Broadcast()
}

func (me *ContentCache) Save(content []byte) (hash string) {
	return me.saveViaMemory(content)
}

func (me *ContentCache) saveViaMemory(content []byte) (hash string) {
	writer := me.NewHashWriter()
	err := writer.WriteClose(content)
	if err != nil {
		log.Println("saveViaMemory:", err)
		return ""
	}
	hash = writer.Sum()

	if me.inMemoryCache != nil {
		me.mutex.Lock()
		defer me.mutex.Unlock()
		me.inMemoryCache.Add(hash, content)
	}
	return hash
}

const _MEMORY_LIMIT = 128 * 1024

func (me *ContentCache) SaveStream(input io.Reader, size int64) (hash string) {
	if size < _MEMORY_LIMIT {
		b, err := ioutil.ReadAll(input)
		if int64(len(b)) != size {
			log.Panicf("SaveStream: short read: %v %v", len(b), err)
		}

		return me.saveViaMemory(b)
	}

	dup := me.NewHashWriter()
	err := dup.CopyClose(input, size)
	if err != nil {
		return ""
	}

	return dup.Sum()
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cloud

import (
	"fmt"

	"github.com/golang/glog"
	clusterv1 "k8s.io/kube-deploy/cluster-api/api/cluster/v1alpha1"
	"k8s.io/kube-deploy/cluster-api/cloud/google"
)

// An actuator that just logs instead of doing anything.
type loggingMachineActuator struct{}

const config = `
apiVersion: v1
kind: config
preferences: {}
`

func NewMachineActuator(cloud string, kubeadmToken string, masterIP string) (MachineActuator, error) {
	switch cloud {
	case "google":
		return google.NewMachineActuator(kubeadmToken, masterIP)
	case "test", "aws", "azure":
		return &loggingMachineActuator{}, nil
	default:
		return nil, fmt.Errorf("Not recognized cloud provider: %s\n", cloud)
	}
}

func (a loggingMachineActuator) Create(machine *clusterv1.Machine) error {
	glog.Infof("actuator received create: %s\n", machine.ObjectMeta.Name)
	return nil
}

func (a loggingMachineActuator) Delete(machine *clusterv1.Machine) error {
	glog.Infof("actuator received delete: %s\n", machine.ObjectMeta.Name)
	return nil

}

func (a loggingMachineActuator) Get(name string) (*clusterv1.Machine, error) {
	glog.Infof("actuator received get %s\n", name)
	return &clusterv1.Machine{}, nil
}

func (a loggingMachineActuator) GetIP(machine *clusterv1.Machine) (string, error) {
	glog.Infof("actuator received GetIP: %s\n", machine.ObjectMeta.Name)
	return "0.0.0.0", nil
}

func (a loggingMachineActuator) GetKubeConfig(master *clusterv1.Machine) (string, error) {
	glog.Infof("actuator received GetKubeConfig: %s\n", master.ObjectMeta.Name)
	return config, nil
}

func (a loggingMachineActuator) CreateMachineController(machines []*clusterv1.Machine) error {
	glog.Infof("actuator received CreateMachineController: %q\n", machines)
	return nil
}

func (a loggingMachineActuator) PostDelete(machines []*clusterv1.Machine) error {
	glog.Infof("actuator received PostDelete: %q\n", machines)
	return nil
}
<commit_msg>Fix log actuator to include update command<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cloud

import (
	"fmt"

	"github.com/golang/glog"
	clusterv1 "k8s.io/kube-deploy/cluster-api/api/cluster/v1alpha1"
	"k8s.io/kube-deploy/cluster-api/cloud/google"
)

// An actuator that just logs instead of doing anything.
type loggingMachineActuator struct{}

const config = `
apiVersion: v1
kind: config
preferences: {}
`

func NewMachineActuator(cloud string, kubeadmToken string, masterIP string) (MachineActuator, error) {
	switch cloud {
	case "google":
		return google.NewMachineActuator(kubeadmToken, masterIP)
	case "test", "aws", "azure":
		return &loggingMachineActuator{}, nil
	default:
		return nil, fmt.Errorf("Not recognized cloud provider: %s\n", cloud)
	}
}

func (a loggingMachineActuator) Create(machine *clusterv1.Machine) error {
	glog.Infof("actuator received create: %s\n", machine.ObjectMeta.Name)
	return nil
}

func (a loggingMachineActuator) Delete(machine *clusterv1.Machine) error {
	glog.Infof("actuator received delete: %s\n", machine.ObjectMeta.Name)
	return nil
}

func (a loggingMachineActuator) Update(oldMachine *clusterv1.Machine, newMachine *clusterv1.Machine) error {
	glog.Infof("actuator received update: %s\n", oldMachine.ObjectMeta.Name)
	return nil
}

func (a loggingMachineActuator) Get(name string) (*clusterv1.Machine, error) {
	glog.Infof("actuator received get %s\n", name)
	return &clusterv1.Machine{}, nil
}

func (a loggingMachineActuator) GetIP(machine *clusterv1.Machine) (string, error) {
	glog.Infof("actuator received GetIP: %s\n", machine.ObjectMeta.Name)
	return "0.0.0.0", nil
}

func (a loggingMachineActuator) GetKubeConfig(master *clusterv1.Machine) (string, error) {
	glog.Infof("actuator received GetKubeConfig: %s\n", master.ObjectMeta.Name)
	return config, nil
}

func (a loggingMachineActuator) CreateMachineController(machines []*clusterv1.Machine) error {
	glog.Infof("actuator received CreateMachineController: %q\n", machines)
	return nil
}

func (a loggingMachineActuator) PostDelete(machines []*clusterv1.Machine) error {
	glog.Infof("actuator received PostDelete: %q\n", machines)
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// A basic integration test for the service.
// Assumes that there is a pre-existing etcd server running on localhost.
package main

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"reflect"
	"runtime"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/controller"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/config"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/wait"
	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler"
	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/factory"

	"github.com/coreos/go-etcd/etcd"
	"github.com/golang/glog"
)

var (
	fakeDocker1, fakeDocker2 kubelet.FakeDockerClient
)

type fakePodInfoGetter struct{}

func (fakePodInfoGetter) GetPodInfo(host, podID string) (api.PodInfo, error) {
	// This is a horrible hack to get around the fact that we can't provide
	// different port numbers per kubelet...
	var c client.PodInfoGetter
	switch host {
	case "localhost":
		c = &client.HTTPPodInfoGetter{
			Client: http.DefaultClient,
			Port:   10250,
		}
	case "machine":
		c = &client.HTTPPodInfoGetter{
			Client: http.DefaultClient,
			Port:   10251,
		}
	default:
		glog.Fatalf("Can't get info for: '%v', '%v'", host, podID)
	}
	return c.GetPodInfo("localhost", podID)
}

type delegateHandler struct {
	delegate http.Handler
}

func (h *delegateHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	if h.delegate != nil {
		h.delegate.ServeHTTP(w, req)
		return
	}
	w.WriteHeader(http.StatusNotFound)
}

func startComponents(manifestURL string) (apiServerURL string) {
	// Setup
	servers := []string{"http://localhost:4001"}
	glog.Infof("Creating etcd client pointing to %v", servers)
	machineList := []string{"localhost", "machine"}

	handler := delegateHandler{}
	apiServer := httptest.NewServer(&handler)

	etcdClient := etcd.NewClient(servers)

	cl := client.New(apiServer.URL, nil)
	cl.PollPeriod = time.Second * 1
	cl.Sync = true

	// Master
	m := master.New(&master.Config{
		Client:        cl,
		EtcdServers:   servers,
		Minions:       machineList,
		PodInfoGetter: fakePodInfoGetter{},
	})
	storage, codec := m.API_v1beta1()
	handler.delegate = apiserver.Handle(storage, codec, "/api/v1beta1")

	// Scheduler
	scheduler.New((&factory.ConfigFactory{cl}).Create()).Run()

	controllerManager := controller.NewReplicationManager(cl)

	// Prove that controllerManager's watch works by making it not sync until after this
	// test is over. (Hopefully we don't take 10 minutes!)
	controllerManager.Run(10 * time.Minute)

	// Kubelet (localhost)
	cfg1 := config.NewPodConfig(config.PodConfigNotificationSnapshotAndUpdates)
	config.NewSourceEtcd(config.EtcdKeyForHost(machineList[0]), etcdClient, cfg1.Channel("etcd"))
	config.NewSourceURL(manifestURL, 5*time.Second, cfg1.Channel("url"))
	myKubelet := kubelet.NewIntegrationTestKubelet(machineList[0], &fakeDocker1)
	go util.Forever(func() { myKubelet.Run(cfg1.Updates()) }, 0)
	go util.Forever(func() {
		kubelet.ListenAndServeKubeletServer(myKubelet, cfg1.Channel("http"), "localhost", 10250)
	}, 0)

	// Kubelet (machine)
	// Create a second kubelet so that the guestbook example's two redis slaves both
	// have a place they can schedule.
	cfg2 := config.NewPodConfig(config.PodConfigNotificationSnapshotAndUpdates)
	config.NewSourceEtcd(config.EtcdKeyForHost(machineList[1]), etcdClient, cfg2.Channel("etcd"))
	otherKubelet := kubelet.NewIntegrationTestKubelet(machineList[1], &fakeDocker2)
	go util.Forever(func() { otherKubelet.Run(cfg2.Updates()) }, 0)
	go util.Forever(func() {
		kubelet.ListenAndServeKubeletServer(otherKubelet, cfg2.Channel("http"), "localhost", 10251)
	}, 0)

	return apiServer.URL
}

// podsOnMinions returns true when all of the selected pods exist on a minion.
func podsOnMinions(c *client.Client, pods api.PodList) wait.ConditionFunc {
	podInfo := fakePodInfoGetter{}
	return func() (bool, error) {
		for i := range pods.Items {
			host, id := pods.Items[i].CurrentState.Host, pods.Items[i].ID
			if len(host) == 0 {
				return false, nil
			}
			if _, err := podInfo.GetPodInfo(host, id); err != nil {
				return false, nil
			}
		}
		return true, nil
	}
}

func runReplicationControllerTest(c *client.Client) {
	data, err := ioutil.ReadFile("api/examples/controller.json")
	if err != nil {
		glog.Fatalf("Unexpected error: %#v", err)
	}
	var controllerRequest api.ReplicationController
	if err := json.Unmarshal(data, &controllerRequest); err != nil {
		glog.Fatalf("Unexpected error: %#v", err)
	}

	glog.Infof("Creating replication controllers")
	if _, err := c.CreateReplicationController(controllerRequest); err != nil {
		glog.Fatalf("Unexpected error: %#v", err)
	}
	glog.Infof("Done creating replication controllers")

	// Give the controllers some time to actually create the pods
	if err := wait.Poll(time.Second, time.Second*10, c.ControllerHasDesiredReplicas(controllerRequest)); err != nil {
		glog.Fatalf("FAILED: pods never created %v", err)
	}

	// wait for minions to indicate they have info about the desired pods
	pods, err := c.ListPods(labels.Set(controllerRequest.DesiredState.ReplicaSelector).AsSelector())
	if err != nil {
		glog.Fatalf("FAILED: unable to get pods to list: %v", err)
	}
	if err := wait.Poll(time.Second, time.Second*10, podsOnMinions(c, pods)); err != nil {
		glog.Fatalf("FAILED: pods never started running %v", err)
	}

	glog.Infof("Pods created")
}

func runAtomicPutTest(c *client.Client) {
	var svc api.Service
	err := c.Post().Path("services").Body(
		api.Service{
			JSONBase: api.JSONBase{ID: "atomicservice", APIVersion: "v1beta1"},
			Port:     12345,
			Labels: map[string]string{
				"name": "atomicService",
			},
			// This is here because validation requires it.
			Selector: map[string]string{
				"foo": "bar",
			},
		},
	).Do().Into(&svc)
	if err != nil {
		glog.Fatalf("Failed creating atomicService: %v", err)
	}
	glog.Info("Created atomicService")
	testLabels := labels.Set{
		"foo": "bar",
	}
	for i := 0; i < 5; i++ {
		// a: z, b: y, etc...
		testLabels[string([]byte{byte('a' + i)})] = string([]byte{byte('z' - i)})
	}
	var wg sync.WaitGroup
	wg.Add(len(testLabels))
	for label, value := range testLabels {
		go func(l, v string) {
			for {
				glog.Infof("Starting to update (%s, %s)", l, v)
				var tmpSvc api.Service
				err := c.Get().
					Path("services").
					Path(svc.ID).
					PollPeriod(100 * time.Millisecond).
					Do().
					Into(&tmpSvc)
				if err != nil {
					glog.Errorf("Error getting atomicService: %v", err)
					continue
				}
				if tmpSvc.Selector == nil {
					tmpSvc.Selector = map[string]string{l: v}
				} else {
					tmpSvc.Selector[l] = v
				}
				glog.Infof("Posting update (%s, %s)", l, v)
				err = c.Put().Path("services").Path(svc.ID).Body(&tmpSvc).Do().Error()
				if err != nil {
					if se, ok := err.(*client.StatusErr); ok {
						if se.Status.Code == http.StatusConflict {
							glog.Infof("Conflict: (%s, %s)", l, v)
							// This is what we expect.
							continue
						}
					}
					glog.Errorf("Unexpected error putting atomicService: %v", err)
					continue
				}
				break
			}
			glog.Infof("Done update (%s, %s)", l, v)
			wg.Done()
		}(label, value)
	}
	wg.Wait()
	if err := c.Get().Path("services").Path(svc.ID).Do().Into(&svc); err != nil {
		glog.Fatalf("Failed getting atomicService after writers are complete: %v", err)
	}
	if !reflect.DeepEqual(testLabels, labels.Set(svc.Selector)) {
		glog.Fatalf("Selector PUTs were not atomic: wanted %v, got %v", testLabels, svc.Selector)
	}
	glog.Info("Atomic PUTs work.")
}

type testFunc func(*client.Client)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	util.ReallyCrash = true
	util.InitLogs()
	defer util.FlushLogs()

	go func() {
		defer util.FlushLogs()
		time.Sleep(3 * time.Minute)
		glog.Fatalf("This test has timed out.")
	}()

	manifestURL := ServeCachedManifestFile()

	apiServerURL := startComponents(manifestURL)

	// Ok. we're good to go.
	glog.Infof("API Server started on %s", apiServerURL)
	// Wait for the synchronization threads to come up.
	time.Sleep(time.Second * 10)

	kubeClient := client.New(apiServerURL, nil)

	// Run tests in parallel
	testFuncs := []testFunc{
		runReplicationControllerTest,
		runAtomicPutTest,
	}
	var wg sync.WaitGroup
	wg.Add(len(testFuncs))
	for i := range testFuncs {
		f := testFuncs[i]
		go func() {
			f(kubeClient)
			wg.Done()
		}()
	}
	wg.Wait()

	// Check that kubelet tried to make the pods.
	// Using a set to list unique creation attempts. Our fake is
	// really stupid, so kubelet tries to create these multiple times.
	createdPods := util.StringSet{}
	for _, p := range fakeDocker1.Created {
		// The last 8 characters are random, so slice them off.
		if n := len(p); n > 8 {
			createdPods.Insert(p[:n-8])
		}
	}
	for _, p := range fakeDocker2.Created {
		// The last 8 characters are random, so slice them off.
		if n := len(p); n > 8 {
			createdPods.Insert(p[:n-8])
		}
	}
	// We expect 5: 2 net containers + 2 pods from the replication controller +
	//              1 net container + 2 pods from the URL.
	if len(createdPods) != 7 {
		glog.Fatalf("Unexpected list of created pods:\n\n%#v\n\n%#v\n\n%#v\n\n", createdPods.List(), fakeDocker1.Created, fakeDocker2.Created)
	}
	glog.Infof("OK - found created pods: %#v", createdPods.List())
}

// ServeCachedManifestFile serves a file for kubelet to read.
func ServeCachedManifestFile() (servingAddress string) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/manifest" {
			w.Write([]byte(testManifestFile))
			return
		}
		glog.Fatalf("Got request: %#v\n", r)
		http.NotFound(w, r)
	}))
	return server.URL + "/manifest"
}

const (
	// This is copied from, and should be kept in sync with:
	// https://raw.githubusercontent.com/GoogleCloudPlatform/container-vm-guestbook-redis-python/master/manifest.yaml
	testManifestFile = `version: v1beta2
id: container-vm-guestbook
containers:
  - name: redis
    image: dockerfile/redis
    volumeMounts:
      - name: redis-data
        mountPath: /data

  - name: guestbook
    image: google/guestbook-python-redis
    ports:
      - name: www
        hostPort: 80
        containerPort: 80

volumes:
  - name: redis-data`
)
<commit_msg>Delete all keys prior to running integration test<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// A basic integration test for the service.
// Assumes that there is a pre-existing etcd server running on localhost.
package main

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"reflect"
	"runtime"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/controller"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/config"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/wait"
	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler"
	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/factory"

	"github.com/coreos/go-etcd/etcd"
	"github.com/golang/glog"
)

var (
	fakeDocker1, fakeDocker2 kubelet.FakeDockerClient
)

type fakePodInfoGetter struct{}

func (fakePodInfoGetter) GetPodInfo(host, podID string) (api.PodInfo, error) {
	// This is a horrible hack to get around the fact that we can't provide
	// different port numbers per kubelet...
	var c client.PodInfoGetter
	switch host {
	case "localhost":
		c = &client.HTTPPodInfoGetter{
			Client: http.DefaultClient,
			Port:   10250,
		}
	case "machine":
		c = &client.HTTPPodInfoGetter{
			Client: http.DefaultClient,
			Port:   10251,
		}
	default:
		glog.Fatalf("Can't get info for: '%v', '%v'", host, podID)
	}
	return c.GetPodInfo("localhost", podID)
}

type delegateHandler struct {
	delegate http.Handler
}

func (h *delegateHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	if h.delegate != nil {
		h.delegate.ServeHTTP(w, req)
		return
	}
	w.WriteHeader(http.StatusNotFound)
}

func startComponents(manifestURL string) (apiServerURL string) {
	// Setup
	servers := []string{"http://localhost:4001"}
	glog.Infof("Creating etcd client pointing to %v", servers)
	machineList := []string{"localhost", "machine"}

	handler := delegateHandler{}
	apiServer := httptest.NewServer(&handler)

	etcdClient := etcd.NewClient(servers)
	keys, err := etcdClient.Get("/", false, false)
	if err != nil {
		glog.Fatalf("Unable to list root etcd keys: %v", err)
	}
	for _, node := range keys.Node.Nodes {
		if _, err := etcdClient.Delete(node.Key, true); err != nil {
			glog.Fatalf("Unable delete key: %v", err)
		}
	}

	cl := client.New(apiServer.URL, nil)
	cl.PollPeriod = time.Second * 1
	cl.Sync = true

	// Master
	m := master.New(&master.Config{
		Client:        cl,
		EtcdServers:   servers,
		Minions:       machineList,
		PodInfoGetter: fakePodInfoGetter{},
	})
	storage, codec := m.API_v1beta1()
	handler.delegate = apiserver.Handle(storage, codec, "/api/v1beta1")

	// Scheduler
	scheduler.New((&factory.ConfigFactory{cl}).Create()).Run()

	controllerManager := controller.NewReplicationManager(cl)

	// Prove that controllerManager's watch works by making it not sync until after this
	// test is over. (Hopefully we don't take 10 minutes!)
	controllerManager.Run(10 * time.Minute)

	// Kubelet (localhost)
	cfg1 := config.NewPodConfig(config.PodConfigNotificationSnapshotAndUpdates)
	config.NewSourceEtcd(config.EtcdKeyForHost(machineList[0]), etcdClient, cfg1.Channel("etcd"))
	config.NewSourceURL(manifestURL, 5*time.Second, cfg1.Channel("url"))
	myKubelet := kubelet.NewIntegrationTestKubelet(machineList[0], &fakeDocker1)
	go util.Forever(func() { myKubelet.Run(cfg1.Updates()) }, 0)
	go util.Forever(func() {
		kubelet.ListenAndServeKubeletServer(myKubelet, cfg1.Channel("http"), "localhost", 10250)
	}, 0)

	// Kubelet (machine)
	// Create a second kubelet so that the guestbook example's two redis slaves both
	// have a place they can schedule.
	cfg2 := config.NewPodConfig(config.PodConfigNotificationSnapshotAndUpdates)
	config.NewSourceEtcd(config.EtcdKeyForHost(machineList[1]), etcdClient, cfg2.Channel("etcd"))
	otherKubelet := kubelet.NewIntegrationTestKubelet(machineList[1], &fakeDocker2)
	go util.Forever(func() { otherKubelet.Run(cfg2.Updates()) }, 0)
	go util.Forever(func() {
		kubelet.ListenAndServeKubeletServer(otherKubelet, cfg2.Channel("http"), "localhost", 10251)
	}, 0)

	return apiServer.URL
}

// podsOnMinions returns true when all of the selected pods exist on a minion.
func podsOnMinions(c *client.Client, pods api.PodList) wait.ConditionFunc {
	podInfo := fakePodInfoGetter{}
	return func() (bool, error) {
		for i := range pods.Items {
			host, id := pods.Items[i].CurrentState.Host, pods.Items[i].ID
			if len(host) == 0 {
				return false, nil
			}
			if _, err := podInfo.GetPodInfo(host, id); err != nil {
				return false, nil
			}
		}
		return true, nil
	}
}

func runReplicationControllerTest(c *client.Client) {
	data, err := ioutil.ReadFile("api/examples/controller.json")
	if err != nil {
		glog.Fatalf("Unexpected error: %#v", err)
	}
	var controllerRequest api.ReplicationController
	if err := json.Unmarshal(data, &controllerRequest); err != nil {
		glog.Fatalf("Unexpected error: %#v", err)
	}

	glog.Infof("Creating replication controllers")
	if _, err := c.CreateReplicationController(controllerRequest); err != nil {
		glog.Fatalf("Unexpected error: %#v", err)
	}
	glog.Infof("Done creating replication controllers")

	// Give the controllers some time to actually create the pods
	if err := wait.Poll(time.Second, time.Second*10, c.ControllerHasDesiredReplicas(controllerRequest)); err != nil {
		glog.Fatalf("FAILED: pods never created %v", err)
	}

	// wait for minions to indicate they have info about the desired pods
	pods, err := c.ListPods(labels.Set(controllerRequest.DesiredState.ReplicaSelector).AsSelector())
	if err != nil {
		glog.Fatalf("FAILED: unable to get pods to list: %v", err)
	}
	if err := wait.Poll(time.Second, time.Second*10, podsOnMinions(c, pods)); err != nil {
		glog.Fatalf("FAILED: pods never started running %v", err)
	}

	glog.Infof("Pods created")
}

func runAtomicPutTest(c *client.Client) {
	var svc api.Service
	err := c.Post().Path("services").Body(
		api.Service{
			JSONBase: api.JSONBase{ID: "atomicservice", APIVersion: "v1beta1"},
			Port:     12345,
			Labels: map[string]string{
				"name": "atomicService",
			},
			// This is here because validation requires it.
			Selector: map[string]string{
				"foo": "bar",
			},
		},
	).Do().Into(&svc)
	if err != nil {
		glog.Fatalf("Failed creating atomicService: %v", err)
	}
	glog.Info("Created atomicService")
	testLabels := labels.Set{
		"foo": "bar",
	}
	for i := 0; i < 5; i++ {
		// a: z, b: y, etc...
		testLabels[string([]byte{byte('a' + i)})] = string([]byte{byte('z' - i)})
	}
	var wg sync.WaitGroup
	wg.Add(len(testLabels))
	for label, value := range testLabels {
		go func(l, v string) {
			for {
				glog.Infof("Starting to update (%s, %s)", l, v)
				var tmpSvc api.Service
				err := c.Get().
					Path("services").
					Path(svc.ID).
					PollPeriod(100 * time.Millisecond).
					Do().
					Into(&tmpSvc)
				if err != nil {
					glog.Errorf("Error getting atomicService: %v", err)
					continue
				}
				if tmpSvc.Selector == nil {
					tmpSvc.Selector = map[string]string{l: v}
				} else {
					tmpSvc.Selector[l] = v
				}
				glog.Infof("Posting update (%s, %s)", l, v)
				err = c.Put().Path("services").Path(svc.ID).Body(&tmpSvc).Do().Error()
				if err != nil {
					if se, ok := err.(*client.StatusErr); ok {
						if se.Status.Code == http.StatusConflict {
							glog.Infof("Conflict: (%s, %s)", l, v)
							// This is what we expect.
							continue
						}
					}
					glog.Errorf("Unexpected error putting atomicService: %v", err)
					continue
				}
				break
			}
			glog.Infof("Done update (%s, %s)", l, v)
			wg.Done()
		}(label, value)
	}
	wg.Wait()
	if err := c.Get().Path("services").Path(svc.ID).Do().Into(&svc); err != nil {
		glog.Fatalf("Failed getting atomicService after writers are complete: %v", err)
	}
	if !reflect.DeepEqual(testLabels, labels.Set(svc.Selector)) {
		glog.Fatalf("Selector PUTs were not atomic: wanted %v, got %v", testLabels, svc.Selector)
	}
	glog.Info("Atomic PUTs work.")
}

type testFunc func(*client.Client)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	util.ReallyCrash = true
	util.InitLogs()
	defer util.FlushLogs()

	go func() {
		defer util.FlushLogs()
		time.Sleep(3 * time.Minute)
		glog.Fatalf("This test has timed out.")
	}()

	manifestURL := ServeCachedManifestFile()

	apiServerURL := startComponents(manifestURL)

	// Ok. we're good to go.
	glog.Infof("API Server started on %s", apiServerURL)
	// Wait for the synchronization threads to come up.
	time.Sleep(time.Second * 10)

	kubeClient := client.New(apiServerURL, nil)

	// Run tests in parallel
	testFuncs := []testFunc{
		runReplicationControllerTest,
		runAtomicPutTest,
	}
	var wg sync.WaitGroup
	wg.Add(len(testFuncs))
	for i := range testFuncs {
		f := testFuncs[i]
		go func() {
			f(kubeClient)
			wg.Done()
		}()
	}
	wg.Wait()

	// Check that kubelet tried to make the pods.
	// Using a set to list unique creation attempts. Our fake is
	// really stupid, so kubelet tries to create these multiple times.
	createdPods := util.StringSet{}
	for _, p := range fakeDocker1.Created {
		// The last 8 characters are random, so slice them off.
		if n := len(p); n > 8 {
			createdPods.Insert(p[:n-8])
		}
	}
	for _, p := range fakeDocker2.Created {
		// The last 8 characters are random, so slice them off.
		if n := len(p); n > 8 {
			createdPods.Insert(p[:n-8])
		}
	}
	// We expect 5: 2 net containers + 2 pods from the replication controller +
	//              1 net container + 2 pods from the URL.
	if len(createdPods) != 7 {
		glog.Fatalf("Unexpected list of created pods:\n\n%#v\n\n%#v\n\n%#v\n\n", createdPods.List(), fakeDocker1.Created, fakeDocker2.Created)
	}
	glog.Infof("OK - found created pods: %#v", createdPods.List())
}

// ServeCachedManifestFile serves a file for kubelet to read.
func ServeCachedManifestFile() (servingAddress string) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/manifest" {
			w.Write([]byte(testManifestFile))
			return
		}
		glog.Fatalf("Got request: %#v\n", r)
		http.NotFound(w, r)
	}))
	return server.URL + "/manifest"
}

const (
	// This is copied from, and should be kept in sync with:
	// https://raw.githubusercontent.com/GoogleCloudPlatform/container-vm-guestbook-redis-python/master/manifest.yaml
	testManifestFile = `version: v1beta2
id: container-vm-guestbook
containers:
  - name: redis
    image: dockerfile/redis
    volumeMounts:
      - name: redis-data
        mountPath: /data

  - name: guestbook
    image: google/guestbook-python-redis
    ports:
      - name: www
        hostPort: 80
        containerPort: 80

volumes:
  - name: redis-data`
)
<|endoftext|>