<commit_before>package collector

import (
	"fullerite/config"
	"fullerite/metric"

	"strings"
	"time"

	l "github.com/Sirupsen/logrus"

	"github.com/fsouza/go-dockerclient"
)

const (
	mesosTaskID         = "MESOS_TASK_ID"
	endpoint            = "unix:///var/run/docker.sock"
	defaultStatsTimeout = 7
)

// DockerStats collector type.
// previousCPUValues contains the last cpu-usage values per container.
// dockerClient is the client for the Docker remote API.
type DockerStats struct {
	baseCollector
	previousCPUValues map[string]*CPUValues
	dockerClient      *docker.Client
	statsTimeout      int
}

// CPUValues struct contains the last cpu-usage values in order to compute properly the current values.
// (see calculateCPUPercent() for more details)
type CPUValues struct {
	totCPU, systemCPU uint64
}

// NewDockerStats creates a new DockerStats collector.
func NewDockerStats(channel chan metric.Metric, initialInterval int, log *l.Entry) *DockerStats {
	d := new(DockerStats)

	d.log = log
	d.channel = channel
	d.interval = initialInterval

	d.name = "DockerStats"
	d.previousCPUValues = make(map[string]*CPUValues)
	d.dockerClient, _ = docker.NewClient(endpoint)
	d.statsTimeout = defaultStatsTimeout

	return d
}

// Configure takes a dictionary of values with which the handler can configure itself.
func (d *DockerStats) Configure(configMap map[string]interface{}) {
	if timeout, exists := configMap["dockerStatsTimeout"]; exists {
		d.statsTimeout = config.GetAsInt(timeout, defaultStatsTimeout)
	}
	d.configureCommonParams(configMap)
}

// Collect iterates on all the docker containers alive and, if possible, collects the correspondent
// memory and cpu statistics.
// For each container a gorutine is started to spin up the collection process.
func (d DockerStats) Collect() {
	containers, err := d.dockerClient.ListContainers(docker.ListContainersOptions{All: false})
	if err != nil {
		d.log.Error("ListContainers() failed: ", err)
		return
	}
	for _, apiContainer := range containers {
		container, err := d.dockerClient.InspectContainer(apiContainer.ID)
		if err != nil {
			d.log.Error("InspectContainer() failed: ", err)
			continue
		}
		if _, ok := d.previousCPUValues[container.ID]; !ok {
			d.previousCPUValues[container.ID] = new(CPUValues)
		}
		go d.getDockerContainerInfo(container)
	}
}

// getDockerContainerInfo gets container statistics for the given container.
// results is a channel to make possible the synchronization between the main process and the gorutines (wait-notify pattern).
func (d DockerStats) getDockerContainerInfo(container *docker.Container) {
	errC := make(chan error, 1)
	statsC := make(chan *docker.Stats, 1)
	done := make(chan bool)

	go func() {
		errC <- d.dockerClient.Stats(docker.StatsOptions{container.ID, statsC, false, done, 0})
	}()
	select {
	case stats, ok := <-statsC:
		if !ok {
			err := <-errC
			d.log.Error("Failed to collect docker container stats: ", err)
			break
		}
		done <- true

		ret := d.buildMetrics(container, float64(stats.MemoryStats.Usage), float64(stats.MemoryStats.Limit), calculateCPUPercent(d.previousCPUValues[container.ID].totCPU, d.previousCPUValues[container.ID].systemCPU, stats))

		d.sendMetrics(ret)

		d.previousCPUValues[container.ID].totCPU = stats.CPUStats.CPUUsage.TotalUsage
		d.previousCPUValues[container.ID].systemCPU = stats.CPUStats.SystemCPUUsage

		break
	case <-time.After(time.Duration(d.statsTimeout) * time.Second):
		d.log.Error("Timed out collecting stats for container ", container.ID)
		done <- true
		break
	}
}

// buildMetrics creates the actual metrics for the given container.
func (d DockerStats) buildMetrics(container *docker.Container, memUsed, memLimit, cpuPercentage float64) []metric.Metric {
	ret := []metric.Metric{
		buildDockerMetric("DockerMemoryUsed", memUsed),
		buildDockerMetric("DockerMemoryLimit", memLimit),
		buildDockerMetric("DockerCpuPercentage", cpuPercentage),
	}
	additionalDimensions := map[string]string{
		"container_id":   container.ID,
		"container_name": strings.TrimPrefix(container.Name, "/"),
	}
	metric.AddToAll(&ret, additionalDimensions)
	metric.AddToAll(&ret, getServiceDimensions(container))

	return ret
}

// sendMetrics writes all the metrics received to the collector channel.
func (d DockerStats) sendMetrics(metrics []metric.Metric) {
	for _, m := range metrics {
		d.Channel() <- m
	}
}

// Function that extracts the service and instance name from mesos id in order to add them as dimensions
// in these metrics.
func getServiceDimensions(container *docker.Container) map[string]string {
	envVars := container.Config.Env

	tmp := make(map[string]string)
	for _, envVariable := range envVars {
		envArray := strings.Split(envVariable, "=")
		if envArray[0] == mesosTaskID {
			serviceName, instance := getInfoFromMesosTaskID(envArray[1])
			tmp["service_name"] = serviceName
			tmp["instance_name"] = instance
			break
		}
	}
	return tmp
}

func getInfoFromMesosTaskID(taskID string) (serviceName, instance string) {
	varArray := strings.Split(taskID, ".")
	return strings.Replace(varArray[0], "--", "_", -1), strings.Replace(varArray[1], "--", "_", -1)
}

func buildDockerMetric(name string, value float64) (m metric.Metric) {
	m = metric.New(name)
	m.Value = value
	m.AddDimension("collector", "DockerStats")
	return m
}

// Function that compute the current cpu usage percentage combining current and last values.
func calculateCPUPercent(previousCPU, previousSystem uint64, stats *docker.Stats) float64 {
	var (
		cpuPercent = 0.0
		// calculate the change for the cpu usage of the container in between readings
		cpuDelta = float64(stats.CPUStats.CPUUsage.TotalUsage - previousCPU)
		// calculate the change for the entire system between readings
		systemDelta = float64(stats.CPUStats.SystemCPUUsage - previousSystem)
	)

	if systemDelta > 0.0 && cpuDelta > 0.0 {
		cpuPercent = (cpuDelta / systemDelta) * float64(len(stats.CPUStats.CPUUsage.PercpuUsage)) * 100.0
	}
	return cpuPercent
}
<commit_msg>Retry endpoint connection<commit_after>package collector

import (
	"fullerite/config"
	"fullerite/metric"

	"strings"
	"time"

	l "github.com/Sirupsen/logrus"

	"github.com/fsouza/go-dockerclient"
)

const (
	mesosTaskID         = "MESOS_TASK_ID"
	endpoint            = "unix:///var/run/docker.sock"
	defaultStatsTimeout = 7
)

// DockerStats collector type.
// previousCPUValues contains the last cpu-usage values per container.
// dockerClient is the client for the Docker remote API.
type DockerStats struct {
	baseCollector
	previousCPUValues map[string]*CPUValues
	dockerClient      *docker.Client
	statsTimeout      int
}

// CPUValues struct contains the last cpu-usage values in order to compute properly the current values.
// (see calculateCPUPercent() for more details)
type CPUValues struct {
	totCPU, systemCPU uint64
}

// NewDockerStats creates a new DockerStats collector.
func NewDockerStats(channel chan metric.Metric, initialInterval int, log *l.Entry) *DockerStats {
	d := new(DockerStats)

	d.log = log
	d.channel = channel
	d.interval = initialInterval

	d.name = "DockerStats"
	d.previousCPUValues = make(map[string]*CPUValues)
	d.dockerClient, _ = docker.NewClient(endpoint)
	d.statsTimeout = defaultStatsTimeout

	return d
}

// Configure takes a dictionary of values with which the handler can configure itself.
func (d *DockerStats) Configure(configMap map[string]interface{}) {
	if timeout, exists := configMap["dockerStatsTimeout"]; exists {
		d.statsTimeout = config.GetAsInt(timeout, defaultStatsTimeout)
	}
	d.configureCommonParams(configMap)
}

// Collect iterates on all the docker containers alive and, if possible, collects the correspondent
// memory and cpu statistics.
// For each container a gorutine is started to spin up the collection process.
func (d *DockerStats) Collect() {
	containers, err := d.dockerClient.ListContainers(docker.ListContainersOptions{All: false})
	if err != nil {
		d.log.Error("ListContainers() failed: ", err)
		if err = d.dockerClient.Ping(); err != nil {
			d.dockerClient, _ = docker.NewClient(endpoint)
		}
		return
	}
	for _, apiContainer := range containers {
		container, err := d.dockerClient.InspectContainer(apiContainer.ID)
		if err != nil {
			d.log.Error("InspectContainer() failed: ", err)
			continue
		}
		if _, ok := d.previousCPUValues[container.ID]; !ok {
			d.previousCPUValues[container.ID] = new(CPUValues)
		}
		go d.getDockerContainerInfo(container)
	}
}

// getDockerContainerInfo gets container statistics for the given container.
// results is a channel to make possible the synchronization between the main process and the gorutines (wait-notify pattern).
func (d DockerStats) getDockerContainerInfo(container *docker.Container) {
	errC := make(chan error, 1)
	statsC := make(chan *docker.Stats, 1)
	done := make(chan bool)

	go func() {
		errC <- d.dockerClient.Stats(docker.StatsOptions{container.ID, statsC, false, done, 0})
	}()
	select {
	case stats, ok := <-statsC:
		if !ok {
			err := <-errC
			d.log.Error("Failed to collect docker container stats: ", err)
			break
		}
		done <- true

		ret := d.buildMetrics(container, float64(stats.MemoryStats.Usage), float64(stats.MemoryStats.Limit), calculateCPUPercent(d.previousCPUValues[container.ID].totCPU, d.previousCPUValues[container.ID].systemCPU, stats))

		d.sendMetrics(ret)

		d.previousCPUValues[container.ID].totCPU = stats.CPUStats.CPUUsage.TotalUsage
		d.previousCPUValues[container.ID].systemCPU = stats.CPUStats.SystemCPUUsage

		break
	case <-time.After(time.Duration(d.statsTimeout) * time.Second):
		d.log.Error("Timed out collecting stats for container ", container.ID)
		done <- true
		break
	}
}

// buildMetrics creates the actual metrics for the given container.
func (d DockerStats) buildMetrics(container *docker.Container, memUsed, memLimit, cpuPercentage float64) []metric.Metric {
	ret := []metric.Metric{
		buildDockerMetric("DockerMemoryUsed", memUsed),
		buildDockerMetric("DockerMemoryLimit", memLimit),
		buildDockerMetric("DockerCpuPercentage", cpuPercentage),
	}
	additionalDimensions := map[string]string{
		"container_id":   container.ID,
		"container_name": strings.TrimPrefix(container.Name, "/"),
	}
	metric.AddToAll(&ret, additionalDimensions)
	metric.AddToAll(&ret, getServiceDimensions(container))

	return ret
}

// sendMetrics writes all the metrics received to the collector channel.
func (d DockerStats) sendMetrics(metrics []metric.Metric) {
	for _, m := range metrics {
		d.Channel() <- m
	}
}

// Function that extracts the service and instance name from mesos id in order to add them as dimensions
// in these metrics.
func getServiceDimensions(container *docker.Container) map[string]string {
	envVars := container.Config.Env

	tmp := make(map[string]string)
	for _, envVariable := range envVars {
		envArray := strings.Split(envVariable, "=")
		if envArray[0] == mesosTaskID {
			serviceName, instance := getInfoFromMesosTaskID(envArray[1])
			tmp["service_name"] = serviceName
			tmp["instance_name"] = instance
			break
		}
	}
	return tmp
}

func getInfoFromMesosTaskID(taskID string) (serviceName, instance string) {
	varArray := strings.Split(taskID, ".")
	return strings.Replace(varArray[0], "--", "_", -1), strings.Replace(varArray[1], "--", "_", -1)
}

func buildDockerMetric(name string, value float64) (m metric.Metric) {
	m = metric.New(name)
	m.Value = value
	m.AddDimension("collector", "DockerStats")
	return m
}

// Function that compute the current cpu usage percentage combining current and last values.
func calculateCPUPercent(previousCPU, previousSystem uint64, stats *docker.Stats) float64 {
	var (
		cpuPercent = 0.0
		// calculate the change for the cpu usage of the container in between readings
		cpuDelta = float64(stats.CPUStats.CPUUsage.TotalUsage - previousCPU)
		// calculate the change for the entire system between readings
		systemDelta = float64(stats.CPUStats.SystemCPUUsage - previousSystem)
	)

	if systemDelta > 0.0 && cpuDelta > 0.0 {
		cpuPercent = (cpuDelta / systemDelta) * float64(len(stats.CPUStats.CPUUsage.PercpuUsage)) * 100.0
	}
	return cpuPercent
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// apiserver is the main api server and master for the cluster.
// it is responsible for serving the cluster management API.
package main

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"time"

	"github.com/coreos/go-etcd/etcd"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
	kube_client "github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/registry"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
)

var (
	port                        = flag.Uint("port", 8080, "The port to listen on.  Default 8080.")
	address                     = flag.String("address", "127.0.0.1", "The address on the local server to listen to. Default 127.0.0.1")
	apiPrefix                   = flag.String("api_prefix", "/api/v1beta1", "The prefix for API requests on the server. Default '/api/v1beta1'")
	etcdServerList, machineList util.StringList
)

func init() {
	flag.Var(&etcdServerList, "etcd_servers", "Servers for the etcd (http://ip:port), comma separated")
	flag.Var(&machineList, "machines", "List of machines to schedule onto, comma separated.")
}

func main() {
	flag.Parse()

	if len(machineList) == 0 {
		log.Fatal("No machines specified!")
	}

	var (
		podRegistry        registry.PodRegistry
		controllerRegistry registry.ControllerRegistry
		serviceRegistry    registry.ServiceRegistry
	)

	if len(etcdServerList) > 0 {
		log.Printf("Creating etcd client pointing to %v", etcdServerList)
		etcdClient := etcd.NewClient(etcdServerList)
		podRegistry = registry.MakeEtcdRegistry(etcdClient, machineList)
		controllerRegistry = registry.MakeEtcdRegistry(etcdClient, machineList)
		serviceRegistry = registry.MakeEtcdRegistry(etcdClient, machineList)
	} else {
		podRegistry = registry.MakeMemoryRegistry()
		controllerRegistry = registry.MakeMemoryRegistry()
		serviceRegistry = registry.MakeMemoryRegistry()
	}

	containerInfo := &kube_client.HTTPContainerInfo{
		Client: http.DefaultClient,
		Port:   10250,
	}

	random := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
	storage := map[string]apiserver.RESTStorage{
		"pods": registry.MakePodRegistryStorage(podRegistry, containerInfo, registry.MakeFirstFitScheduler(machineList, podRegistry, random)),
		"replicationControllers": registry.MakeControllerRegistryStorage(controllerRegistry),
		"services":               registry.MakeServiceRegistryStorage(serviceRegistry),
	}

	endpoints := registry.MakeEndpointController(serviceRegistry, podRegistry)
	go util.Forever(func() { endpoints.SyncServiceEndpoints() }, time.Second*10)

	s := &http.Server{
		Addr:           fmt.Sprintf("%s:%d", *address, *port),
		Handler:        apiserver.New(storage, *apiPrefix),
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}
	log.Fatal(s.ListenAndServe())
}
<commit_msg>Readability improvements.<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// apiserver is the main api server and master for the cluster.
// it is responsible for serving the cluster management API.
package main

import (
	"flag"
	"log"
	"math/rand"
	"net"
	"net/http"
	"strconv"
	"time"

	"github.com/coreos/go-etcd/etcd"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/registry"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
)

var (
	port                        = flag.Uint("port", 8080, "The port to listen on.  Default 8080.")
	address                     = flag.String("address", "127.0.0.1", "The address on the local server to listen to. Default 127.0.0.1")
	apiPrefix                   = flag.String("api_prefix", "/api/v1beta1", "The prefix for API requests on the server. Default '/api/v1beta1'")
	etcdServerList, machineList util.StringList
)

func init() {
	flag.Var(&etcdServerList, "etcd_servers", "Servers for the etcd (http://ip:port), comma separated")
	flag.Var(&machineList, "machines", "List of machines to schedule onto, comma separated.")
}

func main() {
	flag.Parse()

	if len(machineList) == 0 {
		log.Fatal("No machines specified!")
	}

	var (
		podRegistry        registry.PodRegistry
		controllerRegistry registry.ControllerRegistry
		serviceRegistry    registry.ServiceRegistry
	)

	if len(etcdServerList) > 0 {
		log.Printf("Creating etcd client pointing to %v", etcdServerList)
		etcdClient := etcd.NewClient(etcdServerList)
		podRegistry = registry.MakeEtcdRegistry(etcdClient, machineList)
		controllerRegistry = registry.MakeEtcdRegistry(etcdClient, machineList)
		serviceRegistry = registry.MakeEtcdRegistry(etcdClient, machineList)
	} else {
		podRegistry = registry.MakeMemoryRegistry()
		controllerRegistry = registry.MakeMemoryRegistry()
		serviceRegistry = registry.MakeMemoryRegistry()
	}

	containerInfo := &client.HTTPContainerInfo{
		Client: http.DefaultClient,
		Port:   10250,
	}

	random := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
	storage := map[string]apiserver.RESTStorage{
		"pods": registry.MakePodRegistryStorage(podRegistry, containerInfo, registry.MakeFirstFitScheduler(machineList, podRegistry, random)),
		"replicationControllers": registry.MakeControllerRegistryStorage(controllerRegistry),
		"services":               registry.MakeServiceRegistryStorage(serviceRegistry),
	}

	endpoints := registry.MakeEndpointController(serviceRegistry, podRegistry)
	go util.Forever(func() { endpoints.SyncServiceEndpoints() }, time.Second*10)

	s := &http.Server{
		Addr:           net.JoinHostPort(*address, strconv.Itoa(int(*port))),
		Handler:        apiserver.New(storage, *apiPrefix),
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}
	log.Fatal(s.ListenAndServe())
}
<|endoftext|>


<commit_before>package dropwizard

import (
	"fullerite/metric"
	"regexp"

	l "github.com/Sirupsen/logrus"
)

var defaultLog = l.WithFields(l.Fields{"app": "fullerite", "pkg": "dropwizard"})

const (
	// MetricTypeCounter String for counter metric type
	MetricTypeCounter string = "COUNTER"
	// MetricTypeGauge String for Gauge metric type
	MetricTypeGauge string = "GAUGE"
)

// Parser is an interface for dropwizard parsers
type Parser interface {
	Parse() ([]metric.Metric, error)
	// take actual value and convert it to metric object
	createMetricFromDatam(string, interface{}, string, string) (metric.Metric, bool)
	// take map of data and extract metrics
	metricFromMap(map[string]interface{}, string, string) []metric.Metric
	// take map of maps and extract metrics, this is like first level of parsing
	parseMapOfMap(map[string]map[string]interface{}, string) []metric.Metric
	// is Cumulative Counter enabled for this metric
	isCCEnabled() bool
}

// Format defines format in which dropwizard metrics are emitted
//
// the assumed format is:
// {
// 	"gauges": {},
// 	"histograms": {},
// 	"version": "xxx",
// 	"timers": {
// 		"pyramid_uwsgi_metrics.tweens.status.metrics": {
// 			"count": ###,
// 			"p98": ###,
// 			...
// 		},
// 		"pyramid_uwsgi_metrics.tweens.lookup": {
// 			"count": ###,
// 			...
// 		}
// 	},
// 	"meters": {
// 		"pyramid_uwsgi_metrics.tweens.XXX": {
//			"count": ###,
//			"mean_rate": ###,
// 			"m1_rate": ###
// 		}
// 	},
// 	"counters": {
//		"myname": {
//			"count": ###,
// 	}
// }
type Format struct {
	ServiceDims map[string]interface{} `json:"service_dims"`
	Counters    map[string]map[string]interface{}
	Gauges      map[string]map[string]interface{}
	Histograms  map[string]map[string]interface{}
	Meters      map[string]map[string]interface{}
	Timers      map[string]map[string]interface{}
}

// BaseParser is a base struct for real parsers
type BaseParser struct {
	data      []byte
	log       *l.Entry
	ccEnabled bool // Enable cumulative counters
	schemaVer string
}

// Parse can be called from collector code to parse results
func Parse(raw []byte, schemaVer string, ccEnabled bool) ([]metric.Metric, error) {
	var parser Parser
	if schemaVer == "uwsgi.1.0" || schemaVer == "uwsg.1.1" {
		parser = NewUWSGIMetric(raw, schemaVer, ccEnabled)
	} else if schemaVer == "java-1.1" {
		parser = NewJavaMetric(raw, schemaVer, ccEnabled)
	} else {
		parser = NewLegacyMetric(raw, schemaVer, ccEnabled)
	}
	return parser.Parse()
}

// metricFromMap takes in flattened maps formatted like this::
// {
//    "count":      3443,
//    "mean_rate": 100
// }
// and metricname and metrictype and returns metrics for each name:rollup pair
func (parser *BaseParser) metricFromMap(metricMap map[string]interface{},
	metricName string,
	metricType string) []metric.Metric {
	results := []metric.Metric{}
	dims := make(map[string]string)

	for rollup, value := range metricMap {
		// First check for dimension set if present
		// See uwsgi_metric.go:68 for explanation on the range over value
		if rollup == "dimensions" {
			for dimName, dimVal := range value.(map[string]interface{}) {
				// Handle nil valued dimensions
				if strVal, ok := dimVal.(string); ok {
					dims[dimName] = strVal
				}
			}
			continue
		}

		mName := metricName
		mType := metricType
		matched, _ := regexp.MatchString("m[0-9]+_rate", rollup)

		// If cumulCounterEnabled is true:
		//		1. change metric type meter.count and timer.count moving them to cumulative counter
		//		2. don't send back metered metrics (rollup == 'mXX_rate')
		if parser.ccEnabled && matched {
			continue
		}
		if parser.ccEnabled && rollup != "value" {
			mName = metricName + "." + rollup
			if rollup == "count" {
				mType = metric.CumulativeCounter
			}
		}
		tempMetric, ok := parser.createMetricFromDatam(rollup, value, mName, mType)
		if ok {
			results = append(results, tempMetric)
		}
	}

	metric.AddToAll(&results, dims)
	return results
}

func (parser *BaseParser) isCCEnabled() bool {
	return parser.ccEnabled
}

// createMetricFromDatam takes in rollup, value, metricName, metricType and returns metric only if
// value was numeric
func (parser *BaseParser) createMetricFromDatam(rollup string,
	value interface{},
	metricName string, metricType string) (metric.Metric, bool) {
	m := metric.New(metricName)
	m.MetricType = metricType
	m.AddDimension("rollup", rollup)

	// only add things that have a numeric base
	switch value.(type) {
	case float64:
		m.Value = value.(float64)
	case int:
		m.Value = float64(value.(int))
	default:
		return m, false
	}
	return m, true
}

func extractParsedMetric(parser Parser, parsed *Format) []metric.Metric {
	results := []metric.Metric{}
	appendIt := func(metrics []metric.Metric, typeDimVal string) {
		if !parser.isCCEnabled() {
			metric.AddToAll(&metrics, map[string]string{"type": typeDimVal})
		}
		results = append(results, metrics...)
	}

	appendIt(parser.parseMapOfMap(parsed.Gauges, metric.Gauge), "gauge")
	appendIt(parser.parseMapOfMap(parsed.Counters, metric.Counter), "counter")
	appendIt(parser.parseMapOfMap(parsed.Histograms, metric.Gauge), "histogram")
	appendIt(parser.parseMapOfMap(parsed.Meters, metric.Gauge), "meter")
	appendIt(parser.parseMapOfMap(parsed.Timers, metric.Gauge), "timer")

	return results
}

func (parser *BaseParser) parseMapOfMap(
	metricMap map[string]map[string]interface{},
	metricType string) []metric.Metric {
	return []metric.Metric{}
}

// Parse is just a placehoder function
func (parser *BaseParser) Parse() ([]metric.Metric, error) {
	return []metric.Metric{}, nil
}
<commit_msg>Assign "null" on nil-valued dimensions<commit_after>package dropwizard

import (
	"fullerite/metric"
	"regexp"

	l "github.com/Sirupsen/logrus"
)

var defaultLog = l.WithFields(l.Fields{"app": "fullerite", "pkg": "dropwizard"})

const (
	// MetricTypeCounter String for counter metric type
	MetricTypeCounter string = "COUNTER"
	// MetricTypeGauge String for Gauge metric type
	MetricTypeGauge string = "GAUGE"
)

// Parser is an interface for dropwizard parsers
type Parser interface {
	Parse() ([]metric.Metric, error)
	// take actual value and convert it to metric object
	createMetricFromDatam(string, interface{}, string, string) (metric.Metric, bool)
	// take map of data and extract metrics
	metricFromMap(map[string]interface{}, string, string) []metric.Metric
	// take map of maps and extract metrics, this is like first level of parsing
	parseMapOfMap(map[string]map[string]interface{}, string) []metric.Metric
	// is Cumulative Counter enabled for this metric
	isCCEnabled() bool
}

// Format defines format in which dropwizard metrics are emitted
//
// the assumed format is:
// {
// 	"gauges": {},
// 	"histograms": {},
// 	"version": "xxx",
// 	"timers": {
// 		"pyramid_uwsgi_metrics.tweens.status.metrics": {
// 			"count": ###,
// 			"p98": ###,
// 			...
// 		},
// 		"pyramid_uwsgi_metrics.tweens.lookup": {
// 			"count": ###,
// 			...
// 		}
// 	},
// 	"meters": {
// 		"pyramid_uwsgi_metrics.tweens.XXX": {
//			"count": ###,
//			"mean_rate": ###,
// 			"m1_rate": ###
// 		}
// 	},
// 	"counters": {
//		"myname": {
//			"count": ###,
// 	}
// }
type Format struct {
	ServiceDims map[string]interface{} `json:"service_dims"`
	Counters    map[string]map[string]interface{}
	Gauges      map[string]map[string]interface{}
	Histograms  map[string]map[string]interface{}
	Meters      map[string]map[string]interface{}
	Timers      map[string]map[string]interface{}
}

// BaseParser is a base struct for real parsers
type BaseParser struct {
	data      []byte
	log       *l.Entry
	ccEnabled bool // Enable cumulative counters
	schemaVer string
}

// Parse can be called from collector code to parse results
func Parse(raw []byte, schemaVer string, ccEnabled bool) ([]metric.Metric, error) {
	var parser Parser
	if schemaVer == "uwsgi.1.0" || schemaVer == "uwsg.1.1" {
		parser = NewUWSGIMetric(raw, schemaVer, ccEnabled)
	} else if schemaVer == "java-1.1" {
		parser = NewJavaMetric(raw, schemaVer, ccEnabled)
	} else {
		parser = NewLegacyMetric(raw, schemaVer, ccEnabled)
	}
	return parser.Parse()
}

// metricFromMap takes in flattened maps formatted like this::
// {
//    "count":      3443,
//    "mean_rate": 100
// }
// and metricname and metrictype and returns metrics for each name:rollup pair
func (parser *BaseParser) metricFromMap(metricMap map[string]interface{},
	metricName string,
	metricType string) []metric.Metric {
	results := []metric.Metric{}
	dims := make(map[string]string)

	for rollup, value := range metricMap {
		// First check for dimension set if present
		// See uwsgi_metric.go:68 for explanation on the range over value
		if rollup == "dimensions" {
			for dimName, dimVal := range value.(map[string]interface{}) {
				// Handle nil valued dimensions
				if strVal, ok := dimVal.(string); ok {
					dims[dimName] = strVal
				} else {
					dims[dimName] = "null"
				}
			}
			continue
		}

		mName := metricName
		mType := metricType
		matched, _ := regexp.MatchString("m[0-9]+_rate", rollup)

		// If cumulCounterEnabled is true:
		//		1. change metric type meter.count and timer.count moving them to cumulative counter
		//		2. don't send back metered metrics (rollup == 'mXX_rate')
		if parser.ccEnabled && matched {
			continue
		}
		if parser.ccEnabled && rollup != "value" {
			mName = metricName + "." + rollup
			if rollup == "count" {
				mType = metric.CumulativeCounter
			}
		}
		tempMetric, ok := parser.createMetricFromDatam(rollup, value, mName, mType)
		if ok {
			results = append(results, tempMetric)
		}
	}

	metric.AddToAll(&results, dims)
	return results
}

func (parser *BaseParser) isCCEnabled() bool {
	return parser.ccEnabled
}

// createMetricFromDatam takes in rollup, value, metricName, metricType and returns metric only if
// value was numeric
func (parser *BaseParser) createMetricFromDatam(rollup string,
	value interface{},
	metricName string, metricType string) (metric.Metric, bool) {
	m := metric.New(metricName)
	m.MetricType = metricType
	m.AddDimension("rollup", rollup)

	// only add things that have a numeric base
	switch value.(type) {
	case float64:
		m.Value = value.(float64)
	case int:
		m.Value = float64(value.(int))
	default:
		return m, false
	}
	return m, true
}

func extractParsedMetric(parser Parser, parsed *Format) []metric.Metric {
	results := []metric.Metric{}
	appendIt := func(metrics []metric.Metric, typeDimVal string) {
		if !parser.isCCEnabled() {
			metric.AddToAll(&metrics, map[string]string{"type": typeDimVal})
		}
		results = append(results, metrics...)
	}

	appendIt(parser.parseMapOfMap(parsed.Gauges, metric.Gauge), "gauge")
	appendIt(parser.parseMapOfMap(parsed.Counters, metric.Counter), "counter")
	appendIt(parser.parseMapOfMap(parsed.Histograms, metric.Gauge), "histogram")
	appendIt(parser.parseMapOfMap(parsed.Meters, metric.Gauge), "meter")
	appendIt(parser.parseMapOfMap(parsed.Timers, metric.Gauge), "timer")

	return results
}

func (parser *BaseParser) parseMapOfMap(
	metricMap map[string]map[string]interface{},
	metricType string) []metric.Metric {
	return []metric.Metric{}
}

// Parse is just a placehoder function
func (parser *BaseParser) Parse() ([]metric.Metric, error) {
	return []metric.Metric{}, nil
}
<|endoftext|>


<commit_before>package main_test

import (
	"reflect"
	"strings"
	"testing"
	"time"

	main "github.com/influxdb/influxdb/cmd/influxd"
)

// Ensure that megabyte sizes can be parsed.
func TestSize_UnmarshalText_MB(t *testing.T) {
	var s main.Size
	if err := s.UnmarshalText([]byte("200m")); err != nil {
		t.Fatalf("unexpected error: %s", err)
	} else if s != 200*(1<<20) {
		t.Fatalf("unexpected size: %d", s)
	}
}

// Ensure that gigabyte sizes can be parsed.
func TestSize_UnmarshalText_GB(t *testing.T) {
	if typ := reflect.TypeOf(0); typ.Size() != 8 {
		t.Skip("large gigabyte parsing on 64-bit arch only")
	}

	var s main.Size
	if err := s.UnmarshalText([]byte("10g")); err != nil {
		t.Fatalf("unexpected error: %s", err)
	} else if s != 10*(1<<30) {
		t.Fatalf("unexpected size: %d", s)
	}
}

// Ensure that a TOML configuration file can be parsed into a Config.
func TestParseConfig(t *testing.T) {
	c, err := main.ParseConfig(testFile)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	} else if c.Hostname != "myserver.com" {
		t.Fatalf("hostname mismatch: %v", c.Hostname)
	}

	if c.JoinURLs() != "http://127.0.0.1:8086" {
		t.Fatalf("JoinURLs mistmatch: %v", c.JoinURLs())
	}

	if c.Logging.File != "influxdb.log" {
		t.Fatalf("logging file mismatch: %v", c.Logging.File)
	}

	if !c.Authentication.Enabled {
		t.Fatalf("authentication enabled mismatch: %v", c.Authentication.Enabled)
	}

	if c.Admin.Port != 8083 {
		t.Fatalf("admin port mismatch: %v", c.Admin.Port)
	} else if c.Admin.Assets != "./admin" {
		t.Fatalf("admin assets mismatch: %v", c.Admin.Assets)
	}

	if c.Data.Port != main.DefaultBrokerPort {
		t.Fatalf("data port mismatch: %v", c.Data.Port)
	}

	if len(c.Graphites) != 2 {
		t.Fatalf("graphites  mismatch.  expected %v, got: %v", 2, len(c.Graphites))
	}

	tcpGraphite := c.Graphites[0]
	switch {
	case tcpGraphite.Enabled != true:
		t.Fatalf("graphite tcp enabled mismatch: expected: %v, got %v", true, tcpGraphite.Enabled)
	case tcpGraphite.Addr != "192.168.0.1":
		t.Fatalf("graphite tcp address mismatch: expected %v, got  %v", "192.168.0.1", tcpGraphite.Addr)
	case tcpGraphite.Port != 2003:
		t.Fatalf("graphite tcp port mismatch: expected %v, got %v", 2003, tcpGraphite.Port)
	case tcpGraphite.Database != "graphite_tcp":
		t.Fatalf("graphite tcp database mismatch: expected %v, got %v", "graphite_tcp", tcpGraphite.Database)
	case strings.ToLower(tcpGraphite.Protocol) != "tcp":
		t.Fatalf("graphite tcp protocol mismatch: expected %v, got %v", "tcp", strings.ToLower(tcpGraphite.Protocol))
	case tcpGraphite.LastEnabled() != true:
		t.Fatalf("graphite tcp name-position mismatch: expected %v, got %v", "last", tcpGraphite.NamePosition)
	case tcpGraphite.NameSeparatorString() != "-":
		t.Fatalf("graphite tcp name-separator mismatch: expected %v, got %v", "-", tcpGraphite.NameSeparatorString())
	}

	udpGraphite := c.Graphites[1]
	switch {
	case udpGraphite.Enabled != true:
		t.Fatalf("graphite udp enabled mismatch: expected: %v, got %v", true, udpGraphite.Enabled)
	case udpGraphite.Addr != "192.168.0.2":
		t.Fatalf("graphite udp address mismatch: expected %v, got  %v", "192.168.0.2", udpGraphite.Addr)
	case udpGraphite.Port != 2005:
		t.Fatalf("graphite udp port mismatch: expected %v, got %v", 2005, udpGraphite.Port)
	case udpGraphite.Database != "graphite_udp":
		t.Fatalf("graphite database mismatch: expected %v, got %v", "graphite_udp", udpGraphite.Database)
	case strings.ToLower(udpGraphite.Protocol) != "udp":
		t.Fatalf("graphite udp protocol mismatch: expected %v, got %v", "udp", strings.ToLower(udpGraphite.Protocol))
	}

	switch {
	case c.Collectd.Enabled != true:
		t.Errorf("collectd enabled mismatch: expected: %v, got %v", true, c.Collectd.Enabled)
	case c.Collectd.Addr != "192.168.0.3":
		t.Errorf("collectd address mismatch: expected %v, got  %v", "192.168.0.3", c.Collectd.Addr)
	case c.Collectd.Port != 25827:
		t.Errorf("collectd port mismatch: expected %v, got %v", 2005, c.Collectd.Port)
	case c.Collectd.Database != "collectd_database":
		t.Errorf("collectdabase mismatch: expected %v, got %v", "collectd_database", c.Collectd.Database)
	case c.Collectd.TypesDB != "foo-db-type":
		t.Errorf("collectd typesdb mismatch: expected %v, got %v", "foo-db-type", c.Collectd.TypesDB)
	}

	if c.Broker.Port != 8086 {
		t.Fatalf("broker port mismatch: %v", c.Broker.Port)
	} else if c.Broker.Dir != "/tmp/influxdb/development/broker" {
		t.Fatalf("broker dir mismatch: %v", c.Broker.Dir)
	} else if time.Duration(c.Broker.Timeout) != time.Second {
		t.Fatalf("broker duration mismatch: %v", c.Broker.Timeout)
	}

	if c.Data.Dir != "/tmp/influxdb/development/db" {
		t.Fatalf("data dir mismatch: %v", c.Data.Dir)
	}

	// TODO: UDP Servers testing.
	/*
		c.Assert(config.UdpServers, HasLen, 1)
		c.Assert(config.UdpServers[0].Enabled, Equals, true)
		c.Assert(config.UdpServers[0].Port, Equals, 4444)
		c.Assert(config.UdpServers[0].Database, Equals, "test")
	*/
}

// Testing configuration file.
const testFile = `
# Welcome to the InfluxDB configuration file.

# If hostname (on the OS) doesn't return a name that can be resolved by the other
# systems in the cluster, you'll have to set the hostname to an IP or something
# that can be resolved here.
hostname = "myserver.com"

# Controls certain parameters that only take effect until an initial successful
# start-up has occurred.
[initialization]
join-urls = "http://127.0.0.1:8086"

# Control authentication
[authentication]
enabled = true

[logging]
file   = "influxdb.log"

# Configure the admin server
[admin]
port   = 8083                   # binding is disabled if the port isn't set
assets = "./admin"

# Configure the http api
[api]
ssl-port = 8087    # Ssl support is enabled if you set a port and cert
ssl-cert = "../cert.pem"

# connections will timeout after this amount of time. Ensures that clients that misbehave
# and keep alive connections they don't use won't end up connection a million times.
# However, if a request is taking longer than this to complete, could be a problem.
read-timeout = "5s"

[input_plugins]

  [input_plugins.udp]
  enabled = true
  port = 4444
  database = "test"

# Configure the Graphite servers
[[graphite]]
protocol = "TCP"
enabled = true
address = "192.168.0.1"
port = 2003
database = "graphite_tcp"  # store graphite data in this database
name-position = "last"
name-separator = "-"

[[graphite]]
protocol = "udP"
enabled = true
address = "192.168.0.2"
port = 2005
database = "graphite_udp"  # store graphite data in this database

# Configure collectd server
[collectd]
enabled = true
address = "192.168.0.3"
port = 25827
database = "collectd_database"
typesdb = "foo-db-type"

# Broker configuration
[broker]
# The broker port should be open between all servers in a cluster.
# However, this port shouldn't be accessible from the internet.
port = 8086

# Where the broker logs are stored. The user running InfluxDB will need read/write access.
dir  = "/tmp/influxdb/development/broker"

# election-timeout = "2s"

[data]
dir = "/tmp/influxdb/development/db"

[cluster]
`

func TestCollectd_ConnectionString(t *testing.T) {
	var tests = []struct {
		name             string
		defaultBindAddr  string
		connectionString string
		config           main.Collectd
	}{
		{
			name:             "No address or port provided from config",
			defaultBindAddr:  "192.168.0.1",
			connectionString: "192.168.0.1:25826",
			config:           main.Collectd{},
		},
		{
			name:             "address provided, no port provided from config",
			defaultBindAddr:  "192.168.0.1",
			connectionString: "192.168.0.2:25826",
			config:           main.Collectd{Addr: "192.168.0.2"},
		},
		{
			name:             "no address provided, port provided from config",
			defaultBindAddr:  "192.168.0.1",
			connectionString: "192.168.0.1:25827",
			config:           main.Collectd{Port: 25827},
		},
		{
			name:             "both address and port provided from config",
			defaultBindAddr:  "192.168.0.1",
			connectionString: "192.168.0.2:25827",
			config:           main.Collectd{Addr: "192.168.0.2", Port: 25827},
		},
	}

	for _, test := range tests {
		t.Logf("test: %q", test.name)
		s := test.config.ConnectionString(test.defaultBindAddr)
		if s != test.connectionString {
			t.Errorf("connection string mismatch, expected: %q, got: %q", test.connectionString, s)
		}
	}
}
<commit_msg>Add unit test for 'cluster dir' configuration<commit_after>package main_test

import (
	"reflect"
	"strings"
	"testing"
	"time"

	main "github.com/influxdb/influxdb/cmd/influxd"
)

// Ensure that megabyte sizes can be parsed.
func TestSize_UnmarshalText_MB(t *testing.T) {
	var s main.Size
	if err := s.UnmarshalText([]byte("200m")); err != nil {
		t.Fatalf("unexpected error: %s", err)
	} else if s != 200*(1<<20) {
		t.Fatalf("unexpected size: %d", s)
	}
}

// Ensure that gigabyte sizes can be parsed.
func TestSize_UnmarshalText_GB(t *testing.T) {
	if typ := reflect.TypeOf(0); typ.Size() != 8 {
		t.Skip("large gigabyte parsing on 64-bit arch only")
	}

	var s main.Size
	if err := s.UnmarshalText([]byte("10g")); err != nil {
		t.Fatalf("unexpected error: %s", err)
	} else if s != 10*(1<<30) {
		t.Fatalf("unexpected size: %d", s)
	}
}

// Ensure that a TOML configuration file can be parsed into a Config.
func TestParseConfig(t *testing.T) {
	c, err := main.ParseConfig(testFile)
	if err != nil {
		t.Fatalf("unexpected error: %s", err)
	} else if c.Hostname != "myserver.com" {
		t.Fatalf("hostname mismatch: %v", c.Hostname)
	}

	if c.JoinURLs() != "http://127.0.0.1:8086" {
		t.Fatalf("JoinURLs mistmatch: %v", c.JoinURLs())
	}

	if c.Logging.File != "influxdb.log" {
		t.Fatalf("logging file mismatch: %v", c.Logging.File)
	}

	if !c.Authentication.Enabled {
		t.Fatalf("authentication enabled mismatch: %v", c.Authentication.Enabled)
	}

	if c.Admin.Port != 8083 {
		t.Fatalf("admin port mismatch: %v", c.Admin.Port)
	} else if c.Admin.Assets != "./admin" {
		t.Fatalf("admin assets mismatch: %v", c.Admin.Assets)
	}

	if c.Data.Port != main.DefaultBrokerPort {
		t.Fatalf("data port mismatch: %v", c.Data.Port)
	}

	if len(c.Graphites) != 2 {
		t.Fatalf("graphites  mismatch.  expected %v, got: %v", 2, len(c.Graphites))
	}

	tcpGraphite := c.Graphites[0]
	switch {
	case tcpGraphite.Enabled != true:
		t.Fatalf("graphite tcp enabled mismatch: expected: %v, got %v", true, tcpGraphite.Enabled)
	case tcpGraphite.Addr != "192.168.0.1":
		t.Fatalf("graphite tcp address mismatch: expected %v, got  %v", "192.168.0.1", tcpGraphite.Addr)
	case tcpGraphite.Port != 2003:
		t.Fatalf("graphite tcp port mismatch: expected %v, got %v", 2003, tcpGraphite.Port)
	case tcpGraphite.Database != "graphite_tcp":
		t.Fatalf("graphite tcp database mismatch: expected %v, got %v", "graphite_tcp", tcpGraphite.Database)
	case strings.ToLower(tcpGraphite.Protocol) != "tcp":
		t.Fatalf("graphite tcp protocol mismatch: expected %v, got %v", "tcp", strings.ToLower(tcpGraphite.Protocol))
	case tcpGraphite.LastEnabled() != true:
		t.Fatalf("graphite tcp name-position mismatch: expected %v, got %v", "last", tcpGraphite.NamePosition)
	case tcpGraphite.NameSeparatorString() != "-":
		t.Fatalf("graphite tcp name-separator mismatch: expected %v, got %v", "-", tcpGraphite.NameSeparatorString())
	}

	udpGraphite := c.Graphites[1]
	switch {
	case udpGraphite.Enabled != true:
		t.Fatalf("graphite udp enabled mismatch: expected: %v, got %v", true, udpGraphite.Enabled)
	case udpGraphite.Addr != "192.168.0.2":
		t.Fatalf("graphite udp address mismatch: expected %v, got  %v", "192.168.0.2", udpGraphite.Addr)
	case udpGraphite.Port != 2005:
		t.Fatalf("graphite udp port mismatch: expected %v, got %v", 2005, udpGraphite.Port)
	case udpGraphite.Database != "graphite_udp":
		t.Fatalf("graphite database mismatch: expected %v, got %v", "graphite_udp", udpGraphite.Database)
	case strings.ToLower(udpGraphite.Protocol) != "udp":
		t.Fatalf("graphite udp protocol mismatch: expected %v, got %v", "udp", strings.ToLower(udpGraphite.Protocol))
	}

	switch {
	case c.Collectd.Enabled != true:
		t.Errorf("collectd enabled mismatch: expected: %v, got %v", true, c.Collectd.Enabled)
	case c.Collectd.Addr != "192.168.0.3":
		t.Errorf("collectd address mismatch: expected %v, got  %v", "192.168.0.3", c.Collectd.Addr)
	case c.Collectd.Port != 25827:
		t.Errorf("collectd port mismatch: expected %v, got %v", 2005, c.Collectd.Port)
	case c.Collectd.Database != "collectd_database":
		t.Errorf("collectdabase mismatch: expected %v, got %v", "collectd_database", c.Collectd.Database)
	case c.Collectd.TypesDB != "foo-db-type":
		t.Errorf("collectd typesdb mismatch: expected %v, got %v", "foo-db-type", c.Collectd.TypesDB)
	}

	if c.Broker.Port != 8086 {
		t.Fatalf("broker port mismatch: %v", c.Broker.Port)
	} else if c.Broker.Dir != "/tmp/influxdb/development/broker" {
		t.Fatalf("broker dir mismatch: %v", c.Broker.Dir)
	} else if time.Duration(c.Broker.Timeout) != time.Second {
		t.Fatalf("broker duration mismatch: %v", c.Broker.Timeout)
	}

	if c.Data.Dir != "/tmp/influxdb/development/db" {
		t.Fatalf("data dir mismatch: %v", c.Data.Dir)
	}

	if c.Cluster.Dir != "/tmp/influxdb/development/cluster" {
		t.Fatalf("cluster dir mismatch: %v", c.Cluster.Dir)
	}

	// TODO: UDP Servers testing.
	/*
		c.Assert(config.UdpServers, HasLen, 1)
		c.Assert(config.UdpServers[0].Enabled, Equals, true)
		c.Assert(config.UdpServers[0].Port, Equals, 4444)
		c.Assert(config.UdpServers[0].Database, Equals, "test")
	*/
}

// Testing configuration file.
const testFile = `
# Welcome to the InfluxDB configuration file.

# If hostname (on the OS) doesn't return a name that can be resolved by the other
# systems in the cluster, you'll have to set the hostname to an IP or something
# that can be resolved here.
hostname = "myserver.com"

# Controls certain parameters that only take effect until an initial successful
# start-up has occurred.
[initialization]
join-urls = "http://127.0.0.1:8086"

# Control authentication
[authentication]
enabled = true

[logging]
file   = "influxdb.log"

# Configure the admin server
[admin]
port   = 8083                   # binding is disabled if the port isn't set
assets = "./admin"

# Configure the http api
[api]
ssl-port = 8087    # Ssl support is enabled if you set a port and cert
ssl-cert = "../cert.pem"

# connections will timeout after this amount of time. Ensures that clients that misbehave
# and keep alive connections they don't use won't end up connection a million times.
# However, if a request is taking longer than this to complete, could be a problem.
read-timeout = "5s"

[input_plugins]

  [input_plugins.udp]
  enabled = true
  port = 4444
  database = "test"

# Configure the Graphite servers
[[graphite]]
protocol = "TCP"
enabled = true
address = "192.168.0.1"
port = 2003
database = "graphite_tcp"  # store graphite data in this database
name-position = "last"
name-separator = "-"

[[graphite]]
protocol = "udP"
enabled = true
address = "192.168.0.2"
port = 2005
database = "graphite_udp"  # store graphite data in this database

# Configure collectd server
[collectd]
enabled = true
address = "192.168.0.3"
port = 25827
database = "collectd_database"
typesdb = "foo-db-type"

# Broker configuration
[broker]
# The broker port should be open between all servers in a cluster.
# However, this port shouldn't be accessible from the internet.
port = 8086

# Where the broker logs are stored. The user running InfluxDB will need read/write access.
dir  = "/tmp/influxdb/development/broker"

# election-timeout = "2s"

[data]
dir = "/tmp/influxdb/development/db"

[cluster]
dir = "/tmp/influxdb/development/cluster"
`

func TestCollectd_ConnectionString(t *testing.T) {
	var tests = []struct {
		name             string
		defaultBindAddr  string
		connectionString string
		config           main.Collectd
	}{
		{
			name:             "No address or port provided from config",
			defaultBindAddr:  "192.168.0.1",
			connectionString: "192.168.0.1:25826",
			config:           main.Collectd{},
		},
		{
			name:             "address provided, no port provided from config",
			defaultBindAddr:  "192.168.0.1",
			connectionString: "192.168.0.2:25826",
			config:           main.Collectd{Addr: "192.168.0.2"},
		},
		{
			name:             "no address provided, port provided from config",
			defaultBindAddr:  "192.168.0.1",
			connectionString: "192.168.0.1:25827",
			config:           main.Collectd{Port: 25827},
		},
		{
			name:             "both address and port provided from config",
			defaultBindAddr:  "192.168.0.1",
			connectionString: "192.168.0.2:25827",
			config:           main.Collectd{Addr: "192.168.0.2", Port: 25827},
		},
	}

	for _, test := range tests {
		t.Logf("test: %q", test.name)
		s := test.config.ConnectionString(test.defaultBindAddr)
		if s != test.connectionString {
			t.Errorf("connection string mismatch, expected: %q, got: %q", test.connectionString, s)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"fmt"
	"io"
	"os"
	"time"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"launchpad.net/gnuflag"
)

const (
	notset           = "juju-backup-<date>-<time>.tar.gz"
	filenameTemplate = "juju-backup-%04d%02d%02d-%02d%02d%02d.tar.gz"
	downloadWarning  = "WARNING: downloading backup archives is recommended; " +
		"backups stored remotely are not guaranteed to be available"
)

const createDoc = `
"create" requests that juju create a backup of its state and print the
backup's unique ID.  You may provide a note to associate with the backup.

The backup archive and associated metadata are stored remotely by juju.

The --download option may be used without the --filename option.  In
that case, the backup archive will be stored in the current working
directory with a name matching juju-backup-<date>-<time>.tar.gz.

WARNING: Remotely stored backups will be lost when the environment is
destroyed.  Furthermore, the remotely backup is not guaranteed to be
available.

Therefore, you should use the --download or --filename options, or use
"juju backups download", to get a local copy of the backup archive.
This local copy can then be used to restore an environment even if that
environment was already destroyed or is otherwise unavailable.
`

// CreateCommand is the sub-command for creating a new backup.
type CreateCommand struct {
	CommandBase
	// Quiet indicates that the full metadata should not be dumped.
	Quiet bool
	// NoDownload means the backups archive should not be downloaded.
	NoDownload bool
	// Filename is where the backup should be downloaded.
	Filename string
	// Notes is the custom message to associated with the new backup.
	Notes string
}

// Info implements Command.Info.
func (c *CreateCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "create",
		Args:    "[<notes>]",
		Purpose: "create a backup",
		Doc:     createDoc,
	}
}

// SetFlags implements Command.SetFlags.
func (c *CreateCommand) SetFlags(f *gnuflag.FlagSet) {
	f.BoolVar(&c.Quiet, "quiet", false, "do not print the metadata")
	f.BoolVar(&c.NoDownload, "no-download", false, "do not download the archive")
	f.StringVar(&c.Filename, "filename", notset, "download to this file")
}

// Init implements Command.Init.
func (c *CreateCommand) Init(args []string) error {
	notes, err := cmd.ZeroOrOneArgs(args)
	if err != nil {
		return err
	}
	c.Notes = notes

	if c.Filename != notset && c.NoDownload {
		return errors.Errorf("cannot mix --no-download and --filename")
	}
	if c.Filename == "" {
		return errors.Errorf("missing filename")
	}

	return nil
}

// Run implements Command.Run.
func (c *CreateCommand) Run(ctx *cmd.Context) error {
	client, err := c.NewAPIClient()
	if err != nil {
		return errors.Trace(err)
	}
	defer client.Close()

	result, err := client.Create(c.Notes)
	if err != nil {
		return errors.Trace(err)
	}

	if !c.Quiet {
		if c.NoDownload {
			fmt.Fprintln(ctx.Stderr, downloadWarning)
		}
		c.dumpMetadata(ctx, result)
	}

	fmt.Fprintln(ctx.Stdout, result.ID)

	// Handle download.
	filename := c.decideFilename(ctx, c.Filename, result.Started)
	if filename != "" {
		if err := c.download(ctx, client, result.ID, filename); err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

func (c *CreateCommand) decideFilename(ctx *cmd.Context, filename string, timestamp time.Time) string {
	if filename != notset {
		return filename
	}
	if c.NoDownload {
		return ""
	}

	// Downloading but no filename given, so generate one.
	y, m, d := timestamp.Date()
	H, M, S := timestamp.Clock()
	return fmt.Sprintf(filenameTemplate, y, m, d, H, M, S)
}

func (c *CreateCommand) download(ctx *cmd.Context, client APIClient, id string, filename string) error {
	fmt.Fprintln(ctx.Stdout, "downloading to "+filename)

	archive, err := client.Download(id)
	if err != nil {
		return errors.Trace(err)
	}
	defer archive.Close()

	outfile, err := os.Create(filename)
	if err != nil {
		return errors.Trace(err)
	}
	defer outfile.Close()

	_, err = io.Copy(outfile, archive)
	return errors.Trace(err)
}
<commit_msg>Use Time.Format().<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"fmt"
	"io"
	"os"
	"time"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"launchpad.net/gnuflag"
)

const (
	notset           = "juju-backup-<date>-<time>.tar.gz"
	filenameTemplate = "juju-backup-20060102-150405.tar.gz"
	downloadWarning  = "WARNING: downloading backup archives is recommended; " +
		"backups stored remotely are not guaranteed to be available"
)

const createDoc = `
"create" requests that juju create a backup of its state and print the
backup's unique ID.  You may provide a note to associate with the backup.

The backup archive and associated metadata are stored remotely by juju.

The --download option may be used without the --filename option.  In
that case, the backup archive will be stored in the current working
directory with a name matching juju-backup-<date>-<time>.tar.gz.

WARNING: Remotely stored backups will be lost when the environment is
destroyed.  Furthermore, the remotely backup is not guaranteed to be
available.

Therefore, you should use the --download or --filename options, or use
"juju backups download", to get a local copy of the backup archive.
This local copy can then be used to restore an environment even if that
environment was already destroyed or is otherwise unavailable.
`

// CreateCommand is the sub-command for creating a new backup.
type CreateCommand struct {
	CommandBase
	// Quiet indicates that the full metadata should not be dumped.
	Quiet bool
	// NoDownload means the backups archive should not be downloaded.
	NoDownload bool
	// Filename is where the backup should be downloaded.
	Filename string
	// Notes is the custom message to associated with the new backup.
	Notes string
}

// Info implements Command.Info.
func (c *CreateCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "create",
		Args:    "[<notes>]",
		Purpose: "create a backup",
		Doc:     createDoc,
	}
}

// SetFlags implements Command.SetFlags.
func (c *CreateCommand) SetFlags(f *gnuflag.FlagSet) {
	f.BoolVar(&c.Quiet, "quiet", false, "do not print the metadata")
	f.BoolVar(&c.NoDownload, "no-download", false, "do not download the archive")
	f.StringVar(&c.Filename, "filename", notset, "download to this file")
}

// Init implements Command.Init.
func (c *CreateCommand) Init(args []string) error {
	notes, err := cmd.ZeroOrOneArgs(args)
	if err != nil {
		return err
	}
	c.Notes = notes

	if c.Filename != notset && c.NoDownload {
		return errors.Errorf("cannot mix --no-download and --filename")
	}
	if c.Filename == "" {
		return errors.Errorf("missing filename")
	}

	return nil
}

// Run implements Command.Run.
func (c *CreateCommand) Run(ctx *cmd.Context) error {
	client, err := c.NewAPIClient()
	if err != nil {
		return errors.Trace(err)
	}
	defer client.Close()

	result, err := client.Create(c.Notes)
	if err != nil {
		return errors.Trace(err)
	}

	if !c.Quiet {
		if c.NoDownload {
			fmt.Fprintln(ctx.Stderr, downloadWarning)
		}
		c.dumpMetadata(ctx, result)
	}

	fmt.Fprintln(ctx.Stdout, result.ID)

	// Handle download.
	filename := c.decideFilename(ctx, c.Filename, result.Started)
	if filename != "" {
		if err := c.download(ctx, client, result.ID, filename); err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

func (c *CreateCommand) decideFilename(ctx *cmd.Context, filename string, timestamp time.Time) string {
	if filename != notset {
		return filename
	}
	if c.NoDownload {
		return ""
	}

	// Downloading but no filename given, so generate one.
	return timestamp.Format(filenameTemplate)
}

func (c *CreateCommand) download(ctx *cmd.Context, client APIClient, id string, filename string) error {
	fmt.Fprintln(ctx.Stdout, "downloading to "+filename)

	archive, err := client.Download(id)
	if err != nil {
		return errors.Trace(err)
	}
	defer archive.Close()

	outfile, err := os.Create(filename)
	if err != nil {
		return errors.Trace(err)
	}
	defer outfile.Close()

	_, err = io.Copy(outfile, archive)
	return errors.Trace(err)
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"net/http"

	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/dummy"
	"launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
)

type BootstrapSuite struct {
	testing.LoggingSuite
	testing.MgoSuite
}

var _ = Suite(&BootstrapSuite{})

func (s *BootstrapSuite) SetUpSuite(c *C) {
	s.LoggingSuite.SetUpSuite(c)
	s.MgoSuite.SetUpSuite(c)
}

func (s *BootstrapSuite) SetUpTest(c *C) {
	s.LoggingSuite.SetUpTest(c)
	s.MgoSuite.SetUpTest(c)
}

func (s *BootstrapSuite) TearDownSuite(c *C) {
	s.MgoSuite.TearDownSuite(c)
	s.LoggingSuite.TearDownSuite(c)
}

func (s *BootstrapSuite) TearDownTest(c *C) {
	s.MgoSuite.TearDownTest(c)
	s.LoggingSuite.TearDownTest(c)
	dummy.Reset()
}

func (*BootstrapSuite) TestBootstrapCommand(c *C) {
	defer makeFakeHome(c, "brokenenv").restore()
	err := ioutil.WriteFile(homePath(".juju", "environments.yaml"), []byte(envConfig), 0666)
	c.Assert(err, IsNil)

	// normal bootstrap
	opc, errc := runCommand(new(BootstrapCommand))
	c.Check(<-errc, IsNil)
	c.Check((<-opc).(dummy.OpBootstrap).Env, Equals, "peckham")

	// Check that the CA certificate has been automatically generated
	// for the environment.
	// TODO(rog) reenable
	//_, err = os.Stat(homePath(".juju", "peckham.pem"))
	//c.Assert(err, IsNil)

	// bootstrap with tool uploading - checking that a file
	// is uploaded should be sufficient, as the detailed semantics
	// of UploadTools are tested in environs.
	opc, errc = runCommand(new(BootstrapCommand), "--upload-tools")
	c.Check(<-errc, IsNil)
	c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham")
	c.Check((<-opc).(dummy.OpBootstrap).Env, Equals, "peckham")

	envs, err := environs.ReadEnvirons("")
	c.Assert(err, IsNil)
	env, err := envs.Open("peckham")
	c.Assert(err, IsNil)

	tools, err := environs.FindTools(env, version.Current, environs.CompatVersion)
	c.Assert(err, IsNil)
	resp, err := http.Get(tools.URL)
	c.Assert(err, IsNil)
	defer resp.Body.Close()

	err = environs.UnpackTools(c.MkDir(), tools, resp.Body)
	c.Assert(err, IsNil)

	// bootstrap with broken environment
	opc, errc = runCommand(new(BootstrapCommand), "-e", "brokenenv")
	c.Check(<-errc, ErrorMatches, "dummy.Bootstrap is broken")
	c.Check(<-opc, IsNil)
}
<commit_msg>cmd/juju: reenable bootstrap key-generation check<commit_after>package main

import (
	"io/ioutil"
	"net/http"

	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/dummy"
	"launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
)

type BootstrapSuite struct {
	testing.LoggingSuite
	testing.MgoSuite
}

var _ = Suite(&BootstrapSuite{})

func (s *BootstrapSuite) SetUpSuite(c *C) {
	s.LoggingSuite.SetUpSuite(c)
	s.MgoSuite.SetUpSuite(c)
}

func (s *BootstrapSuite) SetUpTest(c *C) {
	s.LoggingSuite.SetUpTest(c)
	s.MgoSuite.SetUpTest(c)
}

func (s *BootstrapSuite) TearDownSuite(c *C) {
	s.MgoSuite.TearDownSuite(c)
	s.LoggingSuite.TearDownSuite(c)
}

func (s *BootstrapSuite) TearDownTest(c *C) {
	s.MgoSuite.TearDownTest(c)
	s.LoggingSuite.TearDownTest(c)
	dummy.Reset()
}

func (*BootstrapSuite) TestBootstrapCommand(c *C) {
	defer makeFakeHome(c, "brokenenv").restore()
	err := ioutil.WriteFile(homePath(".juju", "environments.yaml"), []byte(envConfig), 0666)
	c.Assert(err, IsNil)

	// normal bootstrap
	opc, errc := runCommand(new(BootstrapCommand))
	c.Check(<-errc, IsNil)
	c.Check((<-opc).(dummy.OpBootstrap).Env, Equals, "peckham")

	// Check that the CA certificate and key have been automatically generated
	// for the environment.
	_, err = os.Stat(homePath(".juju", "peckham-cert.pem"))
	c.Assert(err, IsNil)
	_, err = os.Stat(homePath(".juju", "peckham-private-key.pem"))
	c.Assert(err, IsNil)

	// bootstrap with tool uploading - checking that a file
	// is uploaded should be sufficient, as the detailed semantics
	// of UploadTools are tested in environs.
	opc, errc = runCommand(new(BootstrapCommand), "--upload-tools")
	c.Check(<-errc, IsNil)
	c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham")
	c.Check((<-opc).(dummy.OpBootstrap).Env, Equals, "peckham")

	envs, err := environs.ReadEnvirons("")
	c.Assert(err, IsNil)
	env, err := envs.Open("peckham")
	c.Assert(err, IsNil)

	tools, err := environs.FindTools(env, version.Current, environs.CompatVersion)
	c.Assert(err, IsNil)
	resp, err := http.Get(tools.URL)
	c.Assert(err, IsNil)
	defer resp.Body.Close()

	err = environs.UnpackTools(c.MkDir(), tools, resp.Body)
	c.Assert(err, IsNil)

	// bootstrap with broken environment
	opc, errc = runCommand(new(BootstrapCommand), "-e", "brokenenv")
	c.Check(<-errc, ErrorMatches, "dummy.Bootstrap is broken")
	c.Check(<-opc, IsNil)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"k8s.io/kops/upup/pkg/api"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kops/upup/pkg/fi/utils"
	"k8s.io/kops/upup/pkg/kutil"
	"k8s.io/kubernetes/pkg/util/sets"
	"os"
	"strings"
)

type CreateClusterCmd struct {
	Yes                      bool
	Target                   string
	Models                   string
	Cloud                    string
	Zones                    string
	MasterZones              string
	NodeSize                 string
	MasterSize               string
	NodeCount                int
	Project                  string
	KubernetesVersion        string
	OutDir                   string
	Image                    string
	SSHPublicKey             string
	VPCID                    string
	NetworkCIDR              string
	DNSZone                  string
	AdminAccess              string
	DisableAssociatePublicIP bool
}

var createCluster CreateClusterCmd

func init() {
	cmd := &cobra.Command{
		Use:   "cluster",
		Short: "Create cluster",
		Long:  `Creates a k8s cluster.`,
		Run: func(cmd *cobra.Command, args []string) {
			err := createCluster.Run(args)
			if err != nil {
				glog.Exitf("%v", err)
			}
		},
	}

	createCmd.AddCommand(cmd)

	cmd.Flags().BoolVar(&createCluster.Yes, "yes", false, "Specify --yes to immediately create the cluster")
	cmd.Flags().StringVar(&createCluster.Target, "target", cloudup.TargetDirect, "Target - direct, terraform")
	cmd.Flags().StringVar(&createCluster.Models, "model", "config,proto,cloudup", "Models to apply (separate multiple models with commas)")

	cmd.Flags().StringVar(&createCluster.Cloud, "cloud", "", "Cloud provider to use - gce, aws")

	cmd.Flags().StringVar(&createCluster.Zones, "zones", "", "Zones in which to run the cluster")
	cmd.Flags().StringVar(&createCluster.MasterZones, "master-zones", "", "Zones in which to run masters (must be an odd number)")

	cmd.Flags().StringVar(&createCluster.Project, "project", "", "Project to use (must be set on GCE)")
	cmd.Flags().StringVar(&createCluster.KubernetesVersion, "kubernetes-version", "", "Version of kubernetes to run (defaults to latest)")

	cmd.Flags().StringVar(&createCluster.SSHPublicKey, "ssh-public-key", "~/.ssh/id_rsa.pub", "SSH public key to use")

	cmd.Flags().StringVar(&createCluster.NodeSize, "node-size", "", "Set instance size for nodes")

	cmd.Flags().StringVar(&createCluster.MasterSize, "master-size", "", "Set instance size for masters")

	cmd.Flags().StringVar(&createCluster.VPCID, "vpc", "", "Set to use a shared VPC")
	cmd.Flags().StringVar(&createCluster.NetworkCIDR, "network-cidr", "", "Set to override the default network CIDR")

	cmd.Flags().IntVar(&createCluster.NodeCount, "node-count", 0, "Set the number of nodes")

	cmd.Flags().StringVar(&createCluster.Image, "image", "", "Image to use")

	cmd.Flags().StringVar(&createCluster.DNSZone, "dns-zone", "", "DNS hosted zone to use (defaults to last two components of cluster name)")
	cmd.Flags().StringVar(&createCluster.OutDir, "out", "", "Path to write any local output")
	cmd.Flags().StringVar(&createCluster.AdminAccess, "admin-access", "", "Restrict access to admin endpoints (SSH, HTTPS) to this CIDR.  If not set, access will not be restricted by IP.")

	cmd.Flags().BoolVar(&createCluster.DisableAssociatePublicIP, "disable-associate-public-ip", false, "Specify --disable-associate-public-ip to disable association of public IP for master ASG and nodes.")
}

func (c *CreateClusterCmd) Run(args []string) error {
	err := rootCommand.ProcessArgs(args)
	if err != nil {
		return err
	}

	isDryrun := false
	// direct requires --yes (others do not, because they don't make changes)
	if c.Target == cloudup.TargetDirect {
		if !c.Yes {
			isDryrun = true
			c.Target = cloudup.TargetDryRun
		}
	}
	if c.Target == cloudup.TargetDryRun {
		isDryrun = true
		c.Target = cloudup.TargetDryRun
	}

	clusterName := rootCommand.clusterName
	if clusterName == "" {
		return fmt.Errorf("--name is required")
	}

	// TODO: Reuse rootCommand stateStore logic?

	if c.OutDir == "" {
		c.OutDir = "out"
	}

	clusterRegistry, err := rootCommand.ClusterRegistry()
	if err != nil {
		return err
	}

	cluster, err := clusterRegistry.Find(clusterName)
	if err != nil {
		return err
	}

	if cluster != nil {
		return fmt.Errorf("cluster %q already exists; use 'kops update cluster' to apply changes", clusterName)
	}

	cluster = &api.Cluster{}
	var instanceGroups []*api.InstanceGroup

	if c.Zones != "" {
		existingZones := make(map[string]*api.ClusterZoneSpec)
		for _, zone := range cluster.Spec.Zones {
			existingZones[zone.Name] = zone
		}

		for _, zone := range parseZoneList(c.Zones) {
			if existingZones[zone] == nil {
				cluster.Spec.Zones = append(cluster.Spec.Zones, &api.ClusterZoneSpec{
					Name: zone,
				})
			}
		}
	}

	if len(cluster.Spec.Zones) == 0 {
		return fmt.Errorf("must specify at least one zone for the cluster (use --zones)")
	}

	var masters []*api.InstanceGroup
	var nodes []*api.InstanceGroup

	for _, group := range instanceGroups {
		if group.IsMaster() {
			masters = append(masters, group)
		} else {
			nodes = append(nodes, group)
		}
	}

	if c.MasterZones == "" {
		if len(masters) == 0 {
			// We default to single-master (not HA), unless the user explicitly specifies it
			// HA master is a little slower, not as well tested yet, and requires more resources
			// Probably best not to make it the silent default!
			for _, zone := range cluster.Spec.Zones {
				g := &api.InstanceGroup{}
				g.Spec.Role = api.InstanceGroupRoleMaster
				g.Spec.Zones = []string{zone.Name}
				g.Spec.MinSize = fi.Int(1)
				g.Spec.MaxSize = fi.Int(1)
				g.Name = "master-" + zone.Name // Subsequent masters (if we support that) could be <zone>-1, <zone>-2
				instanceGroups = append(instanceGroups, g)
				masters = append(masters, g)

				// Don't force HA master
				break
			}
		}
	} else {
		if len(masters) == 0 {
			// Use the specified master zones (this is how the user gets HA master)
			for _, zone := range parseZoneList(c.MasterZones) {
				g := &api.InstanceGroup{}
				g.Spec.Role = api.InstanceGroupRoleMaster
				g.Spec.Zones = []string{zone}
				g.Spec.MinSize = fi.Int(1)
				g.Spec.MaxSize = fi.Int(1)
				g.Name = "master-" + zone
				instanceGroups = append(instanceGroups, g)
				masters = append(masters, g)
			}
		} else {
			// This is hard, because of the etcd cluster
			glog.Errorf("Cannot change master-zones from the CLI")
			os.Exit(1)
		}
	}

	if len(cluster.Spec.EtcdClusters) == 0 {
		zones := sets.NewString()
		for _, group := range instanceGroups {
			for _, zone := range group.Spec.Zones {
				zones.Insert(zone)
			}
		}
		etcdZones := zones.List()

		for _, etcdCluster := range cloudup.EtcdClusters {
			etcd := &api.EtcdClusterSpec{}
			etcd.Name = etcdCluster
			for _, zone := range etcdZones {
				m := &api.EtcdMemberSpec{}
				m.Name = zone
				m.Zone = zone
				etcd.Members = append(etcd.Members, m)
			}
			cluster.Spec.EtcdClusters = append(cluster.Spec.EtcdClusters, etcd)
		}
	}

	if len(nodes) == 0 {
		g := &api.InstanceGroup{}
		g.Spec.Role = api.InstanceGroupRoleNode
		g.Name = "nodes"
		instanceGroups = append(instanceGroups, g)
		nodes = append(nodes, g)
	}

	if c.NodeSize != "" {
		for _, group := range nodes {
			group.Spec.MachineType = c.NodeSize
		}
	}

	if c.Image != "" {
		for _, group := range instanceGroups {
			group.Spec.Image = c.Image
		}
	}

	if c.NodeCount != 0 {
		for _, group := range nodes {
			group.Spec.MinSize = fi.Int(c.NodeCount)
			group.Spec.MaxSize = fi.Int(c.NodeCount)
		}
	}

	if c.MasterSize != "" {
		for _, group := range masters {
			group.Spec.MachineType = c.MasterSize
		}
	}

	if c.DNSZone != "" {
		cluster.Spec.DNSZone = c.DNSZone
	}

	if c.Cloud != "" {
		cluster.Spec.CloudProvider = c.Cloud
	}

	if c.Project != "" {
		cluster.Spec.Project = c.Project
	}

	if clusterName != "" {
		cluster.Name = clusterName
	}

	if c.KubernetesVersion != "" {
		cluster.Spec.KubernetesVersion = c.KubernetesVersion
	}

	if c.VPCID != "" {
		cluster.Spec.NetworkID = c.VPCID
	}

	if c.NetworkCIDR != "" {
		cluster.Spec.NetworkCIDR = c.NetworkCIDR
	}

	if cluster.SharedVPC() && cluster.Spec.NetworkCIDR == "" {
		glog.Errorf("Must specify NetworkCIDR when VPC is set")
		os.Exit(1)
	}

	if cluster.Spec.CloudProvider == "" {
		for _, zone := range cluster.Spec.Zones {
			cloud, known := fi.GuessCloudForZone(zone.Name)
			if known {
				glog.Infof("Inferred --cloud=%s from zone %q", cloud, zone.Name)
				cluster.Spec.CloudProvider = string(cloud)
				break
			}
		}
	}

	if c.SSHPublicKey != "" {
		c.SSHPublicKey = utils.ExpandPath(c.SSHPublicKey)
	}

	if c.AdminAccess != "" {
		cluster.Spec.AdminAccess = []string{c.AdminAccess}
	}

	err = cluster.PerformAssignments()
	if err != nil {
		return fmt.Errorf("error populating configuration: %v", err)
	}
	err = api.PerformAssignmentsInstanceGroups(instanceGroups)
	if err != nil {
		return fmt.Errorf("error populating configuration: %v", err)
	}

	strict := false
	err = api.DeepValidate(cluster, instanceGroups, strict)
	if err != nil {
		return err
	}

	fullCluster, err := cloudup.PopulateClusterSpec(cluster, clusterRegistry)
	if err != nil {
		return err
	}

	var fullInstanceGroups []*api.InstanceGroup
	for _, group := range instanceGroups {
		fullGroup, err := cloudup.PopulateInstanceGroupSpec(fullCluster, group)
		if err != nil {
			return err
		}
		fullInstanceGroups = append(fullInstanceGroups, fullGroup)
	}

	err = api.DeepValidate(fullCluster, fullInstanceGroups, true)
	if err != nil {
		return err
	}

	// Note we perform as much validation as we can, before writing a bad config
	err = api.CreateClusterConfig(clusterRegistry, cluster, fullInstanceGroups)
	if err != nil {
		return fmt.Errorf("error writing updated configuration: %v", err)
	}

	err = clusterRegistry.WriteCompletedConfig(fullCluster)
	if err != nil {
		return fmt.Errorf("error writing completed cluster spec: %v", err)
	}

	if isDryrun {
		fmt.Println("Previewing changes that will be made:\n")
	}

	if c.DisableAssociatePublicIP {
		glog.V(1).Info("Disable associate public IP: %v", c.DisableAssociatePublicIP)
		associatePublicIp := false
		fullCluster.Spec.AssociatePublicIP = associatePublicIp
	}

	applyCmd := &cloudup.ApplyClusterCmd{
		Cluster:         fullCluster,
		InstanceGroups:  fullInstanceGroups,
		Models:          strings.Split(c.Models, ","),
		ClusterRegistry: clusterRegistry,
		Target:          c.Target,
		SSHPublicKey:    c.SSHPublicKey,
		OutDir:          c.OutDir,
		DryRun:          isDryrun,
	}

	err = applyCmd.Run()
	if err != nil {
		return err
	}

	if isDryrun {
		fmt.Printf("\n")
		fmt.Printf("Cluster configuration has been created.\n")
		fmt.Printf("\n")
		fmt.Printf("Suggestions:\n")
		fmt.Printf(" * list clusters with: kops get cluster\n")
		fmt.Printf(" * edit this cluster with: kops edit cluster %s\n", clusterName)
		if len(nodes) > 0 {
			fmt.Printf(" * edit your node instance group: kops edit ig --name=%s %s\n", clusterName, nodes[0].Name)
		}
		if len(masters) > 0 {
			fmt.Printf(" * edit your master instance group: kops edit ig --name=%s %s\n", clusterName, masters[0].Name)
		}
		fmt.Printf("\n")
		fmt.Printf("Finally configure your cluster with: kops update cluster %s --yes\n", clusterName)
		fmt.Printf("\n")
	} else {
		glog.Infof("Exporting kubecfg for cluster")

		x := &kutil.CreateKubecfg{
			ClusterName:      cluster.Name,
			KeyStore:         clusterRegistry.KeyStore(cluster.Name),
			MasterPublicName: cluster.Spec.MasterPublicName,
		}
		defer x.Close()

		err = x.WriteKubecfg()
		if err != nil {
			return err
		}
	}

	return nil
}

func parseZoneList(s string) []string {
	var filtered []string
	for _, v := range strings.Split(s, ",") {
		v = strings.TrimSpace(v)
		if v == "" {
			continue
		}
		v = strings.ToLower(v)
		filtered = append(filtered, v)
	}
	return filtered
}
<commit_msg>add parameter --disable-associate-public-ip<commit_after>package main

import (
	"fmt"
	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"k8s.io/kops/upup/pkg/api"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kops/upup/pkg/fi/utils"
	"k8s.io/kops/upup/pkg/kutil"
	"k8s.io/kubernetes/pkg/util/sets"
	"os"
	"strings"
)

type CreateClusterCmd struct {
	Yes                      bool
	Target                   string
	Models                   string
	Cloud                    string
	Zones                    string
	MasterZones              string
	NodeSize                 string
	MasterSize               string
	NodeCount                int
	Project                  string
	KubernetesVersion        string
	OutDir                   string
	Image                    string
	SSHPublicKey             string
	VPCID                    string
	NetworkCIDR              string
	DNSZone                  string
	AdminAccess              string
	DisableAssociatePublicIP bool
}

var createCluster CreateClusterCmd

func init() {
	cmd := &cobra.Command{
		Use:   "cluster",
		Short: "Create cluster",
		Long:  `Creates a k8s cluster.`,
		Run: func(cmd *cobra.Command, args []string) {
			err := createCluster.Run(args)
			if err != nil {
				glog.Exitf("%v", err)
			}
		},
	}

	createCmd.AddCommand(cmd)

	cmd.Flags().BoolVar(&createCluster.Yes, "yes", false, "Specify --yes to immediately create the cluster")
	cmd.Flags().StringVar(&createCluster.Target, "target", cloudup.TargetDirect, "Target - direct, terraform")
	cmd.Flags().StringVar(&createCluster.Models, "model", "config,proto,cloudup", "Models to apply (separate multiple models with commas)")

	cmd.Flags().StringVar(&createCluster.Cloud, "cloud", "", "Cloud provider to use - gce, aws")

	cmd.Flags().StringVar(&createCluster.Zones, "zones", "", "Zones in which to run the cluster")
	cmd.Flags().StringVar(&createCluster.MasterZones, "master-zones", "", "Zones in which to run masters (must be an odd number)")

	cmd.Flags().StringVar(&createCluster.Project, "project", "", "Project to use (must be set on GCE)")
	cmd.Flags().StringVar(&createCluster.KubernetesVersion, "kubernetes-version", "", "Version of kubernetes to run (defaults to latest)")

	cmd.Flags().StringVar(&createCluster.SSHPublicKey, "ssh-public-key", "~/.ssh/id_rsa.pub", "SSH public key to use")

	cmd.Flags().StringVar(&createCluster.NodeSize, "node-size", "", "Set instance size for nodes")

	cmd.Flags().StringVar(&createCluster.MasterSize, "master-size", "", "Set instance size for masters")

	cmd.Flags().StringVar(&createCluster.VPCID, "vpc", "", "Set to use a shared VPC")
	cmd.Flags().StringVar(&createCluster.NetworkCIDR, "network-cidr", "", "Set to override the default network CIDR")

	cmd.Flags().IntVar(&createCluster.NodeCount, "node-count", 0, "Set the number of nodes")

	cmd.Flags().StringVar(&createCluster.Image, "image", "", "Image to use")

	cmd.Flags().StringVar(&createCluster.DNSZone, "dns-zone", "", "DNS hosted zone to use (defaults to last two components of cluster name)")
	cmd.Flags().StringVar(&createCluster.OutDir, "out", "", "Path to write any local output")
	cmd.Flags().StringVar(&createCluster.AdminAccess, "admin-access", "", "Restrict access to admin endpoints (SSH, HTTPS) to this CIDR.  If not set, access will not be restricted by IP.")

	cmd.Flags().BoolVar(&createCluster.DisableAssociatePublicIP, "disable-associate-public-ip", false, "Specify --disable-associate-public-ip to disable association of public IP for master ASG and nodes.")
}

func (c *CreateClusterCmd) Run(args []string) error {
	err := rootCommand.ProcessArgs(args)
	if err != nil {
		return err
	}

	isDryrun := false
	// direct requires --yes (others do not, because they don't make changes)
	if c.Target == cloudup.TargetDirect {
		if !c.Yes {
			isDryrun = true
			c.Target = cloudup.TargetDryRun
		}
	}
	if c.Target == cloudup.TargetDryRun {
		isDryrun = true
		c.Target = cloudup.TargetDryRun
	}

	clusterName := rootCommand.clusterName
	if clusterName == "" {
		return fmt.Errorf("--name is required")
	}

	// TODO: Reuse rootCommand stateStore logic?

	if c.OutDir == "" {
		c.OutDir = "out"
	}

	clusterRegistry, err := rootCommand.ClusterRegistry()
	if err != nil {
		return err
	}

	cluster, err := clusterRegistry.Find(clusterName)
	if err != nil {
		return err
	}

	if cluster != nil {
		return fmt.Errorf("cluster %q already exists; use 'kops update cluster' to apply changes", clusterName)
	}

	cluster = &api.Cluster{}
	var instanceGroups []*api.InstanceGroup

	if c.Zones != "" {
		existingZones := make(map[string]*api.ClusterZoneSpec)
		for _, zone := range cluster.Spec.Zones {
			existingZones[zone.Name] = zone
		}

		for _, zone := range parseZoneList(c.Zones) {
			if existingZones[zone] == nil {
				cluster.Spec.Zones = append(cluster.Spec.Zones, &api.ClusterZoneSpec{
					Name: zone,
				})
			}
		}
	}

	if len(cluster.Spec.Zones) == 0 {
		return fmt.Errorf("must specify at least one zone for the cluster (use --zones)")
	}

	var masters []*api.InstanceGroup
	var nodes []*api.InstanceGroup

	for _, group := range instanceGroups {
		if group.IsMaster() {
			masters = append(masters, group)
		} else {
			nodes = append(nodes, group)
		}
	}

	if c.MasterZones == "" {
		if len(masters) == 0 {
			// We default to single-master (not HA), unless the user explicitly specifies it
			// HA master is a little slower, not as well tested yet, and requires more resources
			// Probably best not to make it the silent default!
			for _, zone := range cluster.Spec.Zones {
				g := &api.InstanceGroup{}
				g.Spec.Role = api.InstanceGroupRoleMaster
				g.Spec.Zones = []string{zone.Name}
				g.Spec.MinSize = fi.Int(1)
				g.Spec.MaxSize = fi.Int(1)
				g.Name = "master-" + zone.Name // Subsequent masters (if we support that) could be <zone>-1, <zone>-2
				instanceGroups = append(instanceGroups, g)
				masters = append(masters, g)

				// Don't force HA master
				break
			}
		}
	} else {
		if len(masters) == 0 {
			// Use the specified master zones (this is how the user gets HA master)
			for _, zone := range parseZoneList(c.MasterZones) {
				g := &api.InstanceGroup{}
				g.Spec.Role = api.InstanceGroupRoleMaster
				g.Spec.Zones = []string{zone}
				g.Spec.MinSize = fi.Int(1)
				g.Spec.MaxSize = fi.Int(1)
				g.Name = "master-" + zone
				instanceGroups = append(instanceGroups, g)
				masters = append(masters, g)
			}
		} else {
			// This is hard, because of the etcd cluster
			glog.Errorf("Cannot change master-zones from the CLI")
			os.Exit(1)
		}
	}

	if len(cluster.Spec.EtcdClusters) == 0 {
		zones := sets.NewString()
		for _, group := range instanceGroups {
			for _, zone := range group.Spec.Zones {
				zones.Insert(zone)
			}
		}
		etcdZones := zones.List()

		for _, etcdCluster := range cloudup.EtcdClusters {
			etcd := &api.EtcdClusterSpec{}
			etcd.Name = etcdCluster
			for _, zone := range etcdZones {
				m := &api.EtcdMemberSpec{}
				m.Name = zone
				m.Zone = zone
				etcd.Members = append(etcd.Members, m)
			}
			cluster.Spec.EtcdClusters = append(cluster.Spec.EtcdClusters, etcd)
		}
	}

	if len(nodes) == 0 {
		g := &api.InstanceGroup{}
		g.Spec.Role = api.InstanceGroupRoleNode
		g.Name = "nodes"
		instanceGroups = append(instanceGroups, g)
		nodes = append(nodes, g)
	}

	if c.NodeSize != "" {
		for _, group := range nodes {
			group.Spec.MachineType = c.NodeSize
		}
	}

	if c.Image != "" {
		for _, group := range instanceGroups {
			group.Spec.Image = c.Image
		}
	}

	if c.NodeCount != 0 {
		for _, group := range nodes {
			group.Spec.MinSize = fi.Int(c.NodeCount)
			group.Spec.MaxSize = fi.Int(c.NodeCount)
		}
	}

	if c.MasterSize != "" {
		for _, group := range masters {
			group.Spec.MachineType = c.MasterSize
		}
	}

	if c.DNSZone != "" {
		cluster.Spec.DNSZone = c.DNSZone
	}

	if c.Cloud != "" {
		cluster.Spec.CloudProvider = c.Cloud
	}

	if c.Project != "" {
		cluster.Spec.Project = c.Project
	}

	if clusterName != "" {
		cluster.Name = clusterName
	}

	if c.KubernetesVersion != "" {
		cluster.Spec.KubernetesVersion = c.KubernetesVersion
	}

	if c.VPCID != "" {
		cluster.Spec.NetworkID = c.VPCID
	}

	if c.NetworkCIDR != "" {
		cluster.Spec.NetworkCIDR = c.NetworkCIDR
	}

	if cluster.SharedVPC() && cluster.Spec.NetworkCIDR == "" {
		glog.Errorf("Must specify NetworkCIDR when VPC is set")
		os.Exit(1)
	}

	if cluster.Spec.CloudProvider == "" {
		for _, zone := range cluster.Spec.Zones {
			cloud, known := fi.GuessCloudForZone(zone.Name)
			if known {
				glog.Infof("Inferred --cloud=%s from zone %q", cloud, zone.Name)
				cluster.Spec.CloudProvider = string(cloud)
				break
			}
		}
	}

	if c.SSHPublicKey != "" {
		c.SSHPublicKey = utils.ExpandPath(c.SSHPublicKey)
	}

	if c.AdminAccess != "" {
		cluster.Spec.AdminAccess = []string{c.AdminAccess}
	}

	err = cluster.PerformAssignments()
	if err != nil {
		return fmt.Errorf("error populating configuration: %v", err)
	}
	err = api.PerformAssignmentsInstanceGroups(instanceGroups)
	if err != nil {
		return fmt.Errorf("error populating configuration: %v", err)
	}

	strict := false
	err = api.DeepValidate(cluster, instanceGroups, strict)
	if err != nil {
		return err
	}

	fullCluster, err := cloudup.PopulateClusterSpec(cluster, clusterRegistry)
	if err != nil {
		return err
	}

	var fullInstanceGroups []*api.InstanceGroup
	for _, group := range instanceGroups {
		fullGroup, err := cloudup.PopulateInstanceGroupSpec(fullCluster, group)
		if err != nil {
			return err
		}
		fullInstanceGroups = append(fullInstanceGroups, fullGroup)
	}

	err = api.DeepValidate(fullCluster, fullInstanceGroups, true)
	if err != nil {
		return err
	}

	// Note we perform as much validation as we can, before writing a bad config
	err = api.CreateClusterConfig(clusterRegistry, cluster, fullInstanceGroups)
	if err != nil {
		return fmt.Errorf("error writing updated configuration: %v", err)
	}

	err = clusterRegistry.WriteCompletedConfig(fullCluster)
	if err != nil {
		return fmt.Errorf("error writing completed cluster spec: %v", err)
	}

	if isDryrun {
		fmt.Println("Previewing changes that will be made:\n")
	}

	if c.DisableAssociatePublicIP {
		glog.V(1).Info("Disable associate public IP: %v", c.DisableAssociatePublicIP)
		associatePublicIp := false
		fullCluster.Spec.AssociatePublicIP = &associatePublicIp
	}

	applyCmd := &cloudup.ApplyClusterCmd{
		Cluster:         fullCluster,
		InstanceGroups:  fullInstanceGroups,
		Models:          strings.Split(c.Models, ","),
		ClusterRegistry: clusterRegistry,
		Target:          c.Target,
		SSHPublicKey:    c.SSHPublicKey,
		OutDir:          c.OutDir,
		DryRun:          isDryrun,
	}

	err = applyCmd.Run()
	if err != nil {
		return err
	}

	if isDryrun {
		fmt.Printf("\n")
		fmt.Printf("Cluster configuration has been created.\n")
		fmt.Printf("\n")
		fmt.Printf("Suggestions:\n")
		fmt.Printf(" * list clusters with: kops get cluster\n")
		fmt.Printf(" * edit this cluster with: kops edit cluster %s\n", clusterName)
		if len(nodes) > 0 {
			fmt.Printf(" * edit your node instance group: kops edit ig --name=%s %s\n", clusterName, nodes[0].Name)
		}
		if len(masters) > 0 {
			fmt.Printf(" * edit your master instance group: kops edit ig --name=%s %s\n", clusterName, masters[0].Name)
		}
		fmt.Printf("\n")
		fmt.Printf("Finally configure your cluster with: kops update cluster %s --yes\n", clusterName)
		fmt.Printf("\n")
	} else {
		glog.Infof("Exporting kubecfg for cluster")

		x := &kutil.CreateKubecfg{
			ClusterName:      cluster.Name,
			KeyStore:         clusterRegistry.KeyStore(cluster.Name),
			MasterPublicName: cluster.Spec.MasterPublicName,
		}
		defer x.Close()

		err = x.WriteKubecfg()
		if err != nil {
			return err
		}
	}

	return nil
}

func parseZoneList(s string) []string {
	var filtered []string
	for _, v := range strings.Split(s, ",") {
		v = strings.TrimSpace(v)
		if v == "" {
			continue
		}
		v = strings.ToLower(v)
		filtered = append(filtered, v)
	}
	return filtered
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/gob"
	"fmt"
	"sync"
	"time"

	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/mdb"
	"github.com/Symantec/Dominator/lib/srpc"
	proto "github.com/Symantec/Dominator/proto/hypervisor"
)

var emptyTags = make(map[string]string)

type hypervisorGeneratorType struct {
	logger       log.DebugLogger
	eventChannel chan<- struct{}
	mutex        sync.Mutex
	vms          map[string]*proto.VmInfo
}

func newHypervisorGenerator(args []string,
	logger log.DebugLogger) (generator, error) {
	g := &hypervisorGeneratorType{
		logger: logger,
		vms:    make(map[string]*proto.VmInfo),
	}
	go g.daemon()
	return g, nil
}

func (g *hypervisorGeneratorType) daemon() {
	address := fmt.Sprintf(":%d", constants.HypervisorPortNumber)
	for {
		if err := g.getUpdates(address); err != nil {
			g.logger.Println(err)
			time.Sleep(time.Second)
		}
	}
}

func (g *hypervisorGeneratorType) getUpdates(hypervisor string) error {
	client, err := srpc.DialHTTP("tcp", hypervisor, 0)
	if err != nil {
		return err
	}
	defer client.Close()
	conn, err := client.Call("Hypervisor.GetUpdates")
	if err != nil {
		return err
	}
	defer conn.Close()
	decoder := gob.NewDecoder(conn)
	initialUpdate := true
	for {
		var update proto.Update
		if err := decoder.Decode(&update); err != nil {
			return err
		}
		g.updateVMs(update.VMs, initialUpdate)
		initialUpdate = false
		select {
		case g.eventChannel <- struct{}{}:
		default:
		}
	}
}

func (g *hypervisorGeneratorType) Generate(unused_datacentre string,
	logger log.Logger) (*mdb.Mdb, error) {
	var newMdb mdb.Mdb
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for ipAddr, vm := range g.vms {
		if vm.State == proto.StateRunning {
			tags := vm.Tags
			if tags == nil {
				tags = emptyTags
			}
			_, disableUpdates := tags["DisableUpdates"]
			var ownerGroup string
			if len(vm.OwnerGroups) > 0 {
				ownerGroup = vm.OwnerGroups[0]
			}
			newMdb.Machines = append(newMdb.Machines, mdb.Machine{
				Hostname:       ipAddr,
				IpAddress:      ipAddr,
				RequiredImage:  tags["RequiredImage"],
				PlannedImage:   tags["PlannedImage"],
				DisableUpdates: disableUpdates,
				OwnerGroup:     ownerGroup,
				Tags:           vm.Tags,
			})
		}
	}
	return &newMdb, nil
}

func (g *hypervisorGeneratorType) RegisterEventChannel(events chan<- struct{}) {
	g.eventChannel = events
}

func (g *hypervisorGeneratorType) updateVMs(vms map[string]*proto.VmInfo,
	initialUpdate bool) {
	vmsToDelete := make(map[string]struct{}, len(g.vms))
	if initialUpdate {
		for ipAddr := range g.vms {
			vmsToDelete[ipAddr] = struct{}{}
		}
	}
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for ipAddr, vm := range vms {
		if vm == nil {
			delete(g.vms, ipAddr)
		} else {
			g.vms[ipAddr] = vm
			delete(vmsToDelete, ipAddr)
		}
	}
	for ipAddr := range vmsToDelete {
		delete(g.vms, ipAddr)
	}
}
<commit_msg>Fix bug in cmd/mdbd hypervisor driver: VM deletions were being ignored.<commit_after>package main

import (
	"encoding/gob"
	"fmt"
	"sync"
	"time"

	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/mdb"
	"github.com/Symantec/Dominator/lib/srpc"
	proto "github.com/Symantec/Dominator/proto/hypervisor"
)

var emptyTags = make(map[string]string)

type hypervisorGeneratorType struct {
	logger       log.DebugLogger
	eventChannel chan<- struct{}
	mutex        sync.Mutex
	vms          map[string]*proto.VmInfo
}

func newHypervisorGenerator(args []string,
	logger log.DebugLogger) (generator, error) {
	g := &hypervisorGeneratorType{
		logger: logger,
		vms:    make(map[string]*proto.VmInfo),
	}
	go g.daemon()
	return g, nil
}

func (g *hypervisorGeneratorType) daemon() {
	address := fmt.Sprintf(":%d", constants.HypervisorPortNumber)
	for {
		if err := g.getUpdates(address); err != nil {
			g.logger.Println(err)
			time.Sleep(time.Second)
		}
	}
}

func (g *hypervisorGeneratorType) getUpdates(hypervisor string) error {
	client, err := srpc.DialHTTP("tcp", hypervisor, 0)
	if err != nil {
		return err
	}
	defer client.Close()
	conn, err := client.Call("Hypervisor.GetUpdates")
	if err != nil {
		return err
	}
	defer conn.Close()
	decoder := gob.NewDecoder(conn)
	initialUpdate := true
	for {
		var update proto.Update
		if err := decoder.Decode(&update); err != nil {
			return err
		}
		g.updateVMs(update.VMs, initialUpdate)
		initialUpdate = false
		select {
		case g.eventChannel <- struct{}{}:
		default:
		}
	}
}

func (g *hypervisorGeneratorType) Generate(unused_datacentre string,
	logger log.Logger) (*mdb.Mdb, error) {
	var newMdb mdb.Mdb
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for ipAddr, vm := range g.vms {
		if vm.State == proto.StateRunning {
			tags := vm.Tags
			if tags == nil {
				tags = emptyTags
			}
			_, disableUpdates := tags["DisableUpdates"]
			var ownerGroup string
			if len(vm.OwnerGroups) > 0 {
				ownerGroup = vm.OwnerGroups[0]
			}
			newMdb.Machines = append(newMdb.Machines, mdb.Machine{
				Hostname:       ipAddr,
				IpAddress:      ipAddr,
				RequiredImage:  tags["RequiredImage"],
				PlannedImage:   tags["PlannedImage"],
				DisableUpdates: disableUpdates,
				OwnerGroup:     ownerGroup,
				Tags:           vm.Tags,
			})
		}
	}
	return &newMdb, nil
}

func (g *hypervisorGeneratorType) RegisterEventChannel(events chan<- struct{}) {
	g.eventChannel = events
}

func (g *hypervisorGeneratorType) updateVMs(vms map[string]*proto.VmInfo,
	initialUpdate bool) {
	vmsToDelete := make(map[string]struct{}, len(g.vms))
	if initialUpdate {
		for ipAddr := range g.vms {
			vmsToDelete[ipAddr] = struct{}{}
		}
	}
	g.mutex.Lock()
	defer g.mutex.Unlock()
	for ipAddr, vm := range vms {
		if vm == nil || len(vm.Volumes) < 1 {
			delete(g.vms, ipAddr)
		} else {
			g.vms[ipAddr] = vm
			delete(vmsToDelete, ipAddr)
		}
	}
	for ipAddr := range vmsToDelete {
		delete(g.vms, ipAddr)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Command gendoc can be used for generating detailed godoc comments
// for cmdline-based tools. The user specifies the cmdline-based tool
// source file directory <dir> using the first command-line argument
// and gendoc executes the tool with flags that generate detailed
// godoc comment and output it to <dir>/doc.go. If more than one
// command-line argument is provided, they are passed through to the
// tool the gendoc executes.
//
// NOTE: The reason this command is located in under a testdata
// directory is to enforce its idiomatic use through "go run
// <path>/testdata/gendoc.go <dir> [args]".
//
// NOTE: The gendoc command itself is not based on the cmdline library
// to avoid non-trivial bootstrapping. In particular, if the
// compilation of gendoc requires GOPATH to contain the vanadium Go
// workspaces, then running the gendoc command requires the v23 tool,
// which in turn my depend on the gendoc command.
package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

func main() {
	if err := generate(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func generate() error {
	if got, want := len(os.Args[1:]), 1; got < want {
		return fmt.Errorf("gendoc requires at least one argument\nusage: gendoc <dir> [args]")
	}
	pkg := os.Args[1]

	// Build the gendoc binary in a temporary folder.
	tmpDir, err := ioutil.TempDir("", "")
	if err != nil {
		return fmt.Errorf("TempDir() failed: %v", err)
	}
	defer os.RemoveAll(tmpDir)
	gendocBin := filepath.Join(tmpDir, "gendoc")
	args := []string{"go", "build", "-o", gendocBin}
	args = append(args, pkg)
	buildCmd := exec.Command("v23", args...)
	if err := buildCmd.Run(); err != nil {
		return fmt.Errorf("%q failed: %v\n", strings.Join(buildCmd.Args, " "), err)
	}

	// Use it to generate the documentation.
	var out bytes.Buffer
	env := os.Environ()
	if len(os.Args) == 2 {
		args = []string{"help", "-style=godoc", "..."}
	} else {
		args = os.Args[2:]
		env = append(env, "CMDLINE_STYLE=godoc")
	}
	runCmd := exec.Command(gendocBin, args...)
	runCmd.Stdout = &out
	runCmd.Env = env
	if err := runCmd.Run(); err != nil {
		return fmt.Errorf("%q failed: %v\n%v\n", strings.Join(runCmd.Args, " "), err)
	}
	doc := fmt.Sprintf(`// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file was auto-generated via go generate.
// DO NOT UPDATE MANUALLY

/*
%s*/
package main
`, out.String())

	// Write the result to doc.go.
	path, perm := filepath.Join(pkg, "doc.go"), os.FileMode(0644)
	if err := ioutil.WriteFile(path, []byte(doc), perm); err != nil {
		return fmt.Errorf("WriteFile(%v, %v) failed: %v\n", path, perm, err)
	}
	return nil
}
<commit_msg>lib: Add -tags support to cmdline/testdata/gendoc<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Command gendoc can be used for generating detailed godoc comments for
// cmdline-based tools.  The user specifies the cmdline-based tool source file
// directory <dir> using the first command-line argument and gendoc executes the
// tool with flags that generate detailed godoc comment and output it to
// <dir>/doc.go.  If more than one command-line argument is provided, they are
// passed through to the tool the gendoc executes.
//
// NOTE: The reason this command is located under a testdata directory is to
// enforce its idiomatic use through "go run <path>/testdata/gendoc.go <dir>
// [args]".
//
// NOTE: The gendoc command itself is not based on the cmdline library to avoid
// non-trivial bootstrapping.  In particular, if the compilation of gendoc
// requires GOPATH to contain the vanadium Go workspaces, then running the
// gendoc command requires the v23 tool, which in turn may depend on the gendoc
// command.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

var flagTags string

func main() {
	flag.StringVar(&flagTags, "tags", "", "Tags for go build, also added as build constraints in the generated doc.go.")
	flag.Parse()
	if err := generate(flag.Args()); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func generate(args []string) error {
	if got, want := len(args), 1; got < want {
		return fmt.Errorf("gendoc requires at least one argument\nusage: gendoc <dir> [args]")
	}
	pkg, args := args[0], args[1:]

	// Build the gendoc binary in a temporary folder.
	tmpDir, err := ioutil.TempDir("", "")
	if err != nil {
		return fmt.Errorf("TempDir() failed: %v", err)
	}
	defer os.RemoveAll(tmpDir)
	gendocBin := filepath.Join(tmpDir, "gendoc")
	buildArgs := []string{"go", "build", "-a", "-tags=" + flagTags, "-o=" + gendocBin, pkg}
	buildCmd := exec.Command("v23", buildArgs...)
	if err := buildCmd.Run(); err != nil {
		return fmt.Errorf("%q failed: %v\n", strings.Join(buildCmd.Args, " "), err)
	}

	// Use it to generate the documentation.
	var tagsConstraint string
	if flagTags != "" {
		tagsConstraint = fmt.Sprintf("// +build %s\n\n", flagTags)
	}
	var out bytes.Buffer
	env := os.Environ()
	if len(args) == 0 {
		args = []string{"help", "-style=godoc", "..."}
	} else {
		env = append(env, "CMDLINE_STYLE=godoc")
	}
	runCmd := exec.Command(gendocBin, args...)
	runCmd.Stdout = &out
	runCmd.Env = env
	if err := runCmd.Run(); err != nil {
		return fmt.Errorf("%q failed: %v\n%v\n", strings.Join(runCmd.Args, " "), err)
	}
	doc := fmt.Sprintf(`// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file was auto-generated via go generate.
// DO NOT UPDATE MANUALLY

%s/*
%s*/
package main
`, tagsConstraint, out.String())

	// Write the result to doc.go.
	path, perm := filepath.Join(pkg, "doc.go"), os.FileMode(0644)
	if err := ioutil.WriteFile(path, []byte(doc), perm); err != nil {
		return fmt.Errorf("WriteFile(%v, %v) failed: %v\n", path, perm, err)
	}
	return nil
}
<|endoftext|>


<commit_before>package pathers

import (
	"fmt"
)

// GroupName is the double-form of the name of a Group, which should be enough to find the group.
type GroupName struct {
	Group   string
	Account AccountName
}

// DefaultGroup is the default group name (just the group part - don't add dots!). Defaults to "default". Wow.
var DefaultGroup = "default"

func (g GroupName) defaultIfNeeded() {
	if g.Group == "" {
		g.Group = DefaultGroup
	}
}

func (g GroupName) String() string {
	g.defaultIfNeeded()
	if g.Account == "" {
		return g.Group
	}
	return g.Group + "." + string(g.Account)
}

// GroupPath returns a Brain URL for this group, or an error if the group is
// invalid
func (g GroupName) GroupPath() (string, error) {
	g.defaultIfNeeded()
	base, err := g.AccountPath()
	return base + fmt.Sprintf("/groups/%s", g.Group)
}

// AccountPath returns a Brain URL for the account specified in this GroupName,
// or an error if it is blank
func (g GroupName) AccountPath() (string, error) {
	return g.Account.AccountPath()
}
<commit_msg>implement AccountPath on GroupName<commit_after>package pathers

import (
	"fmt"
)

// GroupName is the double-form of the name of a Group, which should be enough to find the group.
type GroupName struct {
	Group   string
	Account AccountName
}

// DefaultGroup is the default group name (just the group part - don't add dots!). Defaults to "default". Wow.
var DefaultGroup = "default"

func (g GroupName) defaultIfNeeded() {
	if g.Group == "" {
		g.Group = DefaultGroup
	}
}

func (g GroupName) String() string {
	g.defaultIfNeeded()
	if g.Account == "" {
		return g.Group
	}
	return g.Group + "." + string(g.Account)
}

// GroupPath returns a Brain URL for this group, or an error if the group is
// invalid
func (g GroupName) GroupPath() (string, error) {
	g.defaultIfNeeded()
	base, err := g.AccountPath()
	return base + fmt.Sprintf("/groups/%s", g.Group), err
}

// AccountPath returns a Brain URL for the account specified in this GroupName,
// or an error if it is blank
func (g GroupName) AccountPath() (string, error) {
	return g.Account.AccountPath()
}
<|endoftext|>