<commit_before>package emitter

import (
	"fmt"
	"net"
	"net/http"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/metric"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

type PrometheusEmitter struct {
	buildsStarted     prometheus.Counter
	buildsFinished    prometheus.Counter
	buildsSucceeded   prometheus.Counter
	buildsErrored     prometheus.Counter
	buildsFailed      prometheus.Counter
	buildsAborted     prometheus.Counter
	buildsFinishedVec *prometheus.CounterVec
	buildDurationsVec *prometheus.HistogramVec

	workerContainers *prometheus.GaugeVec
	workerVolumes    *prometheus.GaugeVec

	httpRequestsDuration *prometheus.HistogramVec

	schedulingFullDuration    *prometheus.GaugeVec
	schedulingLoadingDuration *prometheus.GaugeVec
	schedulingJobDuration     *prometheus.GaugeVec
}

type PrometheusConfig struct {
	BindIP   string `long:"prometheus-bind-ip" description:"IP to listen on to expose Prometheus metrics."`
	BindPort string `long:"prometheus-bind-port" description:"Port to listen on to expose Prometheus metrics."`
}

func init() {
	metric.RegisterEmitter(&PrometheusConfig{})
}

func (config *PrometheusConfig) Description() string { return "Prometheus" }
func (config *PrometheusConfig) IsConfigured() bool {
	return config.BindPort != "" && config.BindIP != ""
}
func (config *PrometheusConfig) bind() string {
	return fmt.Sprintf("%s:%s", config.BindIP, config.BindPort)
}

func (config *PrometheusConfig) NewEmitter() (metric.Emitter, error) {
	// build metrics
	buildsStarted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "started_total",
		Help:      "Total number of Concourse builds started.",
	})
	prometheus.MustRegister(buildsStarted)

	buildsFinished := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "finished_total",
		Help:      "Total number of Concourse builds finished.",
	})
	prometheus.MustRegister(buildsFinished)

	buildsSucceeded := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "succeeded_total",
		Help:      "Total number of Concourse builds succeeded.",
	})
	prometheus.MustRegister(buildsSucceeded)

	buildsErrored := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "errored_total",
		Help:      "Total number of Concourse builds errored.",
	})
	prometheus.MustRegister(buildsErrored)

	buildsFailed := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "failed_total",
		Help:      "Total number of Concourse builds failed.",
	})
	prometheus.MustRegister(buildsFailed)

	buildsAborted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "aborted_total",
		Help:      "Total number of Concourse builds aborted.",
	})
	prometheus.MustRegister(buildsAborted)

	buildsFinishedVec := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "finished",
			Help:      "Count of builds finished across various dimensions.",
		},
		[]string{"team", "pipeline", "status"},
	)
	prometheus.MustRegister(buildsFinishedVec)
	buildDurationsVec := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "duration_seconds",
			Help:      "Build time in seconds",
			Buckets:   []float64{1, 60, 180, 300, 600, 900, 1200, 1800, 2700, 3600, 7200, 18000, 36000},
		},
		[]string{"team", "pipeline"},
	)
	prometheus.MustRegister(buildDurationsVec)

	// worker metrics
	workerContainers := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "containers",
			Help:      "Number of containers per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerContainers)

	workerVolumes := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "volumes",
			Help:      "Number of volumes per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerVolumes)

	// http metrics
	httpRequestsDuration := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "http_responses",
			Name:      "duration_seconds",
			Help:      "Response time in seconds",
		},
		[]string{"method", "route"},
	)
	prometheus.MustRegister(httpRequestsDuration)

	// scheduling metrics
	schedulingFullDuration := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "scheduling",
			Name:      "full_duration_seconds",
			Help:      "Last time taken to schedule an entire pipeline.",
		},
		[]string{"pipeline"},
	)
	prometheus.MustRegister(schedulingFullDuration)

	schedulingLoadingDuration := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "scheduling",
			Name:      "loading_duration_seconds",
			Help:      "Last time taken to load version information from the database for a pipeline.",
		},
		[]string{"pipeline"},
	)
	prometheus.MustRegister(schedulingLoadingDuration)

	schedulingJobDuration := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "scheduling",
			Name:      "job_duration_seconds",
			Help:      "Last time taken to calculate the set of valid input versions for a pipeline.",
		},
		[]string{"pipeline"},
	)
	prometheus.MustRegister(schedulingJobDuration)

	listener, err := net.Listen("tcp", config.bind())
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, promhttp.Handler())

	return &PrometheusEmitter{
		buildsStarted:     buildsStarted,
		buildsFinished:    buildsFinished,
		buildsFinishedVec: buildsFinishedVec,
		buildDurationsVec: buildDurationsVec,
		buildsSucceeded:   buildsSucceeded,
		buildsErrored:     buildsErrored,
		buildsFailed:      buildsFailed,
		buildsAborted:     buildsAborted,

		workerContainers: workerContainers,
		workerVolumes:    workerVolumes,

		httpRequestsDuration: httpRequestsDuration,

		schedulingFullDuration:    schedulingFullDuration,
		schedulingLoadingDuration: schedulingLoadingDuration,
		schedulingJobDuration:     schedulingJobDuration,
	}, nil
}

// Emit processes incoming metrics.
// In order to provide idiomatic Prometheus metrics, we'll have to convert the various
// Event types (differentiated by the less-than-ideal string Name field) into different
// Prometheus metrics.
func (emitter *PrometheusEmitter) Emit(logger lager.Logger, event metric.Event) {
	switch event.Name {
	case "build started":
		emitter.buildsStarted.Inc()
	case "build finished":
		emitter.buildFinishedMetrics(logger, event)
	case "worker containers":
		emitter.workerContainersMetric(logger, event)
	case "worker volumes":
		emitter.workerVolumesMetric(logger, event)
	case "http response time":
		emitter.httpResponseTimeMetrics(logger, event)
	case "scheduling: full duration (ms)":
		emitter.schedulingMetrics(logger, event)
	case "scheduling: loading versions duration (ms)":
		emitter.schedulingMetrics(logger, event)
	case "scheduling: job duration (ms)":
		emitter.schedulingMetrics(logger, event)
	default:
		// unless we have a specific metric, we do nothing
	}
}

func (emitter *PrometheusEmitter) buildFinishedMetrics(logger lager.Logger, event metric.Event) {
	// concourse_builds_finished_total
	emitter.buildsFinished.Inc()

	// concourse_builds_finished
	team, exists := event.Attributes["team_name"]
	if !exists {
		logger.Error("failed-to-find-team-name-in-event", fmt.Errorf("expected team_name to exist in event.Attributes"))
	}

	pipeline, exists := event.Attributes["pipeline"]
	if !exists {
		logger.Error("failed-to-find-pipeline-in-event", fmt.Errorf("expected pipeline to exist in event.Attributes"))
	}

	buildStatus, exists := event.Attributes["build_status"]
	if !exists {
		logger.Error("failed-to-find-build_status-in-event", fmt.Errorf("expected build_status to exist in event.Attributes"))
	}
	emitter.buildsFinishedVec.WithLabelValues(team, pipeline, buildStatus).Inc()

	// concourse_builds_(aborted|succeeded|failed|errored)_total
	switch buildStatus {
	case string(db.BuildStatusAborted):
		// concourse_builds_aborted_total
		emitter.buildsAborted.Inc()
	case string(db.BuildStatusSucceeded):
		// concourse_builds_succeeded_total
		emitter.buildsSucceeded.Inc()
	case string(db.BuildStatusFailed):
		// concourse_builds_failed_total
		emitter.buildsFailed.Inc()
	case string(db.BuildStatusErrored):
		// concourse_builds_errored_total
		emitter.buildsErrored.Inc()
	}

	// concourse_builds_duration_seconds
	duration, ok := event.Value.(float64)
	if !ok {
		logger.Error("build-finished-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}
	// seconds are the standard prometheus base unit for time
	duration = duration / 1000
	emitter.buildDurationsVec.WithLabelValues(team, pipeline).Observe(duration)
}

func (emitter *PrometheusEmitter) workerContainersMetric(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	containers, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerContainers.WithLabelValues(worker).Set(float64(containers))
}

func (emitter *PrometheusEmitter) workerVolumesMetric(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	volumes, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerVolumes.WithLabelValues(worker).Set(float64(volumes))
}

func (emitter *PrometheusEmitter) httpResponseTimeMetrics(logger lager.Logger, event metric.Event) {
	route, exists := event.Attributes["route"]
	if !exists {
		logger.Error("failed-to-find-route-in-event", fmt.Errorf("expected method to exist in event.Attributes"))
	}

	method, exists := event.Attributes["method"]
	if !exists {
		logger.Error("failed-to-find-method-in-event", fmt.Errorf("expected method to exist in event.Attributes"))
	}

	responseTime, ok := event.Value.(float64)
	if !ok {
		logger.Error("http-response-time-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}

	emitter.httpRequestsDuration.WithLabelValues(method, route).Observe(responseTime / 1000)
}

func (emitter *PrometheusEmitter) schedulingMetrics(logger lager.Logger, event metric.Event) {
	pipeline, exists := event.Attributes["pipeline"]
	if !exists {
		logger.Error("failed-to-find-pipeline-in-event", fmt.Errorf("expected pipeline to exist in event.Attributes"))
	}

	duration, ok := event.Value.(float64)
	if !ok {
		logger.Error("scheduling-full-duration-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}

	switch event.Name {
	case "scheduling: full duration (ms)":
		// concourse_scheduling_full_duration_seconds
		emitter.schedulingFullDuration.WithLabelValues(pipeline).Set(duration / 1000)
	case "scheduling: loading versions duration (ms)":
		// concourse_scheduling_loading_duration_seconds
		emitter.schedulingLoadingDuration.WithLabelValues(pipeline).Set(duration / 1000)
	case "scheduling: job duration (ms)":
		// concourse_scheduling_job_duration_seconds
		emitter.schedulingJobDuration.WithLabelValues(pipeline).Set(duration / 1000)
	default:
	}
}
<commit_msg>export database metrics<commit_after>package emitter

import (
	"fmt"
	"net"
	"net/http"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/metric"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

type PrometheusEmitter struct {
	buildsStarted     prometheus.Counter
	buildsFinished    prometheus.Counter
	buildsSucceeded   prometheus.Counter
	buildsErrored     prometheus.Counter
	buildsFailed      prometheus.Counter
	buildsAborted     prometheus.Counter
	buildsFinishedVec *prometheus.CounterVec
	buildDurationsVec *prometheus.HistogramVec

	workerContainers *prometheus.GaugeVec
	workerVolumes    *prometheus.GaugeVec

	httpRequestsDuration *prometheus.HistogramVec

	schedulingFullDuration    *prometheus.GaugeVec
	schedulingLoadingDuration *prometheus.GaugeVec
	schedulingJobDuration     *prometheus.GaugeVec
}

type PrometheusConfig struct {
	BindIP   string `long:"prometheus-bind-ip" description:"IP to listen on to expose Prometheus metrics."`
	BindPort string `long:"prometheus-bind-port" description:"Port to listen on to expose Prometheus metrics."`
}

func init() {
	metric.RegisterEmitter(&PrometheusConfig{})
}

func (config *PrometheusConfig) Description() string { return "Prometheus" }
func (config *PrometheusConfig) IsConfigured() bool {
	return config.BindPort != "" && config.BindIP != ""
}
func (config *PrometheusConfig) bind() string {
	return fmt.Sprintf("%s:%s", config.BindIP, config.BindPort)
}

func (config *PrometheusConfig) NewEmitter() (metric.Emitter, error) {
	// build metrics
	buildsStarted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "started_total",
		Help:      "Total number of Concourse builds started.",
	})
	prometheus.MustRegister(buildsStarted)

	buildsFinished := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "finished_total",
		Help:      "Total number of Concourse builds finished.",
	})
	prometheus.MustRegister(buildsFinished)

	buildsSucceeded := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "succeeded_total",
		Help:      "Total number of Concourse builds succeeded.",
	})
	prometheus.MustRegister(buildsSucceeded)

	buildsErrored := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "errored_total",
		Help:      "Total number of Concourse builds errored.",
	})
	prometheus.MustRegister(buildsErrored)

	buildsFailed := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "failed_total",
		Help:      "Total number of Concourse builds failed.",
	})
	prometheus.MustRegister(buildsFailed)

	buildsAborted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "aborted_total",
		Help:      "Total number of Concourse builds aborted.",
	})
	prometheus.MustRegister(buildsAborted)

	buildsFinishedVec := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "finished",
			Help:      "Count of builds finished across various dimensions.",
		},
		[]string{"team", "pipeline", "status"},
	)
	prometheus.MustRegister(buildsFinishedVec)
	buildDurationsVec := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "duration_seconds",
			Help:      "Build time in seconds",
			Buckets:   []float64{1, 60, 180, 300, 600, 900, 1200, 1800, 2700, 3600, 7200, 18000, 36000},
		},
		[]string{"team", "pipeline"},
	)
	prometheus.MustRegister(buildDurationsVec)

	// worker metrics
	workerContainers := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "containers",
			Help:      "Number of containers per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerContainers)

	workerVolumes := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "volumes",
			Help:      "Number of volumes per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerVolumes)

	// http metrics
	httpRequestsDuration := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "http_responses",
			Name:      "duration_seconds",
			Help:      "Response time in seconds",
		},
		[]string{"method", "route"},
	)
	prometheus.MustRegister(httpRequestsDuration)

	// scheduling metrics
	schedulingFullDuration := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "scheduling",
			Name:      "full_duration_seconds",
			Help:      "Last time taken to schedule an entire pipeline.",
		},
		[]string{"pipeline"},
	)
	prometheus.MustRegister(schedulingFullDuration)

	schedulingLoadingDuration := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "scheduling",
			Name:      "loading_duration_seconds",
			Help:      "Last time taken to load version information from the database for a pipeline.",
		},
		[]string{"pipeline"},
	)
	prometheus.MustRegister(schedulingLoadingDuration)

	schedulingJobDuration := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "scheduling",
			Name:      "job_duration_seconds",
			Help:      "Last time taken to calculate the set of valid input versions for a pipeline.",
		},
		[]string{"pipeline"},
	)
	prometheus.MustRegister(schedulingJobDuration)

	// dbPromMetricsCollector defines database metrics
	prometheus.MustRegister(newDBPromCollector())

	listener, err := net.Listen("tcp", config.bind())
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, promhttp.Handler())

	return &PrometheusEmitter{
		buildsStarted:     buildsStarted,
		buildsFinished:    buildsFinished,
		buildsFinishedVec: buildsFinishedVec,
		buildDurationsVec: buildDurationsVec,
		buildsSucceeded:   buildsSucceeded,
		buildsErrored:     buildsErrored,
		buildsFailed:      buildsFailed,
		buildsAborted:     buildsAborted,

		workerContainers: workerContainers,
		workerVolumes:    workerVolumes,

		httpRequestsDuration: httpRequestsDuration,

		schedulingFullDuration:    schedulingFullDuration,
		schedulingLoadingDuration: schedulingLoadingDuration,
		schedulingJobDuration:     schedulingJobDuration,
	}, nil
}

// Emit processes incoming metrics.
// In order to provide idiomatic Prometheus metrics, we'll have to convert the various
// Event types (differentiated by the less-than-ideal string Name field) into different
// Prometheus metrics.
func (emitter *PrometheusEmitter) Emit(logger lager.Logger, event metric.Event) {
	switch event.Name {
	case "build started":
		emitter.buildsStarted.Inc()
	case "build finished":
		emitter.buildFinishedMetrics(logger, event)
	case "worker containers":
		emitter.workerContainersMetric(logger, event)
	case "worker volumes":
		emitter.workerVolumesMetric(logger, event)
	case "http response time":
		emitter.httpResponseTimeMetrics(logger, event)
	case "scheduling: full duration (ms)":
		emitter.schedulingMetrics(logger, event)
	case "scheduling: loading versions duration (ms)":
		emitter.schedulingMetrics(logger, event)
	case "scheduling: job duration (ms)":
		emitter.schedulingMetrics(logger, event)
	default:
		// unless we have a specific metric, we do nothing
	}
}

type dbPromMetricsCollector struct {
	dbConns   *prometheus.Desc
	dbQueries *prometheus.Desc
}

func newDBPromCollector() prometheus.Collector {
	return &dbPromMetricsCollector{
		dbConns: prometheus.NewDesc(
			"concourse_db_connections",
			"Current number of concourse database connections",
			[]string{"dbname"},
			nil,
		),
		// this needs to be a recent number, because it is reset every 10 seconds
		// by the periodic metrics emitter
		dbQueries: prometheus.NewDesc(
			"concourse_db_queries",
			"Recent number of Concourse database queries",
			nil,
			nil,
		),
	}
}

func (c *dbPromMetricsCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- c.dbConns
	ch <- c.dbQueries
}

func (c *dbPromMetricsCollector) Collect(ch chan<- prometheus.Metric) {
	for _, database := range metric.Databases {
		ch <- prometheus.MustNewConstMetric(
			c.dbConns,
			prometheus.GaugeValue,
			float64(database.Stats().OpenConnections),
			database.Name(),
		)
	}

	ch <- prometheus.MustNewConstMetric(
		c.dbQueries,
		prometheus.GaugeValue,
		float64(metric.DatabaseQueries),
	)
}

func (emitter *PrometheusEmitter) buildFinishedMetrics(logger lager.Logger, event metric.Event) {
	// concourse_builds_finished_total
	emitter.buildsFinished.Inc()

	// concourse_builds_finished
	team, exists := event.Attributes["team_name"]
	if !exists {
		logger.Error("failed-to-find-team-name-in-event", fmt.Errorf("expected team_name to exist in event.Attributes"))
	}

	pipeline, exists := event.Attributes["pipeline"]
	if !exists {
		logger.Error("failed-to-find-pipeline-in-event", fmt.Errorf("expected pipeline to exist in event.Attributes"))
	}

	buildStatus, exists := event.Attributes["build_status"]
	if !exists {
		logger.Error("failed-to-find-build_status-in-event", fmt.Errorf("expected build_status to exist in event.Attributes"))
	}
	emitter.buildsFinishedVec.WithLabelValues(team, pipeline, buildStatus).Inc()

	// concourse_builds_(aborted|succeeded|failed|errored)_total
	switch buildStatus {
	case string(db.BuildStatusAborted):
		// concourse_builds_aborted_total
		emitter.buildsAborted.Inc()
	case string(db.BuildStatusSucceeded):
		// concourse_builds_succeeded_total
		emitter.buildsSucceeded.Inc()
	case string(db.BuildStatusFailed):
		// concourse_builds_failed_total
		emitter.buildsFailed.Inc()
	case string(db.BuildStatusErrored):
		// concourse_builds_errored_total
		emitter.buildsErrored.Inc()
	}

	// concourse_builds_duration_seconds
	duration, ok := event.Value.(float64)
	if !ok {
		logger.Error("build-finished-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}
	// seconds are the standard prometheus base unit for time
	duration = duration / 1000
	emitter.buildDurationsVec.WithLabelValues(team, pipeline).Observe(duration)
}

func (emitter *PrometheusEmitter) workerContainersMetric(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	containers, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerContainers.WithLabelValues(worker).Set(float64(containers))
}

func (emitter *PrometheusEmitter) workerVolumesMetric(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	volumes, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerVolumes.WithLabelValues(worker).Set(float64(volumes))
}

func (emitter *PrometheusEmitter) httpResponseTimeMetrics(logger lager.Logger, event metric.Event) {
	route, exists := event.Attributes["route"]
	if !exists {
		logger.Error("failed-to-find-route-in-event", fmt.Errorf("expected method to exist in event.Attributes"))
	}

	method, exists := event.Attributes["method"]
	if !exists {
		logger.Error("failed-to-find-method-in-event", fmt.Errorf("expected method to exist in event.Attributes"))
	}

	responseTime, ok := event.Value.(float64)
	if !ok {
		logger.Error("http-response-time-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}

	emitter.httpRequestsDuration.WithLabelValues(method, route).Observe(responseTime / 1000)
}

func (emitter *PrometheusEmitter) schedulingMetrics(logger lager.Logger, event metric.Event) {
	pipeline, exists := event.Attributes["pipeline"]
	if !exists {
		logger.Error("failed-to-find-pipeline-in-event", fmt.Errorf("expected pipeline to exist in event.Attributes"))
	}

	duration, ok := event.Value.(float64)
	if !ok {
		logger.Error("scheduling-full-duration-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}

	switch event.Name {
	case "scheduling: full duration (ms)":
		// concourse_scheduling_full_duration_seconds
		emitter.schedulingFullDuration.WithLabelValues(pipeline).Set(duration / 1000)
	case "scheduling: loading versions duration (ms)":
		// concourse_scheduling_loading_duration_seconds
		emitter.schedulingLoadingDuration.WithLabelValues(pipeline).Set(duration / 1000)
	case "scheduling: job duration (ms)":
		// concourse_scheduling_job_duration_seconds
		emitter.schedulingJobDuration.WithLabelValues(pipeline).Set(duration / 1000)
	default:
	}
}
<|endoftext|>


<commit_before>package lookup

import (
	"encoding/json"
	"gopkg.in/resilient-http/resilient.go.v0/client"
	"gopkg.in/resilient-http/resilient.go.v0/middlewares"
	"io/ioutil"
	"net/http"
	"time"
)

const (
	MISSING_SERVERS = iota
)

var Error = func(msg string, code int) error {
	return middlewares.NewMiddlewareError("lookup", msg, code)
}

type Options struct {
	Servers []string
	Refresh time.Duration
	Retries int
}

type Discovery struct {
	servers       []string
	lookupServers []string
	updated       int64
	options       Options
	resilient     middlewares.Resilient
}

func New(opts Options) middlewares.Handler {
	return func(r middlewares.Resilient) (middlewares.Middleware, error) {

		if len(opts.Servers) == 0 {
			return nil, Error("Missing servers", MISSING_SERVERS)
		}

		if opts.Refresh == 0 {
			opts.Refresh = 5 * time.Minute
		}

		return &Discovery{
			servers:   opts.Servers,
			options:   opts,
			updated:   0,
			resilient: r,
		}, nil
	}
}

func (m *Discovery) Out(o *client.Options) error {
	if len(m.servers) > 0 && (time.Now().UnixNano()-m.updated) < int64(m.options.Refresh) {
		return nil
	}

	err := m.Discover()
	if err != nil {
		return err
	}

	m.resilient.UseServers(m.servers)
	return nil
}

func (m *Discovery) In(*http.Request, *http.Response, error) error {
	return nil
}

type URLs struct {
	urls []string
}

func (m *Discovery) Discover() error {
	req, err := client.New(&client.Options{
		URL: m.servers[0],
	})
	if err != nil {
		return err
	}

	res, err := req.Do()
	if err != nil {
		return err
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}

	t := []string{}
	err = json.Unmarshal(body, &t)
	if err != nil {
		return err
	}

	m.resilient.UseServers(t)

	return nil
}
<commit_msg>feat(options): add self refresh<commit_after>package lookup

import (
	"encoding/json"
	"gopkg.in/resilient-http/resilient.go.v0/client"
	"gopkg.in/resilient-http/resilient.go.v0/middlewares"
	"io/ioutil"
	"net/http"
	"time"
)

const (
	MISSING_SERVERS = iota
)

var Error = func(msg string, code int) error {
	return middlewares.NewMiddlewareError("lookup", msg, code)
}

type Options struct {
	Servers      []string
	Refresh      time.Duration
	Timeout      time.Duration
	Retries      int
	Background   bool
	SelfLookup   bool
	SelfRefresh  time.Duration
}

type Discovery struct {
	servers       []string
	lookupServers []string
	updated       int64
	options       Options
	resilient     middlewares.Resilient
}

func New(opts Options) middlewares.Handler {
	return func(r middlewares.Resilient) (middlewares.Middleware, error) {

		if len(opts.Servers) == 0 {
			return nil, Error("Missing servers", MISSING_SERVERS)
		}

		if opts.Refresh == 0 {
			opts.Refresh = 5 * time.Minute
		}

		return &Discovery{
			servers:   opts.Servers,
			options:   opts,
			updated:   0,
			resilient: r,
		}, nil
	}
}

func (m *Discovery) Out(o *client.Options) error {
	if len(m.servers) > 0 && (time.Now().UnixNano()-m.updated) < int64(m.options.Refresh) {
		return nil
	}

	err := m.Discover()
	if err != nil {
		return err
	}

	m.resilient.UseServers(m.servers)
	return nil
}

func (m *Discovery) In(*http.Request, *http.Response, error) error {
	return nil
}

type URLs struct {
	urls []string
}

func (m *Discovery) Discover() error {
	req, err := client.New(&client.Options{
		URL: m.servers[0],
	})
	if err != nil {
		return err
	}

	res, err := req.Do()
	if err != nil {
		return err
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}

	t := []string{}
	err = json.Unmarshal(body, &t)
	if err != nil {
		return err
	}

	m.resilient.UseServers(t)

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"flag"
	"log"
	"net/url"
	"os"
)

import (
	"github.com/qiniu/api/conf"
	"github.com/qiniu/api/resumable/io"
	"github.com/qiniu/api/rs"
)

type pfopRet struct {
	PersistentId string `json:"persistentId"`
	Code         int    `json:"code"`
	Error        string `json:"error"`
}

func pfop(pfops, bucket, key, notifyUrl string) (ret pfopRet, err error) {
	client := rs.New(nil)
	param := url.Values{}
	param.Set("bucket", bucket)
	param.Set("key", key)
	param.Set("fops", pfops)
	param.Set("notifyURL", notifyUrl)
	err = client.Conn.CallWithForm(nil, &ret, "http://api.qiniu.com/pfop/", param)
	return
}

func buildOps(convert, saveas string) string {
	pfops := convert
	if saveas != "" {
		pfops = pfops + "|saveas/" + base64.URLEncoding.EncodeToString([]byte(saveas))
	}
	return pfops
}

func genToken(bucket, fops, notifyUrl string) string {
	policy := rs.PutPolicy{
		Scope:               bucket,
		PersistentNotifyUrl: notifyUrl,
		PersistentOps:       fops,
	}
	return policy.Token(nil)
}

func put(token, key, file string) (ret io.PutRet, err error) {
	f, err := os.Open(file)
	if err != nil {
		log.Fatalln("file not exist")
		return
	}
	stat, err := f.Stat()
	if err != nil || stat.IsDir() {
		log.Fatalln("invalid file")
		return
	}

	blockNotify := func(blkIdx int, blkSize int, ret *io.BlkputRet) {
		log.Println("size", stat.Size(), "block id", blkIdx, "offset", ret.Offset)
	}

	params := map[string]string{}
	extra := &io.PutExtra{
		ChunkSize: 8192,
		Notify:    blockNotify,
		Params:    params,
	}

	err = io.PutFile(nil, &ret, token, key, file, extra)
	return
}

func main() {
	file := flag.String("f", "", "local file")
	bucket := flag.String("b", "", "bucket")
	key := flag.String("k", "", "file key")
	accessKey := flag.String("ak", "", "access key")
	secretKey := flag.String("sk", "", "secret key")
	convert := flag.String("c", "", "pfop single convert option avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240")
	notify := flag.String("n", "", "notify url")
	saveas := flag.String("o", "", "output key, outputBucket:key")
	flag.Parse()
	if *bucket == "" || *key == "" || *accessKey == "" || *accessKey == "" || *convert == "" {
		flag.PrintDefaults()
		log.Fatalln("invalid args")
		return
	}
	notifyUrl := *notify
	if notifyUrl == "" {
		notifyUrl = "fakenotify.com"
	}
	conf.ACCESS_KEY = *accessKey
	conf.SECRET_KEY = *secretKey
	ops := buildOps(*convert, *saveas)
	if *file != "" {
		token := genToken(*bucket, ops, notifyUrl)
		ret, err := put(token, *key, *file)
		if err != nil {
			log.Fatalln("put error", err, ret)
			return
		}
		log.Println("put ret", ret.Key)
		return
	}
	ret, err := pfop(ops, *bucket, *key, notifyUrl)
	if err != nil {
		log.Fatalln("pfop error", err, ret)
		return
	}
	log.Println("pfop id", ret.PersistentId)
}
<commit_msg>update api to 6.05<commit_after>package main

import (
	"encoding/base64"
	"flag"
	"fmt"
	"net/url"
	"os"
)

import (
	"github.com/qiniu/api/conf"
	"github.com/qiniu/api/resumable/io"
	"github.com/qiniu/api/rs"
)

type pfopRet struct {
	PersistentId string `json:"persistentId"`
	Code         int    `json:"code"`
	Error        string `json:"error"`
}

func pfop(pfops, bucket, key, notifyUrl string) (ret pfopRet, err error) {
	client := rs.New(nil)
	param := url.Values{}
	param.Set("bucket", bucket)
	param.Set("key", key)
	param.Set("fops", pfops)
	if notifyUrl != "" {
		param.Set("notifyURL", notifyUrl)
	}
	err = client.Conn.CallWithForm(nil, &ret, "http://api.qiniu.com/pfop", param)
	return
}

func buildOps(convert, saveas string) string {
	pfops := convert
	if saveas != "" {
		pfops = pfops + "|saveas/" + base64.URLEncoding.EncodeToString([]byte(saveas))
	}
	return pfops
}

func genToken(bucket, fops, notifyUrl, pipeline string) string {
	policy := rs.PutPolicy{
		Scope:         bucket,
		PersistentOps: fops,

		ReturnBody: `{"persistentId": $(persistentId)}`,
	}
	if notifyUrl != "" {
		policy.PersistentNotifyUrl = notifyUrl
	}
	if pipeline != "" {
		policy.PersistentPipeline = pipeline
	}

	return policy.Token(nil)
}

func put(token, key, file string) (ret pfopRet, err error) {
	f, err := os.Open(file)
	if err != nil {
		fmt.Fprintln(os.Stderr, "file not exist")
		os.Exit(1)
		return
	}
	stat, err := f.Stat()
	if err != nil || stat.IsDir() {
		fmt.Fprintln(os.Stderr, "invalid file")
		os.Exit(1)
		return
	}

	// blockNotify := func(blkIdx int, blkSize int, ret *io.BlkputRet) {
	// 	fmt.Println("size", stat.Size(), "block id", blkIdx, "offset", ret.Offset)
	// }

	params := map[string]string{}
	extra := &io.PutExtra{
		ChunkSize: 8192,
		// Notify:    blockNotify,
		Params: params,
	}

	err = io.PutFile(nil, &ret, token, key, file, extra)
	return
}

func main() {
	file := flag.String("f", "", "local file")
	bucket := flag.String("b", "", "bucket")
	key := flag.String("k", "", "file key")
	accessKey := flag.String("ak", "", "access key")
	secretKey := flag.String("sk", "", "secret key")
	convert := flag.String("c", "", "pfop single convert option avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240")
	notify := flag.String("n", "", "notify url")
	pipeline := flag.String("p", "", "pipline")
	saveas := flag.String("o", "", "output key, outputBucket:key")
	flag.Parse()
	if *bucket == "" || *key == "" || *accessKey == "" || *accessKey == "" || *convert == "" {
		flag.PrintDefaults()
		fmt.Fprintln(os.Stderr, "invalid args")
		os.Exit(1)
		return
	}

	conf.ACCESS_KEY = *accessKey
	conf.SECRET_KEY = *secretKey
	ops := buildOps(*convert, *saveas)
	if *file != "" {
		token := genToken(*bucket, ops, *notify, *pipeline)
		ret, err := put(token, *key, *file)
		if err != nil {
			fmt.Fprintln(os.Stderr, "put error", err, ret)
			os.Exit(1)
			return
		}
		fmt.Fprintln(os.Stdout, ret.PersistentId)
		return
	}
	ret, err := pfop(ops, *bucket, *key, *notify)
	if err != nil {
		fmt.Fprintln(os.Stderr, "pfop error", err, ret)
		os.Exit(1)
		return
	}
	fmt.Fprintln(os.Stdout, ret.PersistentId)
}
<|endoftext|>


<commit_before>// Copyright 2018 Adam Shannon
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"

	"github.com/adamdecaf/cert-manage/pkg/store"
	"github.com/adamdecaf/cert-manage/pkg/whitelist"
)

func WhitelistForApp(app, whpath string) error {
	// load whitelist
	wh, err := whitelist.FromFile(whpath)
	if err != nil {
		return err
	}

	// diff
	st, err := store.ForApp(app)
	if err != nil {
		return err
	}
	err = st.Remove(wh)
	if err != nil {
		return err
	}

	fmt.Println("Whitelist completed successfully")
	return nil
}

func WhitelistForPlatform(whpath string) error {
	// load whitelist
	wh, err := whitelist.FromFile(whpath)
	if err != nil {
		return err
	}

	// diff
	st := store.Platform()
	err = st.Remove(wh)
	if err != nil {
		return err
	}

	fmt.Println("Whitelist completed successfully")
	return nil
}
<commit_msg>cmd/whitelist: fail if we don't have a backup<commit_after>// Copyright 2018 Adam Shannon
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"runtime"

	"github.com/adamdecaf/cert-manage/pkg/store"
	"github.com/adamdecaf/cert-manage/pkg/whitelist"
)

func WhitelistForApp(app, whpath string) error {
	// load whitelist
	wh, err := whitelist.FromFile(whpath)
	if err != nil {
		return err
	}

	// diff
	s, err := store.ForApp(app)
	if err != nil {
		return err
	}

	// check for a backup
	latest, err := s.GetLatestBackup()
	if err != nil {
		return fmt.Errorf("can't get latest %s backup err=%v", app, err)
	}
	if latest == "" {
		return fmt.Errorf("no backup for %s found", app)
	}

	// perform whitelist
	err = s.Remove(wh)
	if err != nil {
		return err
	}

	fmt.Println("Whitelist completed successfully")
	return nil
}

func WhitelistForPlatform(whpath string) error {
	// load whitelist
	wh, err := whitelist.FromFile(whpath)
	if err != nil {
		return err
	}

	// diff
	s := store.Platform()

	// check for backup
	latest, err := s.GetLatestBackup()
	if err != nil {
		return fmt.Errorf("can't get latest backup for %s err=%v", runtime.GOOS, err)
	}
	if latest == "" {
		return fmt.Errorf("no %s backup found", runtime.GOOS)
	}

	// perform whitelist
	err = s.Remove(wh)
	if err != nil {
		return err
	}

	fmt.Println("Whitelist completed successfully")
	return nil
}
<|endoftext|>


<commit_before>package proxyconfig

import (
	"fmt"
	"io/ioutil"

	"github.com/prometheus/prometheus/config"

	"github.com/jacksontj/promxy/pkg/servergroup"

	yaml "gopkg.in/yaml.v2"
)

var DefaultPromxyConfig = PromxyConfig{
	BoundaryTimeWorkaround: true,
}

// ConfigFromFile loads a config file at path
func ConfigFromFile(path string) (*Config, error) {
	// load the config file
	cfg := &Config{
		PromConfig:   config.DefaultConfig,
		PromxyConfig: DefaultPromxyConfig,
	}
	configBytes, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("Error loading config: %v", err)
	}
	err = yaml.Unmarshal([]byte(configBytes), &cfg)
	if err != nil {
		return nil, fmt.Errorf("Error unmarshaling config: %v", err)
	}

	return cfg, nil
}

// Config is the entire config file. This includes both the Prometheus Config
// as well as the Promxy config. This is done by "inline-ing" the promxy
// config into the prometheus config under the "promxy" key
type Config struct {
	// Prometheus configs -- this includes configurations for
	// recording rules, alerting rules, etc.
	PromConfig config.Config `yaml:",inline"`

	// Promxy specific configuration -- under its own namespace
	PromxyConfig `yaml:"promxy"`
}

// PromxyConfig is the configuration for Promxy itself
type PromxyConfig struct {
	// Config for each of the server groups promxy is configured to aggregate
	ServerGroups []*servergroup.Config `yaml:"server_groups"`

	// BoundaryTimeWorkaround enables a workaround to prometheus' internal boundary
	// times being un-supported serverside. Newer versions of prometheus (2.11+)
	// don't need this workaround enabled as it was worked around server-side.
	BoundaryTimeWorkaround bool `yaml:"boundary_time_workaround"`
}
<commit_msg>Add missing comment<commit_after>package proxyconfig

import (
	"fmt"
	"io/ioutil"

	"github.com/prometheus/prometheus/config"

	"github.com/jacksontj/promxy/pkg/servergroup"

	yaml "gopkg.in/yaml.v2"
)

// DefaultPromxyConfig is the default promxy config that the config file
// is loaded into
var DefaultPromxyConfig = PromxyConfig{
	BoundaryTimeWorkaround: true,
}

// ConfigFromFile loads a config file at path
func ConfigFromFile(path string) (*Config, error) {
	// load the config file
	cfg := &Config{
		PromConfig:   config.DefaultConfig,
		PromxyConfig: DefaultPromxyConfig,
	}
	configBytes, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("Error loading config: %v", err)
	}
	err = yaml.Unmarshal([]byte(configBytes), &cfg)
	if err != nil {
		return nil, fmt.Errorf("Error unmarshaling config: %v", err)
	}

	return cfg, nil
}

// Config is the entire config file. This includes both the Prometheus Config
// as well as the Promxy config. This is done by "inline-ing" the promxy
// config into the prometheus config under the "promxy" key
type Config struct {
	// Prometheus configs -- this includes configurations for
	// recording rules, alerting rules, etc.
	PromConfig config.Config `yaml:",inline"`

	// Promxy specific configuration -- under its own namespace
	PromxyConfig `yaml:"promxy"`
}

// PromxyConfig is the configuration for Promxy itself
type PromxyConfig struct {
	// Config for each of the server groups promxy is configured to aggregate
	ServerGroups []*servergroup.Config `yaml:"server_groups"`

	// BoundaryTimeWorkaround enables a workaround to prometheus' internal boundary
	// times being un-supported serverside. Newer versions of prometheus (2.11+)
	// don't need this workaround enabled as it was worked around server-side.
	BoundaryTimeWorkaround bool `yaml:"boundary_time_workaround"`
}
<|endoftext|>


<commit_before>package states

import (
	"io"
	. "DNA/errors"
	"DNA/core/code"
	"bytes"
	"DNA/common/serialization"
	"DNA/smartcontract/types"
	"DNA/common"
)

type ContractState struct {
	Code *code.FunctionCode
	Name string
	Version string
	Author string
	Email string
	Description string
	Language types.LangType
	ProgramHash common.Uint160
	StateBase
}

func(contractState *ContractState) Serialize(w io.Writer) error {
	err := contractState.Code.Serialize(w)
	if err != nil {
		return err
	}
	err = serialization.WriteVarString(w, contractState.Name)
	if err != nil {
		return err
	}
	err = serialization.WriteVarString(w, contractState.Version)
	if err != nil {
		return err
	}
	err = serialization.WriteVarString(w, contractState.Author)
	if err != nil {
		return err
	}
	err = serialization.WriteVarString(w, contractState.Email)
	if err != nil {
		return err
	}
	err = serialization.WriteVarString(w, contractState.Description)
	if err != nil {
		return err
	}
	err = serialization.WriteByte(w, byte(contractState.Language))
	if err != nil {
		return err
	}
	_, err = contractState.ProgramHash.Serialize(w)
	if err != nil {
		return err
	}
	return nil
}

func(contractState *ContractState) Deserialize(r io.Reader) error {
	f := new(code.FunctionCode)
	err := f.Deserialize(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "ContractState Code Deserialize fail.")
	}
	contractState.Code = f
	contractState.Name, err = serialization.ReadVarString(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "ContractState Name Deserialize fail.")

	}
	contractState.Version, err = serialization.ReadVarString(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "ContractState Version Deserialize fail.")

	}
	contractState.Author, err = serialization.ReadVarString(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "ContractState Author Deserialize fail.")

	}
	contractState.Email, err = serialization.ReadVarString(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "ContractState Email Deserialize fail.")

	}
	contractState.Description, err = serialization.ReadVarString(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "ContractState Description Deserialize fail.")

	}
	language, err := serialization.ReadByte(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "ContractState Description Deserialize fail.")

	}
	contractState.Language = types.LangType(language)
	u := new(common.Uint160)
	err = u.Deserialize(r)
	if err != nil {
		return err
	}
	contractState.ProgramHash = *u
	return nil
}

func(contractState *ContractState) ToArray() []byte {
	b := new(bytes.Buffer)
	contractState.Serialize(b)
	return b.Bytes()
}

<commit_msg>add version control<commit_after>package states

import (
	"io"
	. "DNA/errors"
	"DNA/core/code"
	"bytes"
	"DNA/common/serialization"
	"DNA/smartcontract/types"
	"DNA/common"
)

type ContractState struct {
	StateBase
	Code *code.FunctionCode
	Name string
	Version string
	Author string
	Email string
	Description string
	Language types.LangType
	ProgramHash common.Uint160
}

func(contractState *ContractState) Serialize(w io.Writer) error {
	contractState.StateBase.Serialize(w)
	err := contractState.Code.Serialize(w)
	if err != nil {
		return err
	}
	err = serialization.WriteVarString(w, contractState.Name)
	if err != nil {
		return err
	}
	err = serialization.WriteVarString(w, contractState.Version)
	if err != nil {
		return err
	}
	err = serialization.WriteVarString(w, contractState.Author)
	if err != nil {
		return err
	}
	err = serialization.WriteVarString(w, contractState.Email)
	if err != nil {
		return err
	}
	err = serialization.WriteVarString(w, contractState.Description)
	if err != nil {
		return err
	}
	err = serialization.WriteByte(w, byte(contractState.Language))
	if err != nil {
		return err
	}
	_, err = contractState.ProgramHash.Serialize(w)
	if err != nil {
		return err
	}
	return nil
}

func(contractState *ContractState) Deserialize(r io.Reader) error {
	stateBase := new(StateBase)
	err := stateBase.Deserialize(r)
	if err != nil {
		return err
	}
	contractState.StateBase = *stateBase
	f := new(code.FunctionCode)
	err = f.Deserialize(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "ContractState Code Deserialize fail.")
	}
	contractState.Code = f
	contractState.Name, err = serialization.ReadVarString(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "ContractState Name Deserialize fail.")

	}
	contractState.Version, err = serialization.ReadVarString(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "ContractState Version Deserialize fail.")

	}
	contractState.Author, err = serialization.ReadVarString(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "ContractState Author Deserialize fail.")

	}
	contractState.Email, err = serialization.ReadVarString(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "ContractState Email Deserialize fail.")

	}
	contractState.Description, err = serialization.ReadVarString(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "ContractState Description Deserialize fail.")

	}
	language, err := serialization.ReadByte(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "ContractState Description Deserialize fail.")

	}
	contractState.Language = types.LangType(language)
	u := new(common.Uint160)
	err = u.Deserialize(r)
	if err != nil {
		return err
	}
	contractState.ProgramHash = *u
	return nil
}

func(contractState *ContractState) ToArray() []byte {
	b := new(bytes.Buffer)
	contractState.Serialize(b)
	return b.Bytes()
}

<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package avalanche

import (
	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/snow/choices"
	"github.com/ava-labs/gecko/snow/consensus/avalanche"
	"github.com/ava-labs/gecko/snow/engine/common"
	"github.com/ava-labs/gecko/snow/engine/common/queue"
	"github.com/ava-labs/gecko/utils/formatting"
	"github.com/prometheus/client_golang/prometheus"
)

// BootstrapConfig ...
type BootstrapConfig struct {
	common.Config

	// VtxBlocked tracks operations that are blocked on vertices
	// TxBlocked tracks operations that are blocked on transactions
	VtxBlocked, TxBlocked *queue.Jobs

	State State
	VM    DAGVM
}

type bootstrapper struct {
	BootstrapConfig
	metrics
	common.Bootstrapper

	pending    ids.Set
	finished   bool
	onFinished func()
}

// Initialize this engine.
func (b *bootstrapper) Initialize(config BootstrapConfig) {
	b.BootstrapConfig = config

	b.VtxBlocked.SetParser(&vtxParser{
		numAccepted: b.numBootstrappedVtx,
		numDropped:  b.numDroppedVtx,
		state:       b.State,
	})

	b.TxBlocked.SetParser(&txParser{
		numAccepted: b.numBootstrappedTx,
		numDropped:  b.numDroppedTx,
		vm:          b.VM,
	})

	config.Bootstrapable = b
	b.Bootstrapper.Initialize(config.Config)
}

// CurrentAcceptedFrontier ...
func (b *bootstrapper) CurrentAcceptedFrontier() ids.Set {
	acceptedFrontier := ids.Set{}
	acceptedFrontier.Add(b.State.Edge()...)
	return acceptedFrontier
}

// FilterAccepted ...
func (b *bootstrapper) FilterAccepted(containerIDs ids.Set) ids.Set {
	acceptedVtxIDs := ids.Set{}
	for _, vtxID := range containerIDs.List() {
		if vtx, err := b.State.GetVertex(vtxID); err == nil && vtx.Status() == choices.Accepted {
			acceptedVtxIDs.Add(vtxID)
		}
	}
	return acceptedVtxIDs
}

// ForceAccepted ...
func (b *bootstrapper) ForceAccepted(acceptedContainerIDs ids.Set) {
	for _, vtxID := range acceptedContainerIDs.List() {
		b.fetch(vtxID)
	}

	if numPending := b.pending.Len(); numPending == 0 {
		// TODO: This typically indicates bootstrapping has failed, so this
		// should be handled appropriately
		b.finish()
	}
}

// Put ...
func (b *bootstrapper) Put(vdr ids.ShortID, requestID uint32, vtxID ids.ID, vtxBytes []byte) {
	b.BootstrapConfig.Context.Log.Verbo("Put called for vertexID %s", vtxID)

	if !b.pending.Contains(vtxID) {
		return
	}

	vtx, err := b.State.ParseVertex(vtxBytes)
	if err != nil {
		b.BootstrapConfig.Context.Log.Warn("ParseVertex failed due to %s for block:\n%s",
			err,
			formatting.DumpBytes{Bytes: vtxBytes})
		b.GetFailed(vdr, requestID, vtxID)
		return
	}

	b.addVertex(vtx)
}

// GetFailed ...
func (b *bootstrapper) GetFailed(_ ids.ShortID, _ uint32, vtxID ids.ID) { b.sendRequest(vtxID) }

func (b *bootstrapper) fetch(vtxID ids.ID) {
	if b.pending.Contains(vtxID) {
		return
	}

	vtx, err := b.State.GetVertex(vtxID)
	if err != nil {
		b.sendRequest(vtxID)
		return
	}
	b.storeVertex(vtx)
}

func (b *bootstrapper) sendRequest(vtxID ids.ID) {
	validators := b.BootstrapConfig.Validators.Sample(1)
	if len(validators) == 0 {
		b.BootstrapConfig.Context.Log.Error("Dropping request for %s as there are no validators", vtxID)
		return
	}
	validatorID := validators[0].ID()
	b.RequestID++

	b.pending.Add(vtxID)
	b.BootstrapConfig.Sender.Get(validatorID, b.RequestID, vtxID)

	b.numPendingRequests.Set(float64(b.pending.Len()))
}

func (b *bootstrapper) addVertex(vtx avalanche.Vertex) {
	b.storeVertex(vtx)

	if numPending := b.pending.Len(); numPending == 0 {
		b.finish()
	}
}

func (b *bootstrapper) storeVertex(vtx avalanche.Vertex) {
	vts := []avalanche.Vertex{vtx}

	for len(vts) > 0 {
		newLen := len(vts) - 1
		vtx := vts[newLen]
		vts = vts[:newLen]

		vtxID := vtx.ID()
		switch status := vtx.Status(); status {
		case choices.Unknown:
			b.sendRequest(vtxID)
		case choices.Processing:
			b.pending.Remove(vtxID)

			if err := b.VtxBlocked.Push(&vertexJob{
				numAccepted: b.numBootstrappedVtx,
				numDropped:  b.numDroppedVtx,
				vtx:         vtx,
			}); err == nil {
				b.numBlockedVtx.Inc()
			}
			for _, tx := range vtx.Txs() {
				if err := b.TxBlocked.Push(&txJob{
					numAccepted: b.numBootstrappedVtx,
					numDropped:  b.numDroppedVtx,
					tx:          tx,
				}); err == nil {
					b.numBlockedTx.Inc()
				}
			}

			for _, parent := range vtx.Parents() {
				vts = append(vts, parent)
			}
		case choices.Accepted:
			b.BootstrapConfig.Context.Log.Verbo("Bootstrapping confirmed %s", vtxID)
		case choices.Rejected:
			b.BootstrapConfig.Context.Log.Error("Bootstrapping wants to accept %s, however it was previously rejected", vtxID)
		}
	}

	numPending := b.pending.Len()
	b.numPendingRequests.Set(float64(numPending))
}

func (b *bootstrapper) finish() {
	if b.finished {
		return
	}

	b.executeAll(b.TxBlocked, b.numBlockedTx)
	b.executeAll(b.VtxBlocked, b.numBlockedVtx)

	// Start consensus
	b.onFinished()
	b.finished = true
}

func (b *bootstrapper) executeAll(jobs *queue.Jobs, numBlocked prometheus.Gauge) {
	for job, err := jobs.Pop(); err == nil; job, err = jobs.Pop() {
		numBlocked.Dec()
		b.BootstrapConfig.Context.Log.Debug("Executing: %s", job.ID())
		if err := jobs.Execute(job); err != nil {
			b.BootstrapConfig.Context.Log.Warn("Error executing: %s", err)
		}
	}
}
<commit_msg>replaced loop<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package avalanche

import (
	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/snow/choices"
	"github.com/ava-labs/gecko/snow/consensus/avalanche"
	"github.com/ava-labs/gecko/snow/engine/common"
	"github.com/ava-labs/gecko/snow/engine/common/queue"
	"github.com/ava-labs/gecko/utils/formatting"
	"github.com/prometheus/client_golang/prometheus"
)

// BootstrapConfig ...
type BootstrapConfig struct {
	common.Config

	// VtxBlocked tracks operations that are blocked on vertices
	// TxBlocked tracks operations that are blocked on transactions
	VtxBlocked, TxBlocked *queue.Jobs

	State State
	VM    DAGVM
}

type bootstrapper struct {
	BootstrapConfig
	metrics
	common.Bootstrapper

	pending    ids.Set
	finished   bool
	onFinished func()
}

// Initialize this engine.
func (b *bootstrapper) Initialize(config BootstrapConfig) {
	b.BootstrapConfig = config

	b.VtxBlocked.SetParser(&vtxParser{
		numAccepted: b.numBootstrappedVtx,
		numDropped:  b.numDroppedVtx,
		state:       b.State,
	})

	b.TxBlocked.SetParser(&txParser{
		numAccepted: b.numBootstrappedTx,
		numDropped:  b.numDroppedTx,
		vm:          b.VM,
	})

	config.Bootstrapable = b
	b.Bootstrapper.Initialize(config.Config)
}

// CurrentAcceptedFrontier ...
func (b *bootstrapper) CurrentAcceptedFrontier() ids.Set {
	acceptedFrontier := ids.Set{}
	acceptedFrontier.Add(b.State.Edge()...)
	return acceptedFrontier
}

// FilterAccepted ...
func (b *bootstrapper) FilterAccepted(containerIDs ids.Set) ids.Set {
	acceptedVtxIDs := ids.Set{}
	for _, vtxID := range containerIDs.List() {
		if vtx, err := b.State.GetVertex(vtxID); err == nil && vtx.Status() == choices.Accepted {
			acceptedVtxIDs.Add(vtxID)
		}
	}
	return acceptedVtxIDs
}

// ForceAccepted ...
func (b *bootstrapper) ForceAccepted(acceptedContainerIDs ids.Set) {
	for _, vtxID := range acceptedContainerIDs.List() {
		b.fetch(vtxID)
	}

	if numPending := b.pending.Len(); numPending == 0 {
		// TODO: This typically indicates bootstrapping has failed, so this
		// should be handled appropriately
		b.finish()
	}
}

// Put ...
func (b *bootstrapper) Put(vdr ids.ShortID, requestID uint32, vtxID ids.ID, vtxBytes []byte) {
	b.BootstrapConfig.Context.Log.Verbo("Put called for vertexID %s", vtxID)

	if !b.pending.Contains(vtxID) {
		return
	}

	vtx, err := b.State.ParseVertex(vtxBytes)
	if err != nil {
		b.BootstrapConfig.Context.Log.Warn("ParseVertex failed due to %s for block:\n%s",
			err,
			formatting.DumpBytes{Bytes: vtxBytes})
		b.GetFailed(vdr, requestID, vtxID)
		return
	}

	b.addVertex(vtx)
}

// GetFailed ...
func (b *bootstrapper) GetFailed(_ ids.ShortID, _ uint32, vtxID ids.ID) { b.sendRequest(vtxID) }

func (b *bootstrapper) fetch(vtxID ids.ID) {
	if b.pending.Contains(vtxID) {
		return
	}

	vtx, err := b.State.GetVertex(vtxID)
	if err != nil {
		b.sendRequest(vtxID)
		return
	}
	b.storeVertex(vtx)
}

func (b *bootstrapper) sendRequest(vtxID ids.ID) {
	validators := b.BootstrapConfig.Validators.Sample(1)
	if len(validators) == 0 {
		b.BootstrapConfig.Context.Log.Error("Dropping request for %s as there are no validators", vtxID)
		return
	}
	validatorID := validators[0].ID()
	b.RequestID++

	b.pending.Add(vtxID)
	b.BootstrapConfig.Sender.Get(validatorID, b.RequestID, vtxID)

	b.numPendingRequests.Set(float64(b.pending.Len()))
}

func (b *bootstrapper) addVertex(vtx avalanche.Vertex) {
	b.storeVertex(vtx)

	if numPending := b.pending.Len(); numPending == 0 {
		b.finish()
	}
}

func (b *bootstrapper) storeVertex(vtx avalanche.Vertex) {
	vts := []avalanche.Vertex{vtx}

	for len(vts) > 0 {
		newLen := len(vts) - 1
		vtx := vts[newLen]
		vts = vts[:newLen]

		vtxID := vtx.ID()
		switch status := vtx.Status(); status {
		case choices.Unknown:
			b.sendRequest(vtxID)
		case choices.Processing:
			b.pending.Remove(vtxID)

			if err := b.VtxBlocked.Push(&vertexJob{
				numAccepted: b.numBootstrappedVtx,
				numDropped:  b.numDroppedVtx,
				vtx:         vtx,
			}); err == nil {
				b.numBlockedVtx.Inc()
			}
			for _, tx := range vtx.Txs() {
				if err := b.TxBlocked.Push(&txJob{
					numAccepted: b.numBootstrappedVtx,
					numDropped:  b.numDroppedVtx,
					tx:          tx,
				}); err == nil {
					b.numBlockedTx.Inc()
				}
			}

			vts = append(vts, vtx.Parents()...)
		case choices.Accepted:
			b.BootstrapConfig.Context.Log.Verbo("Bootstrapping confirmed %s", vtxID)
		case choices.Rejected:
			b.BootstrapConfig.Context.Log.Error("Bootstrapping wants to accept %s, however it was previously rejected", vtxID)
		}
	}

	numPending := b.pending.Len()
	b.numPendingRequests.Set(float64(numPending))
}

func (b *bootstrapper) finish() {
	if b.finished {
		return
	}

	b.executeAll(b.TxBlocked, b.numBlockedTx)
	b.executeAll(b.VtxBlocked, b.numBlockedVtx)

	// Start consensus
	b.onFinished()
	b.finished = true
}

func (b *bootstrapper) executeAll(jobs *queue.Jobs, numBlocked prometheus.Gauge) {
	for job, err := jobs.Pop(); err == nil; job, err = jobs.Pop() {
		numBlocked.Dec()
		b.BootstrapConfig.Context.Log.Debug("Executing: %s", job.ID())
		if err := jobs.Execute(job); err != nil {
			b.BootstrapConfig.Context.Log.Warn("Error executing: %s", err)
		}
	}
}
<|endoftext|>


<commit_before>package message

import (
	"net/http"
	"net/url"
	"socialapi/models"
	"socialapi/workers/api/modules/helpers"

	"github.com/jinzhu/gorm"
)

func Create(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	channelId, err := helpers.GetURIInt64(u, "id")
	if err != nil {
		return helpers.NewBadRequestResponse(err)
	}

	// override message type
	// all of the messages coming from client-side
	// should be marked as POST
	req.Type = models.ChannelMessage_TYPE_POST

	// set initial channel id
	req.InitialChannelId = channelId

	if err := req.Create(); err != nil {
		// todo this should be internal server error
		return helpers.NewBadRequestResponse(err)
	}

	cml := models.NewChannelMessageList()

	// override channel id
	cml.ChannelId = channelId
	cml.MessageId = req.Id
	if err := cml.Create(); err != nil {
		// todo this should be internal server error
		return helpers.NewBadRequestResponse(err)
	}

	return helpers.NewOKResponse(req)
}

func Delete(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	id, err := helpers.GetURIInt64(u, "id")
	if err != nil {
		return helpers.NewBadRequestResponse(err)
	}

	req.Id = id

	if err := req.Fetch(); err != nil {
		if err == gorm.RecordNotFound {
			return helpers.NewNotFoundResponse()
		}
		return helpers.NewBadRequestResponse(err)
	}

	if err := req.Delete(); err != nil {
		return helpers.NewBadRequestResponse(err)
	}
	// yes it is deleted but not removed completely from our system
	return helpers.NewDeletedResponse()
}

func Update(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	id, err := helpers.GetURIInt64(u, "id")
	if err != nil {
		return helpers.NewBadRequestResponse(err)
	}
	req.Id = id

	body := req.Body
	if err := req.Fetch(); err != nil {
		if err == gorm.RecordNotFound {
			return helpers.NewNotFoundResponse()
		}
		return helpers.NewBadRequestResponse(err)
	}

	if req.Id == 0 {
		return helpers.NewBadRequestResponse(err)
	}

	req.Body = body
	if err := req.Update(); err != nil {
		return helpers.NewBadRequestResponse(err)
	}

	return helpers.NewOKResponse(req)
}

func Get(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	id, err := helpers.GetURIInt64(u, "id")
	if err != nil {
		return helpers.NewBadRequestResponse(err)
	}

	req.Id = id
	if err := req.Fetch(); err != nil {
		if err == gorm.RecordNotFound {
			return helpers.NewNotFoundResponse()
		}
		return helpers.NewBadRequestResponse(err)
	}

	return helpers.NewOKResponse(req)
}
<commit_msg>Social: change type to typeConstant<commit_after>package message

import (
	"net/http"
	"net/url"
	"socialapi/models"
	"socialapi/workers/api/modules/helpers"

	"github.com/jinzhu/gorm"
)

func Create(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	channelId, err := helpers.GetURIInt64(u, "id")
	if err != nil {
		return helpers.NewBadRequestResponse(err)
	}

	// override message type
	// all of the messages coming from client-side
	// should be marked as POST
	req.TypeConstant = models.ChannelMessage_TYPE_POST

	// set initial channel id
	req.InitialChannelId = channelId

	if err := req.Create(); err != nil {
		// todo this should be internal server error
		return helpers.NewBadRequestResponse(err)
	}

	cml := models.NewChannelMessageList()

	// override channel id
	cml.ChannelId = channelId
	cml.MessageId = req.Id
	if err := cml.Create(); err != nil {
		// todo this should be internal server error
		return helpers.NewBadRequestResponse(err)
	}

	return helpers.NewOKResponse(req)
}

func Delete(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	id, err := helpers.GetURIInt64(u, "id")
	if err != nil {
		return helpers.NewBadRequestResponse(err)
	}

	req.Id = id

	if err := req.Fetch(); err != nil {
		if err == gorm.RecordNotFound {
			return helpers.NewNotFoundResponse()
		}
		return helpers.NewBadRequestResponse(err)
	}

	if err := req.Delete(); err != nil {
		return helpers.NewBadRequestResponse(err)
	}
	// yes it is deleted but not removed completely from our system
	return helpers.NewDeletedResponse()
}

func Update(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	id, err := helpers.GetURIInt64(u, "id")
	if err != nil {
		return helpers.NewBadRequestResponse(err)
	}
	req.Id = id

	body := req.Body
	if err := req.Fetch(); err != nil {
		if err == gorm.RecordNotFound {
			return helpers.NewNotFoundResponse()
		}
		return helpers.NewBadRequestResponse(err)
	}

	if req.Id == 0 {
		return helpers.NewBadRequestResponse(err)
	}

	req.Body = body
	if err := req.Update(); err != nil {
		return helpers.NewBadRequestResponse(err)
	}

	return helpers.NewOKResponse(req)
}

func Get(u *url.URL, h http.Header, req *models.ChannelMessage) (int, http.Header, interface{}, error) {
	id, err := helpers.GetURIInt64(u, "id")
	if err != nil {
		return helpers.NewBadRequestResponse(err)
	}

	req.Id = id
	if err := req.Fetch(); err != nil {
		if err == gorm.RecordNotFound {
			return helpers.NewNotFoundResponse()
		}
		return helpers.NewBadRequestResponse(err)
	}

	return helpers.NewOKResponse(req)
}
<|endoftext|>


<commit_before>package compress

import (
	"testing"
	"github.com/docker/docker/pkg/testutil/assert"
)

func TestExtWindows(t *testing.T) {
	assert.Equal(t, ext("windows"), ".exe")
}

func TestExtOthers(t *testing.T) {
	assert.Equal(t, ext("linux"), "")
}
<commit_msg>fixed import<commit_after>package compress

import (
	"testing"
	"github.com/stretchr/testify/assert"
)

func TestExtWindows(t *testing.T) {
	assert.Equal(t, ext("windows"), ".exe")
}

func TestExtOthers(t *testing.T) {
	assert.Empty(t, ext("linux"))
}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Tobias Schottdorf (tobias.schottdorf@gmail.com)

package terrafarm

import (
	"fmt"
	"io"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/pkg/base"
	"github.com/cockroachdb/cockroach/pkg/internal/client"
	"github.com/cockroachdb/cockroach/pkg/rpc"
	"github.com/cockroachdb/cockroach/pkg/security"
	"github.com/cockroachdb/cockroach/pkg/util"
	"github.com/cockroachdb/cockroach/pkg/util/retry"
	"github.com/pkg/errors"
)

// The constants below are the possible values of the KeepCluster field.
const (
	// KeepClusterAlways lets Farmer always keep the test cluster.
	KeepClusterAlways = "always"
	// KeepClusterFailed lets Farmer keep only failed test clusters.
	KeepClusterFailed = "failed"
	// KeepClusterNever lets Farmer always destroy the test cluster.
	KeepClusterNever = "never"
)

type node struct {
	hostname  string
	processes []string
}

// A Farmer sets up and manipulates a test cluster via terraform.
type Farmer struct {
	Output      io.Writer
	Cwd, LogDir string
	KeyName     string
	Stores      string
	// Prefix will be prepended all names of resources created by Terraform.
	Prefix string
	// StateFile is the file (under `Cwd`) in which Terraform will stores its
	// state.
	StateFile string
	// AddVars are additional Terraform variables to be set during calls to Add.
	AddVars     map[string]string
	KeepCluster string
	nodes       []node
	RPCContext  *rpc.Context
}

func (f *Farmer) refresh() {
	hosts := f.output("instances")
	f.nodes = make([]node, len(hosts))
	for i, host := range hosts {
		out, _, err := f.execSupervisor(host, "status | grep -F RUNNING | cut -f1 -d' '")
		if err != nil {
			panic(err)
		}
		f.nodes[i] = node{
			hostname:  host,
			processes: strings.Split(out, "\n"),
		}
	}
}

func (f *Farmer) initNodes() []node {
	if len(f.nodes) == 0 {
		f.refresh()
	}
	return f.nodes
}

// Hostname implements the Cluster interface.
func (f *Farmer) Hostname(i int) string {
	return f.initNodes()[i].hostname
}

// NumNodes returns the number of nodes.
func (f *Farmer) NumNodes() int {
	return len(f.initNodes())
}

// Add provisions the given number of nodes.
func (f *Farmer) Add(nodes int) error {
	nodes += f.NumNodes()
	args := []string{
		fmt.Sprintf("-var=num_instances=\"%d\"", nodes),
		fmt.Sprintf("-var=stores=%s", f.Stores),
	}

	// Disable update checks for test clusters by setting the required environment
	// variable.
	const skipCheck = "COCKROACH_SKIP_UPDATE_CHECK=1"
	// This is a Terraform variable defined in acceptance/terraform/variables.tf
	// and passed through to the supervisor.conf file through
	// acceptance/terraform/main.tf.
	const envVar = "cockroach_env"
	if env := f.AddVars[envVar]; env == "" {
		f.AddVars[envVar] = skipCheck
	} else {
		f.AddVars[envVar] += "," + skipCheck
	}

	for v, val := range f.AddVars {
		args = append(args, fmt.Sprintf(`-var=%s="%s"`, v, val))
	}

	if nodes == 0 {
		return f.runErr("terraform", f.appendDefaults(append([]string{"destroy", "--force"}, args...))...)
	}
	return f.apply(args...)
}

// Resize is the counterpart to Add which resizes a cluster given
// the desired number of nodes.
func (f *Farmer) Resize(nodes int) error {
	return f.Add(nodes - f.NumNodes())
}

// AbsLogDir returns the absolute log dir to which logs are written.
func (f *Farmer) AbsLogDir() string {
	if f.LogDir == "" || filepath.IsAbs(f.LogDir) {
		return f.LogDir
	}
	return filepath.Clean(filepath.Join(f.Cwd, f.LogDir))
}

// CollectLogs copies all possibly interesting files from all available peers
// if LogDir is not empty.
func (f *Farmer) CollectLogs() {
	if f.LogDir == "" {
		return
	}
	if err := os.MkdirAll(f.AbsLogDir(), 0777); err != nil {
		fmt.Fprint(os.Stderr, err)
		return
	}
	const src = "logs"
	for i, node := range f.initNodes() {
		if err := f.scp(node.hostname, f.defaultKeyFile(), src,
			filepath.Join(f.AbsLogDir(), "node."+strconv.Itoa(i))); err != nil {
			f.logf("error collecting %s from host %s: %s\n", src, node.hostname, err)
		}
	}
}

// Destroy collects the logs and tears down the cluster.
func (f *Farmer) Destroy(t testing.TB) error {
	f.CollectLogs()
	if f.LogDir != "" {
		defer f.logf("logs copied to %s\n", f.AbsLogDir())
	}

	wd, err := os.Getwd()
	if err != nil {
		wd = "acceptance"
	}
	baseDir := filepath.Join(wd, f.Cwd)
	if (t.Failed() && f.KeepCluster == KeepClusterFailed) ||
		f.KeepCluster == KeepClusterAlways {

		t.Logf("not destroying; run:\n(cd %s && terraform destroy -force -state %s && rm %s)",
			baseDir, f.StateFile, f.StateFile)
		return nil
	}

	if err := f.Resize(0); err != nil {
		return err
	}
	return os.Remove(filepath.Join(baseDir, f.StateFile))
}

// MustDestroy calls Destroy(), fataling on error.
func (f *Farmer) MustDestroy(t testing.TB) {
	if err := f.Destroy(t); err != nil {
		t.Fatal(errors.Wrap(err, "cannot destroy cluster"))
	}
}

// Exec executes the given command on the i-th node.
func (f *Farmer) Exec(i int, cmd string) error {
	stdout, stderr, err := f.ssh(f.Hostname(i), f.defaultKeyFile(), cmd)
	if err != nil {
		return fmt.Errorf("failed: %s: %s\nstdout:\n%s\nstderr:\n%s", cmd, err, stdout, stderr)
	}
	return nil
}

// NewClient implements the Cluster interface.
func (f *Farmer) NewClient(ctx context.Context, i int) (*client.DB, error) {
	conn, err := f.RPCContext.GRPCDial(f.Addr(ctx, i, base.DefaultPort))
	if err != nil {
		return nil, err
	}
	return client.NewDB(client.NewSender(conn)), nil
}

// PGUrl returns a URL string for the given node postgres server.
func (f *Farmer) PGUrl(ctx context.Context, i int) string {
	host := f.Hostname(i)
	return fmt.Sprintf("postgresql://%s@%s:26257/system?sslmode=disable", security.RootUser, host)
}

// InternalIP returns the address used for inter-node communication.
func (f *Farmer) InternalIP(ctx context.Context, i int) net.IP {
	// TODO(tschottdorf): This is specific to GCE. On AWS, the following
	// might do it: `curl -sS http://instance-data/latest/meta-data/public-ipv4`.
	// See https://flummox-engineering.blogspot.com/2014/01/get-ip-address-of-google-compute-engine.html
	cmd := `curl -sS "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip" -H "X-Google-Metadata-Request: True"`
	stdout, stderr, err := f.ssh(f.Hostname(i), f.defaultKeyFile(), cmd)
	if err != nil {
		panic(errors.Wrapf(err, stderr+"\n"+stdout))
	}
	stdout = strings.TrimSpace(stdout)
	ip := net.ParseIP(stdout)
	if ip == nil {
		panic(fmt.Sprintf("'%s' did not parse to an IP", stdout))
	}
	return ip
}

// WaitReady waits until the infrastructure is in a state that *should* allow
// for a healthy cluster. Currently, this means waiting for the load balancer
// to resolve from all nodes.
func (f *Farmer) WaitReady(d time.Duration) error {
	var rOpts = retry.Options{
		InitialBackoff: time.Second,
		MaxBackoff:     time.Minute,
		Multiplier:     1.5,
	}
	var err error
	for r := retry.Start(rOpts); r.Next(); {
		instance := f.Hostname(0)
		if err != nil || instance == "" {
			err = fmt.Errorf("no nodes found: %v", err)
			continue
		}
		for i := 0; i < f.NumNodes(); i++ {
			if err = f.Exec(i, "nslookup "+instance); err != nil {
				break
			}
		}
		if err == nil {
			return nil
		}
	}
	return err
}

// Assert verifies that the cluster state is as expected (i.e. no unexpected
// restarts or node deaths occurred). Tests can call this periodically to
// ascertain cluster health.
// TODO(tschottdorf): unimplemented when nodes are expected down.
func (f *Farmer) Assert(ctx context.Context, t testing.TB) {
	for _, node := range f.nodes {
		for _, process := range node.processes {
			f.AssertState(ctx, t, node.hostname, process, "RUNNING")
		}
	}
}

// AssertState verifies that on the specified host, the given process (managed
// by supervisord) is in the expected state.
func (f *Farmer) AssertState(ctx context.Context, t testing.TB, host, proc, expState string) {
	out, _, err := f.execSupervisor(host, "status "+proc)
	if err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(out, expState) {
		t.Fatalf("%s (%s) is not in expected state %s:\n%s", proc, host, expState, out)
	}
}

// AssertAndStop performs the same test as Assert but then proceeds to
// dismantle the cluster.
func (f *Farmer) AssertAndStop(ctx context.Context, t testing.TB) {
	f.Assert(ctx, t)
	f.MustDestroy(t)
}

// ExecRoot executes the given command with super-user privileges.
func (f *Farmer) ExecRoot(ctx context.Context, i int, cmd []string) error {
	// TODO(tschottdorf): This doesn't handle escapes properly. May it never
	// have to.
	return f.Exec(i, "sudo "+strings.Join(cmd, " "))
}

// Kill terminates the cockroach process running on the given node number.
// The given integer must be in the range [0,NumNodes()-1].
func (f *Farmer) Kill(ctx context.Context, i int) error {
	return f.Exec(i, "pkill -9 cockroach")
}

// Restart terminates the cockroach process running on the given node
// number, unless it is already stopped, and restarts it.
// The given integer must be in the range [0,NumNodes()-1].
func (f *Farmer) Restart(ctx context.Context, i int) error {
	_ = f.Kill(ctx, i)
	// supervisorctl is horrible with exit codes (cockroachdb/cockroach-prod#59).
	_, _, err := f.execSupervisor(f.Hostname(i), "start cockroach")
	return err
}

// Start starts the given process on the ith node.
func (f *Farmer) Start(ctx context.Context, i int, process string) error {
	for _, p := range f.nodes[i].processes {
		if p == process {
			return errors.Errorf("already started process '%s'", process)
		}
	}
	if _, _, err := f.execSupervisor(f.Hostname(i), "start "+process); err != nil {
		return err
	}
	f.nodes[i].processes = append(f.nodes[i].processes, process)
	return nil
}

// Stop stops the given process on the ith node. This is useful for terminating
// a load generator cleanly to get stats outputted upon process termination.
func (f *Farmer) Stop(ctx context.Context, i int, process string) error {
	for idx, p := range f.nodes[i].processes {
		if p == process {
			if _, _, err := f.execSupervisor(f.Hostname(i), "stop "+process); err != nil {
				return err
			}
			f.nodes[i].processes = append(f.nodes[i].processes[:idx], f.nodes[i].processes[idx+1:]...)
			return nil
		}
	}
	return errors.Errorf("unable to find process '%s'", process)
}

// URL returns the HTTP(s) endpoint.
func (f *Farmer) URL(ctx context.Context, i int) string {
	return "http://" + net.JoinHostPort(f.Hostname(i), base.DefaultHTTPPort)
}

// Addr returns the host and port from the node in the format HOST:PORT.
func (f *Farmer) Addr(ctx context.Context, i int, port string) string {
	return net.JoinHostPort(f.Hostname(i), port)
}

func (f *Farmer) logf(format string, args ...interface{}) {
	if f.Output != nil {
		fmt.Fprintf(f.Output, format, args...)
	}
}

// StartLoad starts n loadGenerator processes.
func (f *Farmer) StartLoad(ctx context.Context, loadGenerator string, n int) error {
	if n > f.NumNodes() {
		return errors.Errorf("writers (%d) > nodes (%d)", n, f.NumNodes())
	}

	// We may have to retry restarting the load generators, because CockroachDB
	// might have been started too recently to start accepting connections.
	for i := 0; i < n; i++ {
		if err := util.RetryForDuration(10*time.Second, func() error {
			return f.Start(ctx, i, loadGenerator)
		}); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>acceptance: remove Farmer.initNodes<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Tobias Schottdorf (tobias.schottdorf@gmail.com)

package terrafarm

import (
	"fmt"
	"io"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/pkg/base"
	"github.com/cockroachdb/cockroach/pkg/internal/client"
	"github.com/cockroachdb/cockroach/pkg/rpc"
	"github.com/cockroachdb/cockroach/pkg/security"
	"github.com/cockroachdb/cockroach/pkg/util"
	"github.com/cockroachdb/cockroach/pkg/util/retry"
	"github.com/pkg/errors"
)

// The constants below are the possible values of the KeepCluster field.
const (
	// KeepClusterAlways lets Farmer always keep the test cluster.
	KeepClusterAlways = "always"
	// KeepClusterFailed lets Farmer keep only failed test clusters.
	KeepClusterFailed = "failed"
	// KeepClusterNever lets Farmer always destroy the test cluster.
	KeepClusterNever = "never"
)

type node struct {
	hostname  string
	processes []string
}

// A Farmer sets up and manipulates a test cluster via terraform.
type Farmer struct {
	Output      io.Writer
	Cwd, LogDir string
	KeyName     string
	Stores      string
	// Prefix will be prepended all names of resources created by Terraform.
	Prefix string
	// StateFile is the file (under `Cwd`) in which Terraform will stores its
	// state.
	StateFile string
	// AddVars are additional Terraform variables to be set during calls to Add.
	AddVars     map[string]string
	KeepCluster string
	nodes       []node
	RPCContext  *rpc.Context
}

func (f *Farmer) refresh() {
	hosts := f.output("instances")
	f.nodes = make([]node, len(hosts))
	for i, host := range hosts {
		out, _, err := f.execSupervisor(host, "status | grep -F RUNNING | cut -f1 -d' '")
		if err != nil {
			panic(err)
		}
		f.nodes[i] = node{
			hostname:  host,
			processes: strings.Split(out, "\n"),
		}
	}
}

// Hostname implements the Cluster interface.
func (f *Farmer) Hostname(i int) string {
	if len(f.nodes) == 0 {
		f.refresh()
	}
	return f.nodes[i].hostname
}

// NumNodes returns the number of nodes.
func (f *Farmer) NumNodes() int {
	if len(f.nodes) == 0 {
		f.refresh()
	}
	return len(f.nodes)
}

// Add provisions the given number of nodes.
func (f *Farmer) Add(nodes int) error {
	nodes += f.NumNodes()
	args := []string{
		fmt.Sprintf("-var=num_instances=\"%d\"", nodes),
		fmt.Sprintf("-var=stores=%s", f.Stores),
	}

	// Disable update checks for test clusters by setting the required environment
	// variable.
	const skipCheck = "COCKROACH_SKIP_UPDATE_CHECK=1"
	// This is a Terraform variable defined in acceptance/terraform/variables.tf
	// and passed through to the supervisor.conf file through
	// acceptance/terraform/main.tf.
	const envVar = "cockroach_env"
	if env := f.AddVars[envVar]; env == "" {
		f.AddVars[envVar] = skipCheck
	} else {
		f.AddVars[envVar] += "," + skipCheck
	}

	for v, val := range f.AddVars {
		args = append(args, fmt.Sprintf(`-var=%s="%s"`, v, val))
	}

	if nodes == 0 {
		return f.runErr("terraform", f.appendDefaults(append([]string{"destroy", "--force"}, args...))...)
	}
	return f.apply(args...)
}

// Resize is the counterpart to Add which resizes a cluster given
// the desired number of nodes.
func (f *Farmer) Resize(nodes int) error {
	return f.Add(nodes - f.NumNodes())
}

// AbsLogDir returns the absolute log dir to which logs are written.
func (f *Farmer) AbsLogDir() string {
	if f.LogDir == "" || filepath.IsAbs(f.LogDir) {
		return f.LogDir
	}
	return filepath.Clean(filepath.Join(f.Cwd, f.LogDir))
}

// CollectLogs copies all possibly interesting files from all available peers
// if LogDir is not empty.
func (f *Farmer) CollectLogs() {
	if f.LogDir == "" {
		return
	}
	if err := os.MkdirAll(f.AbsLogDir(), 0777); err != nil {
		fmt.Fprint(os.Stderr, err)
		return
	}
	const src = "logs"
	for i := 0; i < f.NumNodes(); i++ {
		if err := f.scp(f.Hostname(i), f.defaultKeyFile(), src,
			filepath.Join(f.AbsLogDir(), "node."+strconv.Itoa(i))); err != nil {
			f.logf("error collecting %s from host %s: %s\n", src, f.Hostname(i), err)
		}
	}
}

// Destroy collects the logs and tears down the cluster.
func (f *Farmer) Destroy(t testing.TB) error {
	f.CollectLogs()
	if f.LogDir != "" {
		defer f.logf("logs copied to %s\n", f.AbsLogDir())
	}

	wd, err := os.Getwd()
	if err != nil {
		wd = "acceptance"
	}
	baseDir := filepath.Join(wd, f.Cwd)
	if (t.Failed() && f.KeepCluster == KeepClusterFailed) ||
		f.KeepCluster == KeepClusterAlways {

		t.Logf("not destroying; run:\n(cd %s && terraform destroy -force -state %s && rm %s)",
			baseDir, f.StateFile, f.StateFile)
		return nil
	}

	if err := f.Resize(0); err != nil {
		return err
	}
	return os.Remove(filepath.Join(baseDir, f.StateFile))
}

// MustDestroy calls Destroy(), fataling on error.
func (f *Farmer) MustDestroy(t testing.TB) {
	if err := f.Destroy(t); err != nil {
		t.Fatal(errors.Wrap(err, "cannot destroy cluster"))
	}
}

// Exec executes the given command on the i-th node.
func (f *Farmer) Exec(i int, cmd string) error {
	stdout, stderr, err := f.ssh(f.Hostname(i), f.defaultKeyFile(), cmd)
	if err != nil {
		return fmt.Errorf("failed: %s: %s\nstdout:\n%s\nstderr:\n%s", cmd, err, stdout, stderr)
	}
	return nil
}

// NewClient implements the Cluster interface.
func (f *Farmer) NewClient(ctx context.Context, i int) (*client.DB, error) {
	conn, err := f.RPCContext.GRPCDial(f.Addr(ctx, i, base.DefaultPort))
	if err != nil {
		return nil, err
	}
	return client.NewDB(client.NewSender(conn)), nil
}

// PGUrl returns a URL string for the given node postgres server.
func (f *Farmer) PGUrl(ctx context.Context, i int) string {
	host := f.Hostname(i)
	return fmt.Sprintf("postgresql://%s@%s:26257/system?sslmode=disable", security.RootUser, host)
}

// InternalIP returns the address used for inter-node communication.
func (f *Farmer) InternalIP(ctx context.Context, i int) net.IP {
	// TODO(tschottdorf): This is specific to GCE. On AWS, the following
	// might do it: `curl -sS http://instance-data/latest/meta-data/public-ipv4`.
	// See https://flummox-engineering.blogspot.com/2014/01/get-ip-address-of-google-compute-engine.html
	cmd := `curl -sS "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip" -H "X-Google-Metadata-Request: True"`
	stdout, stderr, err := f.ssh(f.Hostname(i), f.defaultKeyFile(), cmd)
	if err != nil {
		panic(errors.Wrapf(err, stderr+"\n"+stdout))
	}
	stdout = strings.TrimSpace(stdout)
	ip := net.ParseIP(stdout)
	if ip == nil {
		panic(fmt.Sprintf("'%s' did not parse to an IP", stdout))
	}
	return ip
}

// WaitReady waits until the infrastructure is in a state that *should* allow
// for a healthy cluster. Currently, this means waiting for the load balancer
// to resolve from all nodes.
func (f *Farmer) WaitReady(d time.Duration) error {
	var rOpts = retry.Options{
		InitialBackoff: time.Second,
		MaxBackoff:     time.Minute,
		Multiplier:     1.5,
	}
	var err error
	for r := retry.Start(rOpts); r.Next(); {
		instance := f.Hostname(0)
		if err != nil || instance == "" {
			err = fmt.Errorf("no nodes found: %v", err)
			continue
		}
		for i := 0; i < f.NumNodes(); i++ {
			if err = f.Exec(i, "nslookup "+instance); err != nil {
				break
			}
		}
		if err == nil {
			return nil
		}
	}
	return err
}

// Assert verifies that the cluster state is as expected (i.e. no unexpected
// restarts or node deaths occurred). Tests can call this periodically to
// ascertain cluster health.
// TODO(tschottdorf): unimplemented when nodes are expected down.
func (f *Farmer) Assert(ctx context.Context, t testing.TB) {
	for _, node := range f.nodes {
		for _, process := range node.processes {
			f.AssertState(ctx, t, node.hostname, process, "RUNNING")
		}
	}
}

// AssertState verifies that on the specified host, the given process (managed
// by supervisord) is in the expected state.
func (f *Farmer) AssertState(ctx context.Context, t testing.TB, host, proc, expState string) {
	out, _, err := f.execSupervisor(host, "status "+proc)
	if err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(out, expState) {
		t.Fatalf("%s (%s) is not in expected state %s:\n%s", proc, host, expState, out)
	}
}

// AssertAndStop performs the same test as Assert but then proceeds to
// dismantle the cluster.
func (f *Farmer) AssertAndStop(ctx context.Context, t testing.TB) {
	f.Assert(ctx, t)
	f.MustDestroy(t)
}

// ExecRoot executes the given command with super-user privileges.
func (f *Farmer) ExecRoot(ctx context.Context, i int, cmd []string) error {
	// TODO(tschottdorf): This doesn't handle escapes properly. May it never
	// have to.
	return f.Exec(i, "sudo "+strings.Join(cmd, " "))
}

// Kill terminates the cockroach process running on the given node number.
// The given integer must be in the range [0,NumNodes()-1].
func (f *Farmer) Kill(ctx context.Context, i int) error {
	return f.Exec(i, "pkill -9 cockroach")
}

// Restart terminates the cockroach process running on the given node
// number, unless it is already stopped, and restarts it.
// The given integer must be in the range [0,NumNodes()-1].
func (f *Farmer) Restart(ctx context.Context, i int) error {
	_ = f.Kill(ctx, i)
	// supervisorctl is horrible with exit codes (cockroachdb/cockroach-prod#59).
	_, _, err := f.execSupervisor(f.Hostname(i), "start cockroach")
	return err
}

// Start starts the given process on the ith node.
func (f *Farmer) Start(ctx context.Context, i int, process string) error {
	for _, p := range f.nodes[i].processes {
		if p == process {
			return errors.Errorf("already started process '%s'", process)
		}
	}
	if _, _, err := f.execSupervisor(f.Hostname(i), "start "+process); err != nil {
		return err
	}
	f.nodes[i].processes = append(f.nodes[i].processes, process)
	return nil
}

// Stop stops the given process on the ith node. This is useful for terminating
// a load generator cleanly to get stats outputted upon process termination.
func (f *Farmer) Stop(ctx context.Context, i int, process string) error {
	for idx, p := range f.nodes[i].processes {
		if p == process {
			if _, _, err := f.execSupervisor(f.Hostname(i), "stop "+process); err != nil {
				return err
			}
			f.nodes[i].processes = append(f.nodes[i].processes[:idx], f.nodes[i].processes[idx+1:]...)
			return nil
		}
	}
	return errors.Errorf("unable to find process '%s'", process)
}

// URL returns the HTTP(s) endpoint.
func (f *Farmer) URL(ctx context.Context, i int) string {
	return "http://" + net.JoinHostPort(f.Hostname(i), base.DefaultHTTPPort)
}

// Addr returns the host and port from the node in the format HOST:PORT.
func (f *Farmer) Addr(ctx context.Context, i int, port string) string {
	return net.JoinHostPort(f.Hostname(i), port)
}

func (f *Farmer) logf(format string, args ...interface{}) {
	if f.Output != nil {
		fmt.Fprintf(f.Output, format, args...)
	}
}

// StartLoad starts n loadGenerator processes.
func (f *Farmer) StartLoad(ctx context.Context, loadGenerator string, n int) error {
	if n > f.NumNodes() {
		return errors.Errorf("writers (%d) > nodes (%d)", n, f.NumNodes())
	}

	// We may have to retry restarting the load generators, because CockroachDB
	// might have been started too recently to start accepting connections.
	for i := 0; i < n; i++ {
		if err := util.RetryForDuration(10*time.Second, func() error {
			return f.Start(ctx, i, loadGenerator)
		}); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// +build !linux !amd64

package netlink

import (
	"fmt"
	"net"
)

func NetworkGetRoutes() ([]Route, error) {
	return nil, fmt.Errorf("Not implemented")
}

func NetworkLinkAdd(name string, linkType string) error {
	return fmt.Errorf("Not implemented")
}

func NetworkLinkUp(iface *net.Interface) error {
	return fmt.Errorf("Not implemented")
}

func NetworkLinkAddIp(iface *net.Interface, ip net.IP, ipNet *net.IPNet) error {
	return fmt.Errorf("Not implemented")
}

func AddDefaultGw(ip net.IP) error {
	return fmt.Errorf("Not implemented")

}

func NetworkSetMTU(iface *net.Interface, mtu int) error {
	return fmt.Errorf("Not implemented")
}
<commit_msg>Add new functions to unsupported file Docker-DCO-1.1-Signed-off-by: Michael Crosby <michael@crosbymichael.com> (github: crosbymichael)<commit_after>// +build !linux !amd64

package netlink

import (
	"errors"
	"net"
)

var (
	ErrNotImplemented = errors.New("not implemented")
)

func NetworkGetRoutes() ([]Route, error) {
	return nil, ErrNotImplemented
}

func NetworkLinkAdd(name string, linkType string) error {
	return ErrNotImplemented
}

func NetworkLinkUp(iface *net.Interface) error {
	return ErrNotImplemented
}

func NetworkLinkAddIp(iface *net.Interface, ip net.IP, ipNet *net.IPNet) error {
	return ErrNotImplemented
}

func AddDefaultGw(ip net.IP) error {
	return ErrNotImplemented

}

func NetworkSetMTU(iface *net.Interface, mtu int) error {
	return ErrNotImplemented
}

func NetworkCreateVethPair(name1, name2 string) error {
	return ErrNotImplemented
}

func NetworkChangeName(iface *net.Interface, newName string) error {
	return ErrNotImplemented
}

func NetworkSetNsFd(iface *net.Interface, fd int) error {
	return ErrNotImplemented
}

func NetworkSetNsPid(iface *net.Interface, nspid int) error {
	return ErrNotImplemented
}

func NetworkSetMaster(iface, master *net.Interface) error {
	return ErrNotImplemented
}

func NetworkLinkDown(iface *net.Interface) error {
	return ErrNotImplemented
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2017 Red Hat, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Red Hat trademarks are not licensed under Apache License, Version 2.
// No permission is granted to use or replicate Red Hat trademarks that
// are incorporated in this software or its documentation.
//

package adapters

import (
	b64 "encoding/base64"
	"encoding/json"
	"net/http"
	"net/url"

	logging "github.com/op/go-logging"
	"github.com/openshift/ansible-service-broker/pkg/apb"
	yaml "gopkg.in/yaml.v1"
)

// Adapter - Adapter will wrap the methods that a registry needs to fully manage images.
type Adapter interface {
	// RegistryName will return the registiry prefix for the adapter.
	// Example is docker.io for the dockerhub adapter.
	RegistryName() string
	// GetImageNames will return all the image names for the adapter configuration.
	GetImageNames() ([]string, error)
	// FetchSpecs will retrieve all the specs for the list of images names.
	FetchSpecs([]string) ([]*apb.Spec, error)
}

// BundleSpecLabel - label on the image that we should use to pull out the abp spec.
// TODO: needs to remain ansibleapp UNTIL we redo the apps in dockerhub
const BundleSpecLabel = "com.redhat.apb.spec"

// Configuration - Adapter configuration. Contains the info that the adapter
// would need to complete its request to the images.
type Configuration struct {
	URL        *url.URL
	User       string
	Pass       string
	Org        string
	Images     []string
	Namespaces []string
	Tag        string
}

// Retrieve the spec from a registry manifest request
func imageToSpec(log *logging.Logger, req *http.Request, image string) (*apb.Spec, error) {
	log.Debug("Registry::imageToSpec")
	spec := &apb.Spec{}
	req.Header.Add("Accept", "application/json")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	type label struct {
		Spec string `json:"com.redhat.apb.spec"`
	}

	type config struct {
		Label label `json:"Labels"`
	}

	hist := struct {
		History []map[string]string `json:"history"`
	}{}

	conf := struct {
		Config *config `json:"config"`
	}{}

	if resp.StatusCode == http.StatusUnauthorized {
		log.Errorf("Unable to authenticate to the registry, registry credentials could be invalid.")
		return nil, nil
	}

	err = json.NewDecoder(resp.Body).Decode(&hist)
	if err != nil {
		log.Errorf("Error grabbing JSON body from response: %s", err)
		return nil, err
	}

	if hist.History == nil {
		log.Errorf("V1 Schema Manifest does not exist in registry")
		return nil, nil
	}

	err = json.Unmarshal([]byte(hist.History[0]["v1Compatibility"]), &conf)
	if err != nil {
		log.Errorf("Error unmarshalling intermediary JSON response: %s", err)
		return nil, err
	}
	if conf.Config == nil {
		log.Infof("Did not find v1 Manifest in image history. Skipping image")
		return nil, nil
	}
	if conf.Config.Label.Spec == "" {
		log.Infof("Didn't find encoded Spec label. Assuming image is not APB and skiping")
		return nil, nil
	}

	encodedSpec := conf.Config.Label.Spec
	decodedSpecYaml, err := b64.StdEncoding.DecodeString(encodedSpec)
	if err != nil {
		log.Errorf("Something went wrong decoding spec from label")
		return nil, err
	}

	if err = yaml.Unmarshal(decodedSpecYaml, spec); err != nil {
		log.Errorf("Something went wrong loading decoded spec yaml, %s", err)
		return nil, err
	}

	spec.Image = image

	log.Debugf("adapter::imageToSpec -> Got plans %+v", spec.Plans)
	log.Debugf("Successfully converted Image %s into Spec", spec.Image)

	return spec, nil
}
<commit_msg>Improve logging for missing tags (#536)<commit_after>//
// Copyright (c) 2017 Red Hat, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Red Hat trademarks are not licensed under Apache License, Version 2.
// No permission is granted to use or replicate Red Hat trademarks that
// are incorporated in this software or its documentation.
//

package adapters

import (
	"bytes"
	b64 "encoding/base64"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"

	logging "github.com/op/go-logging"
	"github.com/openshift/ansible-service-broker/pkg/apb"
	yaml "gopkg.in/yaml.v1"
)

// Adapter - Adapter will wrap the methods that a registry needs to fully manage images.
type Adapter interface {
	// RegistryName will return the registiry prefix for the adapter.
	// Example is docker.io for the dockerhub adapter.
	RegistryName() string
	// GetImageNames will return all the image names for the adapter configuration.
	GetImageNames() ([]string, error)
	// FetchSpecs will retrieve all the specs for the list of images names.
	FetchSpecs([]string) ([]*apb.Spec, error)
}

// BundleSpecLabel - label on the image that we should use to pull out the abp spec.
// TODO: needs to remain ansibleapp UNTIL we redo the apps in dockerhub
const BundleSpecLabel = "com.redhat.apb.spec"

// Configuration - Adapter configuration. Contains the info that the adapter
// would need to complete its request to the images.
type Configuration struct {
	URL        *url.URL
	User       string
	Pass       string
	Org        string
	Images     []string
	Namespaces []string
	Tag        string
}

// Retrieve the spec from a registry manifest request
func imageToSpec(log *logging.Logger, req *http.Request, image string) (*apb.Spec, error) {
	log.Debug("Registry::imageToSpec")
	spec := &apb.Spec{}
	req.Header.Add("Accept", "application/json")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	type label struct {
		Spec string `json:"com.redhat.apb.spec"`
	}

	type config struct {
		Label label `json:"Labels"`
	}

	hist := struct {
		History []map[string]string `json:"history"`
	}{}

	conf := struct {
		Config *config `json:"config"`
	}{}

	if resp.StatusCode == http.StatusUnauthorized {
		log.Errorf("Unable to authenticate to the registry, registry credentials could be invalid.")
		return nil, nil
	}

	// resp.Body is an io.Reader, which are a one time use.  Save the
	// contents to a byte[] for debugging, then remake the io.Reader for the
	// JSON decoder.
	debug, _ := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		log.Errorf("Image '%s' may not exist in registry.", image)
		log.Error(string(debug))
		return nil, nil
	}

	r := bytes.NewReader(debug)
	err = json.NewDecoder(r).Decode(&hist)
	if err != nil {
		log.Errorf("Error grabbing JSON body from response: %s", err)
		return nil, err
	}

	if hist.History == nil {
		log.Errorf("V1 Schema Manifest does not exist in registry")
		return nil, nil
	}

	err = json.Unmarshal([]byte(hist.History[0]["v1Compatibility"]), &conf)
	if err != nil {
		log.Errorf("Error unmarshalling intermediary JSON response: %s", err)
		return nil, err
	}
	if conf.Config == nil {
		log.Infof("Did not find v1 Manifest in image history. Skipping image")
		return nil, nil
	}
	if conf.Config.Label.Spec == "" {
		log.Infof("Didn't find encoded Spec label. Assuming image is not APB and skiping")
		return nil, nil
	}

	encodedSpec := conf.Config.Label.Spec
	decodedSpecYaml, err := b64.StdEncoding.DecodeString(encodedSpec)
	if err != nil {
		log.Errorf("Something went wrong decoding spec from label")
		return nil, err
	}

	if err = yaml.Unmarshal(decodedSpecYaml, spec); err != nil {
		log.Errorf("Something went wrong loading decoded spec yaml, %s", err)
		return nil, err
	}

	spec.Image = image

	log.Debugf("adapter::imageToSpec -> Got plans %+v", spec.Plans)
	log.Debugf("Successfully converted Image %s into Spec", spec.Image)

	return spec, nil
}
<|endoftext|>