<commit_before>package appdash

import (
	"errors"
	"fmt"
	"net/url"
	"reflect"
	"time"

	influxDBClient "github.com/influxdb/influxdb/client"
	influxDBServer "github.com/influxdb/influxdb/cmd/influxd/run"
	influxDBModels "github.com/influxdb/influxdb/models"
)

const (
	dbName               string = "appdash" // InfluxDB db name.
	spanMeasurementName  string = "spans"   // InfluxDB container name for trace spans.
	defaultTracesPerPage int    = 10        // Default number of traces per page.
)

// Compile-time "implements" check.
var _ interface {
	Store
	Queryer
} = (*InfluxDBStore)(nil)

// TODO: should be a constant.
var zeroID = fmt.Sprintf("%016x", uint64(0))

type InfluxDBStore struct {
	con           *influxDBClient.Client // InfluxDB client connection.
	server        *influxDBServer.Server // InfluxDB API server.
	tracesPerPage int                    // Number of traces per page.
}

func (in *InfluxDBStore) Collect(id SpanID, anns ...Annotation) error {
	p, err := in.findSpanPoint(id)
	if err != nil {
		return err
	}

	// trace_id, span_id & parent_id are set as tags
	// because InfluxDB tags are indexed & those values
	// are used later on queries.
	tags := map[string]string{
		"trace_id":  id.Trace.String(),
		"span_id":   id.Span.String(),
		"parent_id": id.Parent.String(),
	}

	// Saving annotations as InfluxDB measurement spans fields
	// which are not indexed.
	fields := make(map[string]interface{}, len(anns))
	for _, ann := range anns {
		fields[ann.Key] = string(ann.Value)
	}

	if p != nil { // span exists on DB.
		p.Measurement = spanMeasurementName
		p.Tags = tags
		p.Fields = fields
	} else { // new span to be saved on DB.
		p = &influxDBClient.Point{
			Measurement: spanMeasurementName,
			Tags:        tags,   // indexed metadata.
			Fields:      fields, // non-indexed metadata.
			Time:        time.Now().UTC(),
		}
	}

	// InfluxDB point represents a single span.
	pts := []influxDBClient.Point{*p}
	bps := influxDBClient.BatchPoints{
		Points:          pts,
		Database:        dbName,
		RetentionPolicy: "default",
	}
	_, writeErr := in.con.Write(bps)
	if writeErr != nil {
		return writeErr
	}
	return nil
}

func (in *InfluxDBStore) Trace(id ID) (*Trace, error) {
	trace := &Trace{}

	// GROUP BY * -> meaning group by all tags(trace_id, span_id & parent_id)
	// grouping by all tags includes those and it's values on the query response.
	q := fmt.Sprintf("SELECT * FROM spans WHERE trace_id='%s' GROUP BY *", id)
	result, err := in.executeOneQuery(q)
	if err != nil {
		return nil, err
	}

	// result.Series -> A slice containing all the spans.
	if len(result.Series) == 0 {
		return nil, errors.New("trace not found")
	}

	// Iterate over series(spans) to create trace children's & set trace fields.
	var rootSpanSet bool
	for _, s := range result.Series {
		var isRootSpan bool
		span, err := newSpanFromRow(&s)
		if err != nil {
			return nil, err
		}
		if span.ID.IsRoot() && rootSpanSet {
			return nil, errors.New("unexpected multiple root spans")
		}
		if span.ID.IsRoot() && !rootSpanSet {
			isRootSpan = true
		}
		annotations, err := annotationsFromRow(&s, isRootSpan)
		if err != nil {
			return trace, nil
		}
		span.Annotations = *annotations
		if isRootSpan { // root span.
			trace.Span = *span
			rootSpanSet = true
		} else { // children span.
			trace.Sub = append(trace.Sub, &Trace{Span: *span})
		}
	}
	return trace, nil
}

func (in *InfluxDBStore) Traces() ([]*Trace, error) {
	traces := make([]*Trace, 0)

	// GROUP BY * -> meaning group by all tags(trace_id, span_id & parent_id)
	// grouping by all tags includes those and it's values on the query response.
	rootSpansQuery := fmt.Sprintf("SELECT * FROM spans WHERE parent_id='%s' GROUP BY * LIMIT %d", zeroID, in.tracesPerPage)
	rootSpansResult, err := in.executeOneQuery(rootSpansQuery)
	if err != nil {
		return nil, err
	}

	// result.Series -> A slice containing all the spans.
	if len(rootSpansResult.Series) == 0 {
		return traces, nil
	}

	// Cache to keep track of traces to be returned.
	tracesCache := make(map[ID]*Trace, 0)

	// Iterate over series(spans) to create traces.
	for _, s := range rootSpansResult.Series {
		span, err := newSpanFromRow(&s)
		if err != nil {
			return nil, err
		}
		annotations, err := annotationsFromRow(&s, true)
		if err != nil {
			return nil, err
		}
		span.Annotations = *annotations
		_, present := tracesCache[span.ID.Trace]
		if !present {
			tracesCache[span.ID.Trace] = &Trace{Span: *span}
		} else {
			return nil, errors.New("duplicated root span")
		}
	}

	// Using 'OR' since 'IN' not supported yet.
	where := `WHERE `
	var i int = 1
	for _, trace := range tracesCache {
		where += fmt.Sprintf("(trace_id='%s' AND parent_id!='%s')", trace.Span.ID.Trace, zeroID)

		// Adds 'OR' except for last iteration.
		if i != len(tracesCache) && len(tracesCache) > 1 {
			where += " OR "
		}
		i += 1
	}

	// Queries for all children spans of the traces to be returned.
	childrenSpansQuery := fmt.Sprintf("SELECT * FROM spans %s GROUP BY *", where)
	childrenSpansResult, err := in.executeOneQuery(childrenSpansQuery)
	if err != nil {
		return nil, err
	}

	// Iterate over series(children spans) to create sub-traces
	// and associates sub-traces with it's parent trace.
	for _, s := range childrenSpansResult.Series {
		span, err := newSpanFromRow(&s)
		if err != nil {
			return nil, err
		}
		annotations, err := annotationsFromRow(&s, false)
		if err != nil {
			return nil, err
		}
		span.Annotations = *annotations
		trace, present := tracesCache[span.ID.Trace]
		if !present { // Root trace not added.
			return nil, errors.New("parent not found")
		} else { // Root trace already added so append a sub-trace.
			trace.Sub = append(trace.Sub, &Trace{Span: *span})
		}
	}
	for _, trace := range tracesCache {
		traces = append(traces, trace)
	}
	return traces, nil
}

func (in *InfluxDBStore) Close() {
	in.server.Close()
}

func (in *InfluxDBStore) createDBIfNotExists() error {
	// If no errors query execution was successfully - either DB was created or already exists.
	response, err := in.con.Query(influxDBClient.Query{
		Command: fmt.Sprintf("%s %s", "CREATE DATABASE IF NOT EXISTS", dbName),
	})
	if err != nil {
		return err
	}
	if response.Error() != nil {
		return response.Error()
	}
	return nil
}

func (in *InfluxDBStore) executeOneQuery(command string) (*influxDBClient.Result, error) {
	response, err := in.con.Query(influxDBClient.Query{
		Command:  command,
		Database: dbName,
	})
	if err != nil {
		return nil, err
	}
	if response.Error() != nil {
		return nil, response.Error()
	}

	// Expecting one result, since a single query is executed.
	if len(response.Results) != 1 {
		return nil, errors.New("unexpected number of results for an influxdb single query")
	}
	return &response.Results[0], nil
}

func (in *InfluxDBStore) findSpanPoint(ID SpanID) (*influxDBClient.Point, error) {
	q := fmt.Sprintf(`
		SELECT * FROM spans WHERE trace_id='%s' AND span_id='%s' AND parent_id='%s' GROUP BY *
	`, ID.Trace, ID.Span, ID.Parent)
	result, err := in.executeOneQuery(q)
	if err != nil {
		return nil, err
	}
	if len(result.Series) == 0 {
		return nil, nil
	}
	if len(result.Series) > 1 {
		return nil, errors.New("unexpected multiple series")
	}
	r := result.Series[0]
	if len(r.Values) == 0 {
		return nil, errors.New("unexpected empty series")
	}
	p := influxDBClient.Point{}
	fields := r.Values[0]
	for i, field := range fields {
		key := r.Columns[i]
		if key == "time" {
			switch field.(type) {
			case string:
				t, err := time.Parse(time.RFC3339Nano, field.(string))
				if err != nil {
					return nil, err
				}
				p.Time = t
			default:
				return nil, fmt.Errorf("unexpected time type: %v", reflect.TypeOf(field))
			}
		}
	}
	return &p, err
}

func (in *InfluxDBStore) init(server *influxDBServer.Server) error {
	in.server = server
	url, err := url.Parse(fmt.Sprintf("http://%s:%d", influxDBClient.DefaultHost, influxDBClient.DefaultPort))
	if err != nil {
		return err
	}
	con, err := influxDBClient.NewClient(influxDBClient.Config{URL: *url})
	if err != nil {
		return err
	}
	in.con = con
	if err := in.createDBIfNotExists(); err != nil {
		return err
	}
	in.tracesPerPage = defaultTracesPerPage
	return nil
}

func annotationsFromRow(r *influxDBModels.Row, isRootSpan bool) (*Annotations, error) {
	// Actually an influxDBModels.Row represents a single InfluxDB serie.
	// r.Values[n] is a slice containing span's annotation values.
	var fields []interface{}
	if len(r.Values) == 1 {
		fields = r.Values[0]
	}

	// len(r.Values) might be greater than one - meaning there are
	// some spans to drop, see: InfluxDBStore.Collect(...).
	// If so last one is picked.
	if len(r.Values) > 1 {
		fields = r.Values[len(r.Values)-1]
	}
	annotations := make(Annotations, 0)

	// Iterates over fields which represent span's annotation values.
	for i, field := range fields {
		// It is safe to do column[0] (eg. 'Server.Request.Method')
		// matches fields[0] (eg. 'GET')
		key := r.Columns[i]
		var value []byte
		switch field.(type) {
		case string:
			value = []byte(field.(string))
		case nil:
		default:
			return nil, fmt.Errorf("unexpected field type: %v", reflect.TypeOf(field))
		}
		if isRootSpan && ignoreKey(key) {
			continue
		}
		a := Annotation{
			Key:   key,
			Value: value,
		}
		annotations = append(annotations, a)
	}

	return &annotations, nil
}

func ignoreKey(key string) bool {
	keysToIgnore := []string{"_schema:HTTPClient"}
	for _, k := range keysToIgnore {
		if k == key {
			return true
		}
	}
	return false
}

func newSpanFromRow(r *influxDBModels.Row) (*Span, error) {
	span := &Span{}
	traceID, err := ParseID(r.Tags["trace_id"])
	if err != nil {
		return nil, err
	}
	spanID, err := ParseID(r.Tags["span_id"])
	if err != nil {
		return nil, err
	}
	parentID, err := ParseID(r.Tags["parent_id"])
	if err != nil {
		return nil, err
	}
	span.ID = SpanID{
		Trace:  ID(traceID),
		Span:   ID(spanID),
		Parent: ID(parentID),
	}
	return span, nil
}

type InfluxDBStoreConfig struct {
	Server    *influxDBServer.Config
	BuildInfo *influxDBServer.BuildInfo
}

func NewInfluxDBStore(config InfluxDBStoreConfig) (*InfluxDBStore, error) {
	//TODO: add Authentication.
	s, err := influxDBServer.NewServer(config.Server, config.BuildInfo)
	if err != nil {
		return nil, err
	}
	if err := s.Open(); err != nil {
		return nil, err
	}
	var in InfluxDBStore
	if err := in.init(s); err != nil {
		return nil, err
	}
	return &in, nil
}
<commit_msg>Revert "temp fix for frontend hanging" - Lasting fix on 7a778051de9c556fc142c557ab13d0032bb6288e<commit_after>package appdash

import (
	"errors"
	"fmt"
	"net/url"
	"reflect"
	"time"

	influxDBClient "github.com/influxdb/influxdb/client"
	influxDBServer "github.com/influxdb/influxdb/cmd/influxd/run"
	influxDBModels "github.com/influxdb/influxdb/models"
)

const (
	dbName               string = "appdash" // InfluxDB db name.
	spanMeasurementName  string = "spans"   // InfluxDB container name for trace spans.
	defaultTracesPerPage int    = 10        // Default number of traces per page.
)

// Compile-time "implements" check.
var _ interface {
	Store
	Queryer
} = (*InfluxDBStore)(nil)

// TODO: should be a constant.
var zeroID = fmt.Sprintf("%016x", uint64(0))

type InfluxDBStore struct {
	con           *influxDBClient.Client // InfluxDB client connection.
	server        *influxDBServer.Server // InfluxDB API server.
	tracesPerPage int                    // Number of traces per page.
}

func (in *InfluxDBStore) Collect(id SpanID, anns ...Annotation) error {
	p, err := in.findSpanPoint(id)
	if err != nil {
		return err
	}

	// trace_id, span_id & parent_id are set as tags
	// because InfluxDB tags are indexed & those values
	// are used later on queries.
	tags := map[string]string{
		"trace_id":  id.Trace.String(),
		"span_id":   id.Span.String(),
		"parent_id": id.Parent.String(),
	}

	// Saving annotations as InfluxDB measurement spans fields
	// which are not indexed.
	fields := make(map[string]interface{}, len(anns))
	for _, ann := range anns {
		fields[ann.Key] = string(ann.Value)
	}

	if p != nil { // span exists on DB.
		p.Measurement = spanMeasurementName
		p.Tags = tags
		p.Fields = fields
	} else { // new span to be saved on DB.
		p = &influxDBClient.Point{
			Measurement: spanMeasurementName,
			Tags:        tags,   // indexed metadata.
			Fields:      fields, // non-indexed metadata.
			Time:        time.Now().UTC(),
		}
	}

	// InfluxDB point represents a single span.
	pts := []influxDBClient.Point{*p}
	bps := influxDBClient.BatchPoints{
		Points:          pts,
		Database:        dbName,
		RetentionPolicy: "default",
	}
	_, writeErr := in.con.Write(bps)
	if writeErr != nil {
		return writeErr
	}
	return nil
}

func (in *InfluxDBStore) Trace(id ID) (*Trace, error) {
	trace := &Trace{}

	// GROUP BY * -> meaning group by all tags(trace_id, span_id & parent_id)
	// grouping by all tags includes those and it's values on the query response.
	q := fmt.Sprintf("SELECT * FROM spans WHERE trace_id='%s' GROUP BY *", id)
	result, err := in.executeOneQuery(q)
	if err != nil {
		return nil, err
	}

	// result.Series -> A slice containing all the spans.
	if len(result.Series) == 0 {
		return nil, errors.New("trace not found")
	}

	// Iterate over series(spans) to create trace children's & set trace fields.
	var rootSpanSet bool
	for _, s := range result.Series {
		var isRootSpan bool
		span, err := newSpanFromRow(&s)
		if err != nil {
			return nil, err
		}
		annotations, err := annotationsFromRow(&s)
		if err != nil {
			return trace, nil
		}
		span.Annotations = *annotations
		if span.ID.IsRoot() && rootSpanSet {
			return nil, errors.New("unexpected multiple root spans")
		}
		if span.ID.IsRoot() && !rootSpanSet {
			isRootSpan = true
		}
		if isRootSpan { // root span.
			trace.Span = *span
			rootSpanSet = true
		} else { // children span.
			trace.Sub = append(trace.Sub, &Trace{Span: *span})
		}
	}
	return trace, nil
}

func (in *InfluxDBStore) Traces() ([]*Trace, error) {
	traces := make([]*Trace, 0)

	// GROUP BY * -> meaning group by all tags(trace_id, span_id & parent_id)
	// grouping by all tags includes those and it's values on the query response.
	rootSpansQuery := fmt.Sprintf("SELECT * FROM spans WHERE parent_id='%s' GROUP BY * LIMIT %d", zeroID, in.tracesPerPage)
	rootSpansResult, err := in.executeOneQuery(rootSpansQuery)
	if err != nil {
		return nil, err
	}

	// result.Series -> A slice containing all the spans.
	if len(rootSpansResult.Series) == 0 {
		return traces, nil
	}

	// Cache to keep track of traces to be returned.
	tracesCache := make(map[ID]*Trace, 0)

	// Iterate over series(spans) to create traces.
	for _, s := range rootSpansResult.Series {
		span, err := newSpanFromRow(&s)
		if err != nil {
			return nil, err
		}
		annotations, err := annotationsFromRow(&s)
		if err != nil {
			return nil, err
		}
		span.Annotations = *annotations
		_, present := tracesCache[span.ID.Trace]
		if !present {
			tracesCache[span.ID.Trace] = &Trace{Span: *span}
		} else {
			return nil, errors.New("duplicated root span")
		}
	}

	// Using 'OR' since 'IN' not supported yet.
	where := `WHERE `
	var i int = 1
	for _, trace := range tracesCache {
		where += fmt.Sprintf("(trace_id='%s' AND parent_id!='%s')", trace.Span.ID.Trace, zeroID)

		// Adds 'OR' except for last iteration.
		if i != len(tracesCache) && len(tracesCache) > 1 {
			where += " OR "
		}
		i += 1
	}

	// Queries for all children spans of the traces to be returned.
	childrenSpansQuery := fmt.Sprintf("SELECT * FROM spans %s GROUP BY *", where)
	childrenSpansResult, err := in.executeOneQuery(childrenSpansQuery)
	if err != nil {
		return nil, err
	}

	// Iterate over series(children spans) to create sub-traces
	// and associates sub-traces with it's parent trace.
	for _, s := range childrenSpansResult.Series {
		span, err := newSpanFromRow(&s)
		if err != nil {
			return nil, err
		}
		annotations, err := annotationsFromRow(&s)
		if err != nil {
			return nil, err
		}
		span.Annotations = *annotations
		trace, present := tracesCache[span.ID.Trace]
		if !present { // Root trace not added.
			return nil, errors.New("parent not found")
		} else { // Root trace already added so append a sub-trace.
			trace.Sub = append(trace.Sub, &Trace{Span: *span})
		}
	}
	for _, trace := range tracesCache {
		traces = append(traces, trace)
	}
	return traces, nil
}

func (in *InfluxDBStore) Close() {
	in.server.Close()
}

func (in *InfluxDBStore) createDBIfNotExists() error {
	// If no errors query execution was successfully - either DB was created or already exists.
	response, err := in.con.Query(influxDBClient.Query{
		Command: fmt.Sprintf("%s %s", "CREATE DATABASE IF NOT EXISTS", dbName),
	})
	if err != nil {
		return err
	}
	if response.Error() != nil {
		return response.Error()
	}
	return nil
}

func (in *InfluxDBStore) executeOneQuery(command string) (*influxDBClient.Result, error) {
	response, err := in.con.Query(influxDBClient.Query{
		Command:  command,
		Database: dbName,
	})
	if err != nil {
		return nil, err
	}
	if response.Error() != nil {
		return nil, response.Error()
	}

	// Expecting one result, since a single query is executed.
	if len(response.Results) != 1 {
		return nil, errors.New("unexpected number of results for an influxdb single query")
	}
	return &response.Results[0], nil
}

func (in *InfluxDBStore) findSpanPoint(ID SpanID) (*influxDBClient.Point, error) {
	q := fmt.Sprintf(`
		SELECT * FROM spans WHERE trace_id='%s' AND span_id='%s' AND parent_id='%s' GROUP BY *
	`, ID.Trace, ID.Span, ID.Parent)
	result, err := in.executeOneQuery(q)
	if err != nil {
		return nil, err
	}
	if len(result.Series) == 0 {
		return nil, nil
	}
	if len(result.Series) > 1 {
		return nil, errors.New("unexpected multiple series")
	}
	r := result.Series[0]
	if len(r.Values) == 0 {
		return nil, errors.New("unexpected empty series")
	}
	p := influxDBClient.Point{}
	fields := r.Values[0]
	for i, field := range fields {
		key := r.Columns[i]
		if key == "time" {
			switch field.(type) {
			case string:
				t, err := time.Parse(time.RFC3339Nano, field.(string))
				if err != nil {
					return nil, err
				}
				p.Time = t
			default:
				return nil, fmt.Errorf("unexpected time type: %v", reflect.TypeOf(field))
			}
		}
	}
	return &p, err
}

func (in *InfluxDBStore) init(server *influxDBServer.Server) error {
	in.server = server
	url, err := url.Parse(fmt.Sprintf("http://%s:%d", influxDBClient.DefaultHost, influxDBClient.DefaultPort))
	if err != nil {
		return err
	}
	con, err := influxDBClient.NewClient(influxDBClient.Config{URL: *url})
	if err != nil {
		return err
	}
	in.con = con
	if err := in.createDBIfNotExists(); err != nil {
		return err
	}
	in.tracesPerPage = defaultTracesPerPage
	return nil
}

func annotationsFromRow(r *influxDBModels.Row) (*Annotations, error) {
	// Actually an influxDBModels.Row represents a single InfluxDB serie.
	// r.Values[n] is a slice containing span's annotation values.
	var fields []interface{}
	if len(r.Values) == 1 {
		fields = r.Values[0]
	}

	// len(r.Values) might be greater than one - meaning there are
	// some spans to drop, see: InfluxDBStore.Collect(...).
	// If so last one is picked.
	if len(r.Values) > 1 {
		fields = r.Values[len(r.Values)-1]
	}
	annotations := make(Annotations, 0)

	// Iterates over fields which represent span's annotation values.
	for i, field := range fields {
		// It is safe to do column[0] (eg. 'Server.Request.Method')
		// matches fields[0] (eg. 'GET')
		key := r.Columns[i]
		var value []byte
		switch field.(type) {
		case string:
			value = []byte(field.(string))
		case nil:
		default:
			return nil, fmt.Errorf("unexpected field type: %v", reflect.TypeOf(field))
		}
		a := Annotation{
			Key:   key,
			Value: value,
		}
		annotations = append(annotations, a)
	}

	return &annotations, nil
}

func newSpanFromRow(r *influxDBModels.Row) (*Span, error) {
	span := &Span{}
	traceID, err := ParseID(r.Tags["trace_id"])
	if err != nil {
		return nil, err
	}
	spanID, err := ParseID(r.Tags["span_id"])
	if err != nil {
		return nil, err
	}
	parentID, err := ParseID(r.Tags["parent_id"])
	if err != nil {
		return nil, err
	}
	span.ID = SpanID{
		Trace:  ID(traceID),
		Span:   ID(spanID),
		Parent: ID(parentID),
	}
	return span, nil
}

type InfluxDBStoreConfig struct {
	Server    *influxDBServer.Config
	BuildInfo *influxDBServer.BuildInfo
}

func NewInfluxDBStore(config InfluxDBStoreConfig) (*InfluxDBStore, error) {
	//TODO: add Authentication.
	s, err := influxDBServer.NewServer(config.Server, config.BuildInfo)
	if err != nil {
		return nil, err
	}
	if err := s.Open(); err != nil {
		return nil, err
	}
	var in InfluxDBStore
	if err := in.init(s); err != nil {
		return nil, err
	}
	return &in, nil
}
<|endoftext|>


<commit_before>package local

import (
	"github.com/globocom/tsuru/fs"
	"github.com/globocom/tsuru/log"
	"io/ioutil"
	"os/exec"
	"strings"
)

var fsystem fs.Fs

func filesystem() fs.Fs {
	if fsystem == nil {
		fsystem = fs.OsFs{}
	}
	return fsystem
}

// container represents an lxc container with the given name.
type container struct {
	name string
}

// runCmd executes commands and log the given stdout and stderror.
func runCmd(cmd string, args ...string) error {
	command := exec.Command(cmd, args...)
	out, err := command.CombinedOutput()
	log.Printf("running the cmd: %s with the args: %s", cmd, args)
	log.Print(string(out))
	return err
}

// ip returns the ip for the container.
func (c *container) ip() string {
	file, _ := filesystem().Open("/var/lib/misc/dnsmasq.leases")
	data, _ := ioutil.ReadAll(file)
	for _, line := range strings.Split(string(data), "\n") {
		if strings.Index(line, c.name) != -1 {
			return strings.Split(line, " ")[2]
		}
	}
	return ""
}

// create creates a lxc container with ubuntu template by default.
func (c *container) create() error {
	return runCmd("sudo", "lxc-create", "-t", "ubuntu", "-n", c.name)
}

// start starts a lxc container.
func (c *container) start() error {
	return runCmd("sudo", "lxc-start", "--daemon", "-n", c.name)
}

// stop stops a lxc container.
func (c *container) stop() error {
	return runCmd("sudo", "lxc-stop", "-n", c.name)
}

// destroy destory a lxc container.
func (c *container) destroy() error {
	return runCmd("sudo", "lxc-destroy", "-n", c.name)
}
<commit_msg>provision/local: added some logs on ip.<commit_after>package local

import (
	"github.com/globocom/tsuru/fs"
	"github.com/globocom/tsuru/log"
	"io/ioutil"
	"os/exec"
	"strings"
)

var fsystem fs.Fs

func filesystem() fs.Fs {
	if fsystem == nil {
		fsystem = fs.OsFs{}
	}
	return fsystem
}

// container represents an lxc container with the given name.
type container struct {
	name string
}

// runCmd executes commands and log the given stdout and stderror.
func runCmd(cmd string, args ...string) error {
	command := exec.Command(cmd, args...)
	out, err := command.CombinedOutput()
	log.Printf("running the cmd: %s with the args: %s", cmd, args)
	log.Print(string(out))
	return err
}

// ip returns the ip for the container.
func (c *container) ip() string {
	file, _ := filesystem().Open("/var/lib/misc/dnsmasq.leases")
	data, _ := ioutil.ReadAll(file)
	log.Print(data)
	for _, line := range strings.Split(string(data), "\n") {
		if strings.Index(line, c.name) != -1 {
			log.Printf("ip in %s", line)
			return strings.Split(line, " ")[2]
		}
	}
	return ""
}

// create creates a lxc container with ubuntu template by default.
func (c *container) create() error {
	return runCmd("sudo", "lxc-create", "-t", "ubuntu", "-n", c.name)
}

// start starts a lxc container.
func (c *container) start() error {
	return runCmd("sudo", "lxc-start", "--daemon", "-n", c.name)
}

// stop stops a lxc container.
func (c *container) stop() error {
	return runCmd("sudo", "lxc-stop", "-n", c.name)
}

// destroy destory a lxc container.
func (c *container) destroy() error {
	return runCmd("sudo", "lxc-destroy", "-n", c.name)
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package provision provides interfaces that need to be satisfied in order to
// implement a new provisioner on tsuru.
package provision

import (
	"fmt"
	"io"
)

type Status string

func (s Status) String() string {
	return string(s)
}

const (
	StatusStarted    = Status("started")
	StatusPending    = Status("pending")
	StatusDown       = Status("down")
	StatusError      = Status("error")
	StatusInstalling = Status("installing")
	StatusCreating   = Status("creating")
)

// Unit represents a provision unit. Can be a machine, container or anything
// IP-addressable.
type Unit struct {
	Name       string
	AppName    string
	Type       string
	InstanceId string
	Machine    int
	Ip         string
	Status     Status
}

// Named is something that has a name, providing the GetName method.
type Named interface {
	GetName() string
}

// AppUnit represents a unit in an app.
type AppUnit interface {
	Named
	GetMachine() int
	GetStatus() Status
	GetIp() string
	GetInstanceId() string
}

// App represents a tsuru app.
//
// It contains only relevant information for provisioning.
type App interface {
	Named
	// Log should be used to log messages in the app.
	Log(message, source string) error

	// GetPlatform returns the platform (type) of the app. It is equivalent
	// to the Unit `Type` field.
	GetPlatform() string

	ProvisionUnits() []AppUnit
	RemoveUnit(id string) error

	// Run executes the command in app units, sourcing apprc before running the
	// command.
	Run(cmd string, w io.Writer) error

	Restart(io.Writer) error

	// Ready marks the app as ready for deployment.
	Ready() error
}

type CNameManager interface {
	SetCName(app App, cname string) error
	UnsetCName(app App, cname string) error
}

// Provisioner is the basic interface of this package.
//
// Any tsuru provisioner must implement this interface in order to provision
// tsuru apps.
//
// Tsuru comes with a default provisioner: juju. One can add other provisioners
// by satisfying this interface and registering it using the function Register.
type Provisioner interface {
	// Deploy updates the code of the app in units to match the given
	// version, logging progress in the given writer.
	Deploy(app App, version string, w io.Writer) error

	// Provision is called when tsuru is creating the app.
	Provision(App) error

	// Destroy is called when tsuru is destroying the app.
	Destroy(App) error

	// AddUnits adds units to an app. The first parameter is the app, the
	// second is the number of units to add.
	//
	// It returns a slice containing all added units
	AddUnits(App, uint) ([]Unit, error)

	// RemoveUnit removes a unit from the app. It receives the app and the name
	// of the unit to be removed.
	RemoveUnit(App, string) error

	// ExecuteCommand runs a command in all units of the app.
	ExecuteCommand(stdout, stderr io.Writer, app App, cmd string, args ...string) error

	Restart(App) error

	// CollectStatus returns information about all provisioned units. It's used
	// by tsuru collector when updating the status of apps in the database.
	CollectStatus() ([]Unit, error)

	// Addr returns the address for an app.
	//
	// Tsuru will use this method to get the IP (althought it might not be
	// an actual IP, collector calls it "IP") of the app from the
	// provisioner.
	Addr(App) (string, error)

	// InstallDeps installs the dependencies required for the application
	// to run and writes the log in the received writer.
	InstallDeps(app App, w io.Writer) error
}

var provisioners = make(map[string]Provisioner)

// Register registers a new provisioner in the Provisioner registry.
func Register(name string, p Provisioner) {
	provisioners[name] = p
}

// Get gets the named provisioner from the registry.
func Get(name string) (Provisioner, error) {
	p, ok := provisioners[name]
	if !ok {
		return nil, fmt.Errorf("Unknown provisioner: %q.", name)
	}
	return p, nil
}

type Error struct {
	Reason string
	Err    error
}

func (e *Error) Error() string {
	var err string
	if e.Err != nil {
		err = e.Err.Error() + ": " + e.Reason
	} else {
		err = e.Reason
	}
	return err
}
<commit_msg>provision: add SerializeEnvVars method to the App interface<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package provision provides interfaces that need to be satisfied in order to
// implement a new provisioner on tsuru.
package provision

import (
	"fmt"
	"io"
)

type Status string

func (s Status) String() string {
	return string(s)
}

const (
	StatusStarted    = Status("started")
	StatusPending    = Status("pending")
	StatusDown       = Status("down")
	StatusError      = Status("error")
	StatusInstalling = Status("installing")
	StatusCreating   = Status("creating")
)

// Unit represents a provision unit. Can be a machine, container or anything
// IP-addressable.
type Unit struct {
	Name       string
	AppName    string
	Type       string
	InstanceId string
	Machine    int
	Ip         string
	Status     Status
}

// Named is something that has a name, providing the GetName method.
type Named interface {
	GetName() string
}

// AppUnit represents a unit in an app.
type AppUnit interface {
	Named
	GetMachine() int
	GetStatus() Status
	GetIp() string
	GetInstanceId() string
}

// App represents a tsuru app.
//
// It contains only relevant information for provisioning.
type App interface {
	Named
	// Log should be used to log messages in the app.
	Log(message, source string) error

	// GetPlatform returns the platform (type) of the app. It is equivalent
	// to the Unit `Type` field.
	GetPlatform() string

	ProvisionUnits() []AppUnit
	RemoveUnit(id string) error

	// Run executes the command in app units, sourcing apprc before running the
	// command.
	Run(cmd string, w io.Writer) error

	Restart(io.Writer) error

	SerializeEnvVars() error

	// Ready marks the app as ready for deployment.
	Ready() error
}

type CNameManager interface {
	SetCName(app App, cname string) error
	UnsetCName(app App, cname string) error
}

// Provisioner is the basic interface of this package.
//
// Any tsuru provisioner must implement this interface in order to provision
// tsuru apps.
//
// Tsuru comes with a default provisioner: juju. One can add other provisioners
// by satisfying this interface and registering it using the function Register.
type Provisioner interface {
	// Deploy updates the code of the app in units to match the given
	// version, logging progress in the given writer.
	Deploy(app App, version string, w io.Writer) error

	// Provision is called when tsuru is creating the app.
	Provision(App) error

	// Destroy is called when tsuru is destroying the app.
	Destroy(App) error

	// AddUnits adds units to an app. The first parameter is the app, the
	// second is the number of units to add.
	//
	// It returns a slice containing all added units
	AddUnits(App, uint) ([]Unit, error)

	// RemoveUnit removes a unit from the app. It receives the app and the name
	// of the unit to be removed.
	RemoveUnit(App, string) error

	// ExecuteCommand runs a command in all units of the app.
	ExecuteCommand(stdout, stderr io.Writer, app App, cmd string, args ...string) error

	Restart(App) error

	// CollectStatus returns information about all provisioned units. It's used
	// by tsuru collector when updating the status of apps in the database.
	CollectStatus() ([]Unit, error)

	// Addr returns the address for an app.
	//
	// Tsuru will use this method to get the IP (althought it might not be
	// an actual IP, collector calls it "IP") of the app from the
	// provisioner.
	Addr(App) (string, error)

	// InstallDeps installs the dependencies required for the application
	// to run and writes the log in the received writer.
	InstallDeps(app App, w io.Writer) error
}

var provisioners = make(map[string]Provisioner)

// Register registers a new provisioner in the Provisioner registry.
func Register(name string, p Provisioner) {
	provisioners[name] = p
}

// Get gets the named provisioner from the registry.
func Get(name string) (Provisioner, error) {
	p, ok := provisioners[name]
	if !ok {
		return nil, fmt.Errorf("Unknown provisioner: %q.", name)
	}
	return p, nil
}

type Error struct {
	Reason string
	Err    error
}

func (e *Error) Error() string {
	var err string
	if e.Err != nil {
		err = e.Err.Error() + ": " + e.Reason
	} else {
		err = e.Reason
	}
	return err
}
<|endoftext|>


<commit_before>package proxy

import (
	"encoding/json"
	"fmt"
	"log"
	"strings"

	"github.com/andystanton/proxybastard/util"
)

func (dockerMachineConfiguration DockerMachineConfiguration) validate() error {
	return nil
}

func (dockerMachineConfiguration DockerMachineConfiguration) isEnabled() bool {
	return dockerMachineConfiguration.Enabled
}

func (dockerMachineConfiguration DockerMachineConfiguration) suggestConfiguration() *Configuration {
	dockerMachineExecutable := "docker-machine"
	_, err := util.ShellOut("which", []string{dockerMachineExecutable})
	if err == nil {
		return &Configuration{
			Targets: &TargetsConfiguration{
				DockerMachine: &DockerMachineConfiguration{
					Enabled: true,
					Hosts:   listDockerMachines(),
				},
			},
		}
	}
	return nil
}

func (dockerMachineConfiguration DockerMachineConfiguration) addProxySettings(proxyHost string, proxyPort string, nonProxyHosts []string) {
	for _, machine := range listDockerMachines() {
		sshRunConfiguration := dockerMachineConfiguration.extractRunSSHConfiguration(machine)

		removeFromBoot2DockerProfile(sshRunConfiguration)
		addToBoot2DockerProfile(sshRunConfiguration, proxyHost, proxyPort)
		rebootBoot2docker(sshRunConfiguration)
	}
}

func (dockerMachineConfiguration DockerMachineConfiguration) removeProxySettings() {
	for _, machine := range listDockerMachines() {
		sshRunConfiguration := dockerMachineConfiguration.extractRunSSHConfiguration(machine)

		removeFromBoot2DockerProfile(sshRunConfiguration)
		rebootBoot2docker(sshRunConfiguration)
	}
}

func (dockerMachineConfiguration DockerMachineConfiguration) extractRunSSHConfiguration(machine string) util.RunSSHConfiguration {
	var data map[string]interface{}

	inspection, err := util.ShellOut("docker-machine", []string{"inspect", machine})
	if err != nil {
		log.Fatal(err)
	}
	if err := json.Unmarshal([]byte(inspection), &data); err != nil {
		log.Fatal(err)
	}

	driverInfo := data["Driver"].(map[string]interface{})

	return util.RunSSHConfiguration{
		SSHHost: driverInfo["IPAddress"].(string),
		SSHPort: "22", // a value is specified in driverInfo["SSHPort"] but 22 appears to be used instead
		SSHUser: driverInfo["SSHUser"].(string),
		SSHKey:  fmt.Sprintf("%s/%s", data["StorePath"].(string), "id_rsa"),
	}
}

func listDockerMachines() []string {
	machineOut, err := util.ShellOut("docker-machine", []string{"ls", "-q", "--filter", "driver=virtualbox"})
	if err != nil {
		log.Fatal(err)
	}
	return strings.Split(strings.TrimSpace(machineOut), "\n")
}
<commit_msg>#47 don't suggest/add docker-machine config when either docker-machine is not present or there are no docker-machine hosts<commit_after>package proxy

import (
	"encoding/json"
	"fmt"
	"log"
	"strings"

	"github.com/andystanton/proxybastard/util"
)

func (dockerMachineConfiguration DockerMachineConfiguration) validate() error {
	return nil
}

func (dockerMachineConfiguration DockerMachineConfiguration) isEnabled() bool {
	return dockerMachineConfiguration.Enabled
}

func (dockerMachineConfiguration DockerMachineConfiguration) suggestConfiguration() (configuration *Configuration) {
	configuration = nil
	dockerMachineExecutable := "docker-machine"
	_, err := util.ShellOut("which", []string{dockerMachineExecutable})
	if err == nil {
		hosts := listDockerMachines()
		if len(hosts) > 0 {
			configuration = &Configuration{
				Targets: &TargetsConfiguration{
					DockerMachine: &DockerMachineConfiguration{
						Enabled: true,
						Hosts:   hosts,
					},
				},
			}
		}
	}
	return
}

func (dockerMachineConfiguration DockerMachineConfiguration) addProxySettings(proxyHost string, proxyPort string, nonProxyHosts []string) {
	for _, machine := range listDockerMachines() {
		sshRunConfiguration := dockerMachineConfiguration.extractRunSSHConfiguration(machine)

		removeFromBoot2DockerProfile(sshRunConfiguration)
		addToBoot2DockerProfile(sshRunConfiguration, proxyHost, proxyPort)
		rebootBoot2docker(sshRunConfiguration)
	}
}

func (dockerMachineConfiguration DockerMachineConfiguration) removeProxySettings() {
	for _, machine := range listDockerMachines() {
		sshRunConfiguration := dockerMachineConfiguration.extractRunSSHConfiguration(machine)

		removeFromBoot2DockerProfile(sshRunConfiguration)
		rebootBoot2docker(sshRunConfiguration)
	}
}

func (dockerMachineConfiguration DockerMachineConfiguration) extractRunSSHConfiguration(machine string) util.RunSSHConfiguration {
	var data map[string]interface{}

	inspection, err := util.ShellOut("docker-machine", []string{"inspect", machine})
	if err != nil {
		log.Fatal(err)
	}
	if err := json.Unmarshal([]byte(inspection), &data); err != nil {
		log.Fatal(err)
	}

	driverInfo := data["Driver"].(map[string]interface{})

	return util.RunSSHConfiguration{
		SSHHost: driverInfo["IPAddress"].(string),
		SSHPort: "22", // a value is specified in driverInfo["SSHPort"] but 22 appears to be used instead
		SSHUser: driverInfo["SSHUser"].(string),
		SSHKey:  fmt.Sprintf("%s/%s", data["StorePath"].(string), "id_rsa"),
	}
}

func listDockerMachines() []string {
	machines := []string{}
	machineOut, err := util.ShellOut("docker-machine", []string{"ls", "-q", "--filter", "driver=virtualbox"})
	if err != nil {
		log.Fatal(err)
	}
	trimmed := strings.TrimSpace(machineOut)
	if len(trimmed) > 0 {
		machines = strings.Split(strings.TrimSpace(machineOut), "\n")
	}
	return machines
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"
)

var (
	bashPath    string
	debugging   = false
	erroring    = false
	maxprocs    = 4
	testPattern = regexp.MustCompile(`test[/\\]test-([a-z\-]+)\.sh$`)
)

func mainIntegration() {
	if len(os.Getenv("DEBUG")) > 0 {
		debugging = true
	}

	setBash()

	if maxprocs < 1 {
		maxprocs = 1
	}

	files := testFiles()

	if len(files) == 0 {
		fmt.Println("no tests to run")
		os.Exit(1)
	}

	var wg sync.WaitGroup
	tests := make(chan string, len(files))
	output := make(chan string, len(files))

	for _, file := range files {
		tests <- file
	}

	go printOutput(output)
	for i := 0; i < maxprocs; i++ {
		wg.Add(1)
		go worker(tests, output, &wg)
	}

	close(tests)
	wg.Wait()
	close(output)
	printOutput(output)

	if erroring {
		os.Exit(1)
	}
}

func runTest(output chan string, test string) {
	out, err := exec.Command(bashPath, test).CombinedOutput()
	if err != nil {
		if debugging {
			fmt.Println("Error:", err)
		}
		erroring = true
	}

	output <- strings.TrimSpace(string(out))
}

func printOutput(output <-chan string) {
	for {
		select {
		case out, ok := <-output:
			if !ok {
				return
			}

			fmt.Println(out)
		}
	}
}

func worker(tests <-chan string, output chan string, wg *sync.WaitGroup) {
	defer wg.Done()
	for {
		select {
		case testname, ok := <-tests:
			if !ok {
				return
			}
			runTest(output, testname)
		}
	}
}

func testFiles() []string {
	if len(os.Args) < 4 {
		return allTestFiles()
	}

	fileMap := make(map[string]bool)
	for _, file := range allTestFiles() {
		fileMap[file] = true
	}

	files := make([]string, 0, len(os.Args)-3)
	for _, arg := range os.Args {
		fullname := "test/test-" + arg + ".sh"
		if fileMap[fullname] {
			files = append(files, fullname)
		}
	}

	return files
}

func allTestFiles() []string {
	files := make([]string, 0, 100)
	filepath.Walk("test", func(path string, info os.FileInfo, err error) error {
		if debugging {
			fmt.Println("FOUND:", path)
		}
		if err != nil || info.IsDir() || !testPattern.MatchString(path) {
			return nil
		}

		if debugging {
			fmt.Println("MATCHING:", path)
		}
		files = append(files, path)
		return nil
	})
	return files
}

func setBash() {
	findcmd := "which"
	if runtime.GOOS == "windows" {
		// Can't use paths returned from which even if it's on PATH in Windows
		// Because our Go binary is a separate Windows app & not MinGW, it
		// can't understand paths like '/usr/bin/bash', needs Windows version
		findcmd = "where"
	}
	out, err := exec.Command(findcmd, "bash").Output()
	if err != nil {
		fmt.Println("Unable to find bash:", err)
		os.Exit(1)
	}

	bashPath = strings.TrimSpace(string(out))
	if debugging {
		fmt.Println("Using", bashPath)
	}

	// Test
	_, err = exec.Command(bashPath, "--version").CombinedOutput()
	if err != nil {
		fmt.Println("Error calling bash:", err)
		os.Exit(1)
	}
}
<commit_msg>add timeouts to script/integration, with partial output<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"time"
)

var (
	bashPath    string
	debugging   = false
	erroring    = false
	maxprocs    = 4
	testPattern = regexp.MustCompile(`test[/\\]test-([a-z\-]+)\.sh$`)
)

func mainIntegration() {
	if len(os.Getenv("DEBUG")) > 0 {
		debugging = true
	}

	setBash()

	if maxprocs < 1 {
		maxprocs = 1
	}

	files := testFiles()

	if len(files) == 0 {
		fmt.Println("no tests to run")
		os.Exit(1)
	}

	var wg sync.WaitGroup
	tests := make(chan string, len(files))
	output := make(chan string, len(files))

	for _, file := range files {
		tests <- file
	}

	go printOutput(output)
	for i := 0; i < maxprocs; i++ {
		wg.Add(1)
		go worker(tests, output, &wg)
	}

	close(tests)
	wg.Wait()
	close(output)
	printOutput(output)

	if erroring {
		os.Exit(1)
	}
}

func runTest(output chan string, testname string) {
	buf := &bytes.Buffer{}
	cmd := exec.Command(bashPath, testname)
	cmd.Stdout = buf
	cmd.Stderr = buf

	err := cmd.Start()
	if err != nil {
		sendTestOutput(output, testname, buf, err)
		return
	}

	done := make(chan error)
	go func() {
		if err := cmd.Wait(); err != nil {
			done <- err
		}
		close(done)
	}()

	select {
	case err = <-done:
		sendTestOutput(output, testname, buf, err)
		return
	case <-time.After(3 * time.Minute):
		cmd.Process.Kill()
		sendTestOutput(output, testname, buf, errors.New("Timed out"))
		return
	}

	sendTestOutput(output, testname, buf, nil)
}

func sendTestOutput(output chan string, testname string, buf *bytes.Buffer, err error) {
	cli := strings.TrimSpace(buf.String())
	if len(cli) == 0 {
		cli = fmt.Sprintf("<no output for %s>", testname)
	}

	if err == nil {
		output <- cli
	} else {
		if debugging {
			fmt.Printf("Error on %s: %s\n", testname, err)
		}
		erroring = true
		output <- fmt.Sprintf("error: %s => %s\n%s", testname, err, cli)
	}
}

func printOutput(output <-chan string) {
	for {
		select {
		case out, ok := <-output:
			if !ok {
				return
			}

			fmt.Println(out)
		}
	}
}

func worker(tests <-chan string, output chan string, wg *sync.WaitGroup) {
	defer wg.Done()
	for {
		select {
		case testname, ok := <-tests:
			if !ok {
				return
			}
			runTest(output, testname)
		}
	}
}

func testFiles() []string {
	if len(os.Args) < 4 {
		return allTestFiles()
	}

	fileMap := make(map[string]bool)
	for _, file := range allTestFiles() {
		fileMap[file] = true
	}

	files := make([]string, 0, len(os.Args)-3)
	for _, arg := range os.Args {
		fullname := "test/test-" + arg + ".sh"
		if fileMap[fullname] {
			files = append(files, fullname)
		}
	}

	return files
}

func allTestFiles() []string {
	files := make([]string, 0, 100)
	filepath.Walk("test", func(path string, info os.FileInfo, err error) error {
		if debugging {
			fmt.Println("FOUND:", path)
		}
		if err != nil || info.IsDir() || !testPattern.MatchString(path) {
			return nil
		}

		if debugging {
			fmt.Println("MATCHING:", path)
		}
		files = append(files, path)
		return nil
	})
	return files
}

func setBash() {
	findcmd := "which"
	if runtime.GOOS == "windows" {
		// Can't use paths returned from which even if it's on PATH in Windows
		// Because our Go binary is a separate Windows app & not MinGW, it
		// can't understand paths like '/usr/bin/bash', needs Windows version
		findcmd = "where"
	}
	out, err := exec.Command(findcmd, "bash").Output()
	if err != nil {
		fmt.Println("Unable to find bash:", err)
		os.Exit(1)
	}

	bashPath = strings.TrimSpace(string(out))
	if debugging {
		fmt.Println("Using", bashPath)
	}

	// Test
	_, err = exec.Command(bashPath, "--version").CombinedOutput()
	if err != nil {
		fmt.Println("Error calling bash:", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package benchmark

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"math"
	"path"
	"sort"
	"time"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime/schema"
	coreinformers "k8s.io/client-go/informers/core/v1"
	clientset "k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
	"k8s.io/component-base/metrics/legacyregistry"
	"k8s.io/component-base/metrics/testutil"
	"k8s.io/klog"
	"k8s.io/kubernetes/test/integration/util"
)

const (
	dateFormat                = "2006-01-02T15:04:05Z"
	throughputSampleFrequency = time.Second
)

var dataItemsDir = flag.String("data-items-dir", "", "destination directory for storing generated data items for perf dashboard")

// mustSetupScheduler starts the following components:
// - k8s api server (a.k.a. master)
// - scheduler
// It returns clientset and destroyFunc which should be used to
// remove resources after finished.
// Notes on rate limiter:
//   - client rate limit is set to 5000.
func mustSetupScheduler() (util.ShutdownFunc, coreinformers.PodInformer, clientset.Interface) {
	apiURL, apiShutdown := util.StartApiserver()
	clientSet := clientset.NewForConfigOrDie(&restclient.Config{
		Host:          apiURL,
		ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}},
		QPS:           5000.0,
		Burst:         5000,
	})
	_, podInformer, schedulerShutdown := util.StartScheduler(clientSet)
	fakePVControllerShutdown := util.StartFakePVController(clientSet)

	shutdownFunc := func() {
		fakePVControllerShutdown()
		schedulerShutdown()
		apiShutdown()
	}

	return shutdownFunc, podInformer, clientSet
}

func getScheduledPods(podInformer coreinformers.PodInformer) ([]*v1.Pod, error) {
	pods, err := podInformer.Lister().List(labels.Everything())
	if err != nil {
		return nil, err
	}

	scheduled := make([]*v1.Pod, 0, len(pods))
	for i := range pods {
		pod := pods[i]
		if len(pod.Spec.NodeName) > 0 {
			scheduled = append(scheduled, pod)
		}
	}
	return scheduled, nil
}

// DataItem is the data point.
type DataItem struct {
	// Data is a map from bucket to real data point (e.g. "Perc90" -> 23.5). Notice
	// that all data items with the same label combination should have the same buckets.
	Data map[string]float64 `json:"data"`
	// Unit is the data unit. Notice that all data items with the same label combination
	// should have the same unit.
	Unit string `json:"unit"`
	// Labels is the labels of the data item.
	Labels map[string]string `json:"labels,omitempty"`
}

// DataItems is the data point set. It is the struct that perf dashboard expects.
type DataItems struct {
	Version   string     `json:"version"`
	DataItems []DataItem `json:"dataItems"`
}

func dataItems2JSONFile(dataItems DataItems, namePrefix string) error {
	b, err := json.Marshal(dataItems)
	if err != nil {
		return err
	}

	destFile := fmt.Sprintf("%v_%v.json", namePrefix, time.Now().Format(dateFormat))
	if *dataItemsDir != "" {
		destFile = path.Join(*dataItemsDir, destFile)
	}

	return ioutil.WriteFile(destFile, b, 0644)
}

// metricsCollectorConfig is the config to be marshalled to YAML config file.
type metricsCollectorConfig struct {
	Metrics []string
}

// metricsCollector collects metrics from legacyregistry.DefaultGatherer.Gather() endpoint.
// Currently only Histrogram metrics are supported.
type metricsCollector struct {
	metricsCollectorConfig
	labels map[string]string
}

func newMetricsCollector(config metricsCollectorConfig, labels map[string]string) *metricsCollector {
	return &metricsCollector{
		metricsCollectorConfig: config,
		labels:                 labels,
	}
}

func (*metricsCollector) run(stopCh chan struct{}) {
	// metricCollector doesn't need to start before the tests, so nothing to do here.
}

func (pc *metricsCollector) collect() []DataItem {
	var dataItems []DataItem
	for _, metric := range pc.Metrics {
		dataItem := collectHistogram(metric, pc.labels)
		if dataItem != nil {
			dataItems = append(dataItems, *dataItem)
		}
	}
	return dataItems
}

func collectHistogram(metric string, labels map[string]string) *DataItem {
	hist, err := testutil.GetHistogramFromGatherer(legacyregistry.DefaultGatherer, metric)
	if err != nil {
		klog.Error(err)
		return nil
	}

	if err := hist.Validate(); err != nil {
		klog.Error(err)
		return nil
	}

	q50 := hist.Quantile(0.50)
	q90 := hist.Quantile(0.90)
	q99 := hist.Quantile(0.95)
	avg := hist.Average()

	// clear the metrics so that next test always starts with empty prometheus
	// metrics (since the metrics are shared among all tests run inside the same binary)
	hist.Clear()

	msFactor := float64(time.Second) / float64(time.Millisecond)

	// Copy labels and add "Metric" label for this metric.
	labelMap := map[string]string{"Metric": metric}
	for k, v := range labels {
		labelMap[k] = v
	}
	return &DataItem{
		Labels: labelMap,
		Data: map[string]float64{
			"Perc50":  q50 * msFactor,
			"Perc90":  q90 * msFactor,
			"Perc99":  q99 * msFactor,
			"Average": avg * msFactor,
		},
		Unit: "ms",
	}
}

type throughputCollector struct {
	podInformer           coreinformers.PodInformer
	schedulingThroughputs []float64
	labels                map[string]string
}

func newThroughputCollector(podInformer coreinformers.PodInformer, labels map[string]string) *throughputCollector {
	return &throughputCollector{
		podInformer: podInformer,
		labels:      labels,
	}
}

func (tc *throughputCollector) run(stopCh chan struct{}) {
	podsScheduled, err := getScheduledPods(tc.podInformer)
	if err != nil {
		klog.Fatalf("%v", err)
	}
	lastScheduledCount := len(podsScheduled)
	for {
		select {
		case <-stopCh:
			return
		case <-time.After(throughputSampleFrequency):
			podsScheduled, err := getScheduledPods(tc.podInformer)
			if err != nil {
				klog.Fatalf("%v", err)
			}

			scheduled := len(podsScheduled)
			samplingRatioSeconds := float64(throughputSampleFrequency) / float64(time.Second)
			throughput := float64(scheduled-lastScheduledCount) / samplingRatioSeconds
			tc.schedulingThroughputs = append(tc.schedulingThroughputs, throughput)
			lastScheduledCount = scheduled

			klog.Infof("%d pods scheduled", lastScheduledCount)
		}
	}
}

func (tc *throughputCollector) collect() []DataItem {
	throughputSummary := DataItem{Labels: tc.labels}
	if length := len(tc.schedulingThroughputs); length > 0 {
		sort.Float64s(tc.schedulingThroughputs)
		sum := 0.0
		for i := range tc.schedulingThroughputs {
			sum += tc.schedulingThroughputs[i]
		}

		throughputSummary.Labels = map[string]string{
			"Metric": "SchedulingThroughput",
		}
		throughputSummary.Data = map[string]float64{
			"Average": sum / float64(length),
			"Perc50":  tc.schedulingThroughputs[int(math.Ceil(float64(length*50)/100))-1],
			"Perc90":  tc.schedulingThroughputs[int(math.Ceil(float64(length*90)/100))-1],
			"Perc99":  tc.schedulingThroughputs[int(math.Ceil(float64(length*99)/100))-1],
		}
		throughputSummary.Unit = "pods/s"
	}

	return []DataItem{throughputSummary}
}
<commit_msg>scheduler_perf: do not override throughput labels<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package benchmark

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"math"
	"path"
	"sort"
	"time"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime/schema"
	coreinformers "k8s.io/client-go/informers/core/v1"
	clientset "k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
	"k8s.io/component-base/metrics/legacyregistry"
	"k8s.io/component-base/metrics/testutil"
	"k8s.io/klog"
	"k8s.io/kubernetes/test/integration/util"
)

const (
	dateFormat                = "2006-01-02T15:04:05Z"
	throughputSampleFrequency = time.Second
)

var dataItemsDir = flag.String("data-items-dir", "", "destination directory for storing generated data items for perf dashboard")

// mustSetupScheduler starts the following components:
// - k8s api server (a.k.a. master)
// - scheduler
// It returns clientset and destroyFunc which should be used to
// remove resources after finished.
// Notes on rate limiter:
//   - client rate limit is set to 5000.
func mustSetupScheduler() (util.ShutdownFunc, coreinformers.PodInformer, clientset.Interface) {
	apiURL, apiShutdown := util.StartApiserver()
	clientSet := clientset.NewForConfigOrDie(&restclient.Config{
		Host:          apiURL,
		ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}},
		QPS:           5000.0,
		Burst:         5000,
	})
	_, podInformer, schedulerShutdown := util.StartScheduler(clientSet)
	fakePVControllerShutdown := util.StartFakePVController(clientSet)

	shutdownFunc := func() {
		fakePVControllerShutdown()
		schedulerShutdown()
		apiShutdown()
	}

	return shutdownFunc, podInformer, clientSet
}

func getScheduledPods(podInformer coreinformers.PodInformer) ([]*v1.Pod, error) {
	pods, err := podInformer.Lister().List(labels.Everything())
	if err != nil {
		return nil, err
	}

	scheduled := make([]*v1.Pod, 0, len(pods))
	for i := range pods {
		pod := pods[i]
		if len(pod.Spec.NodeName) > 0 {
			scheduled = append(scheduled, pod)
		}
	}
	return scheduled, nil
}

// DataItem is the data point.
type DataItem struct {
	// Data is a map from bucket to real data point (e.g. "Perc90" -> 23.5). Notice
	// that all data items with the same label combination should have the same buckets.
	Data map[string]float64 `json:"data"`
	// Unit is the data unit. Notice that all data items with the same label combination
	// should have the same unit.
	Unit string `json:"unit"`
	// Labels is the labels of the data item.
	Labels map[string]string `json:"labels,omitempty"`
}

// DataItems is the data point set. It is the struct that perf dashboard expects.
type DataItems struct {
	Version   string     `json:"version"`
	DataItems []DataItem `json:"dataItems"`
}

func dataItems2JSONFile(dataItems DataItems, namePrefix string) error {
	b, err := json.Marshal(dataItems)
	if err != nil {
		return err
	}

	destFile := fmt.Sprintf("%v_%v.json", namePrefix, time.Now().Format(dateFormat))
	if *dataItemsDir != "" {
		destFile = path.Join(*dataItemsDir, destFile)
	}

	return ioutil.WriteFile(destFile, b, 0644)
}

// metricsCollectorConfig is the config to be marshalled to YAML config file.
type metricsCollectorConfig struct {
	Metrics []string
}

// metricsCollector collects metrics from legacyregistry.DefaultGatherer.Gather() endpoint.
// Currently only Histrogram metrics are supported.
type metricsCollector struct {
	metricsCollectorConfig
	labels map[string]string
}

func newMetricsCollector(config metricsCollectorConfig, labels map[string]string) *metricsCollector {
	return &metricsCollector{
		metricsCollectorConfig: config,
		labels:                 labels,
	}
}

func (*metricsCollector) run(stopCh chan struct{}) {
	// metricCollector doesn't need to start before the tests, so nothing to do here.
}

func (pc *metricsCollector) collect() []DataItem {
	var dataItems []DataItem
	for _, metric := range pc.Metrics {
		dataItem := collectHistogram(metric, pc.labels)
		if dataItem != nil {
			dataItems = append(dataItems, *dataItem)
		}
	}
	return dataItems
}

func collectHistogram(metric string, labels map[string]string) *DataItem {
	hist, err := testutil.GetHistogramFromGatherer(legacyregistry.DefaultGatherer, metric)
	if err != nil {
		klog.Error(err)
		return nil
	}

	if err := hist.Validate(); err != nil {
		klog.Error(err)
		return nil
	}

	q50 := hist.Quantile(0.50)
	q90 := hist.Quantile(0.90)
	q99 := hist.Quantile(0.95)
	avg := hist.Average()

	// clear the metrics so that next test always starts with empty prometheus
	// metrics (since the metrics are shared among all tests run inside the same binary)
	hist.Clear()

	msFactor := float64(time.Second) / float64(time.Millisecond)

	// Copy labels and add "Metric" label for this metric.
	labelMap := map[string]string{"Metric": metric}
	for k, v := range labels {
		labelMap[k] = v
	}
	return &DataItem{
		Labels: labelMap,
		Data: map[string]float64{
			"Perc50":  q50 * msFactor,
			"Perc90":  q90 * msFactor,
			"Perc99":  q99 * msFactor,
			"Average": avg * msFactor,
		},
		Unit: "ms",
	}
}

type throughputCollector struct {
	podInformer           coreinformers.PodInformer
	schedulingThroughputs []float64
	labels                map[string]string
}

func newThroughputCollector(podInformer coreinformers.PodInformer, labels map[string]string) *throughputCollector {
	return &throughputCollector{
		podInformer: podInformer,
		labels:      labels,
	}
}

func (tc *throughputCollector) run(stopCh chan struct{}) {
	podsScheduled, err := getScheduledPods(tc.podInformer)
	if err != nil {
		klog.Fatalf("%v", err)
	}
	lastScheduledCount := len(podsScheduled)
	for {
		select {
		case <-stopCh:
			return
		case <-time.After(throughputSampleFrequency):
			podsScheduled, err := getScheduledPods(tc.podInformer)
			if err != nil {
				klog.Fatalf("%v", err)
			}

			scheduled := len(podsScheduled)
			samplingRatioSeconds := float64(throughputSampleFrequency) / float64(time.Second)
			throughput := float64(scheduled-lastScheduledCount) / samplingRatioSeconds
			tc.schedulingThroughputs = append(tc.schedulingThroughputs, throughput)
			lastScheduledCount = scheduled

			klog.Infof("%d pods scheduled", lastScheduledCount)
		}
	}
}

func (tc *throughputCollector) collect() []DataItem {
	throughputSummary := DataItem{Labels: tc.labels}
	if length := len(tc.schedulingThroughputs); length > 0 {
		sort.Float64s(tc.schedulingThroughputs)
		sum := 0.0
		for i := range tc.schedulingThroughputs {
			sum += tc.schedulingThroughputs[i]
		}

		throughputSummary.Labels["Metric"] = "SchedulingThroughput"
		throughputSummary.Data = map[string]float64{
			"Average": sum / float64(length),
			"Perc50":  tc.schedulingThroughputs[int(math.Ceil(float64(length*50)/100))-1],
			"Perc90":  tc.schedulingThroughputs[int(math.Ceil(float64(length*90)/100))-1],
			"Perc99":  tc.schedulingThroughputs[int(math.Ceil(float64(length*99)/100))-1],
		}
		throughputSummary.Unit = "pods/s"
	}

	return []DataItem{throughputSummary}
}
<|endoftext|>


<commit_before>// +build integrationtest

package integrationtest

import (
	"testing"

	"github.com/portworx/torpedo/drivers/scheduler"
	"github.com/stretchr/testify/require"
)

func testMigration(t *testing.T) {
	t.Run("sanityMigrationTest", sanityMigrationTest)
}

func sanityMigrationTest(t *testing.T) {
	var err error
	// create, apply and validate cluster pair specs
	pairCtxs, err := scheduleClusterPair()
	require.NoError(t, err, "Error scheduling cluster pair")

	// schedule mysql app on cluster 1
	mysqlCtxs, err := schedulerDriver.Schedule("singlemysql",
		scheduler.ScheduleOptions{AppKeys: []string{"mysql-1-pvc"}})
	require.NoError(t, err, "Error scheduling task")
	require.Equal(t, 1, len(mysqlCtxs), "Only one task should have started")

	err = schedulerDriver.WaitForRunning(mysqlCtxs[0], defaultWaitTimeout, defaultWaitInterval)
	require.NoError(t, err, "Error waiting for pod to get to running state")

	// apply migration specs
	migrationCtxs, err := schedulerDriver.Schedule("migration",
		scheduler.ScheduleOptions{AppKeys: []string{"migration"}})
	require.NoError(t, err, "Error scheduling migration specs")

	err = schedulerDriver.WaitForRunning(migrationCtxs[0], defaultWaitTimeout, defaultWaitInterval)
	require.NoError(t, err, "Error waiting for migration to get to Ready state")

	// wait on cluster 2 to get mysql pod running
	err = setRemoteConfig(remoteFilePath)
	require.NoError(t, err, "Error setting remote config")
	err = schedulerDriver.WaitForRunning(mysqlCtxs[0], defaultWaitTimeout, defaultWaitInterval)
	require.NoError(t, err, "Error waiting for pod to get to running state")
	// destroy mysql app on cluster 2
	destroyAndWait(t, mysqlCtxs)

	// destroy mysql app on cluster 1
	err = setRemoteConfig("")
	require.NoError(t, err, "Error setting remote config")
	destroyAndWait(t, mysqlCtxs)

	// destroy CRD objects
	destroyAndWait(t, migrationCtxs)
	destroyAndWait(t, pairCtxs)
}
<commit_msg>Create mysql app before scheduling clusterpair (#211)<commit_after>// +build integrationtest

package integrationtest

import (
	"testing"

	"github.com/portworx/torpedo/drivers/scheduler"
	"github.com/stretchr/testify/require"
)

func testMigration(t *testing.T) {
	t.Run("sanityMigrationTest", sanityMigrationTest)
}

func sanityMigrationTest(t *testing.T) {
	var err error
	// schedule mysql app on cluster 1
	mysqlCtxs, err := schedulerDriver.Schedule("singlemysql",
		scheduler.ScheduleOptions{AppKeys: []string{"mysql-1-pvc"}})
	require.NoError(t, err, "Error scheduling task")
	require.Equal(t, 1, len(mysqlCtxs), "Only one task should have started")

	err = schedulerDriver.WaitForRunning(mysqlCtxs[0], defaultWaitTimeout, defaultWaitInterval)
	require.NoError(t, err, "Error waiting for pod to get to running state")

	// this is so mysql-1-pvc-namespace is created
	// create, apply and validate cluster pair specs
	pairCtxs, err := scheduleClusterPair()
	require.NoError(t, err, "Error scheduling cluster pair")

	// apply migration specs
	migrationCtxs, err := schedulerDriver.Schedule("migration",
		scheduler.ScheduleOptions{AppKeys: []string{"migration"}})
	require.NoError(t, err, "Error scheduling migration specs")

	err = schedulerDriver.WaitForRunning(migrationCtxs[0], defaultWaitTimeout, defaultWaitInterval)
	require.NoError(t, err, "Error waiting for migration to get to Ready state")

	// wait on cluster 2 to get mysql pod running
	err = setRemoteConfig(remoteFilePath)
	require.NoError(t, err, "Error setting remote config")
	err = schedulerDriver.WaitForRunning(mysqlCtxs[0], defaultWaitTimeout, defaultWaitInterval)
	require.NoError(t, err, "Error waiting for pod to get to running state")
	// destroy mysql app on cluster 2
	destroyAndWait(t, mysqlCtxs)

	// destroy mysql app on cluster 1
	err = setRemoteConfig("")
	require.NoError(t, err, "Error setting remote config")
	destroyAndWait(t, mysqlCtxs)

	// destroy CRD objects
	destroyAndWait(t, migrationCtxs)
	destroyAndWait(t, pairCtxs)
}
<|endoftext|>


<commit_before>// Fyndiq API v2 implementation as described in http://developers.fyndiq.com/api-v2/
package gocommerce

import (
	"bytes"
	"encoding/json"
)

type FyndiqV2 struct {
	tr Transport
}

type FyndiqV2Settings struct {
	ProductFeedURL             *string `json:"product_feed_url"`
	ProductFeedNotificationURL *string `json:"product_feed_notification_url"`
	OrderNotificationURL       *string `json:"order_notification_url"`
}

func NewFyndiqV2(tr Transport) *FyndiqV2 {
	return &FyndiqV2{tr}
}

// GetSettings returns the current API v2 settings
func (f *FyndiqV2) GetSettings() (*FyndiqV2Settings, error) {
	var settings FyndiqV2Settings
	var err error
	var url string
	if url, err = f.tr.URL("settings/", map[string]string{}); err != nil {
		return nil, err
	}
	var b []byte
	if b, err = f.tr.Get(url); err != nil {
		return nil, err
	}
	if err = json.Unmarshal(b, settings); err != nil {
		return &settings, err
	}
	return &settings, nil
}

// UpdateSettings updates API v2 settings
func (f *FyndiqV2) UpdateSettings(settings *FyndiqV2Settings) error {
	var url string
	var err error
	if url, err = f.tr.URL("settings/", map[string]string{}); err != nil {
		return err
	}
	var b []byte
	if b, err = json.Marshal(settings); err != nil {
		return err
	}
	return f.tr.Patch(url, bytes.NewReader(b))
}
<commit_msg>More Fyndiq api<commit_after>// Fyndiq API v2 implementation as described in http://developers.fyndiq.com/api-v2/
package gocommerce

import (
	"bytes"
	"encoding/json"
	"strconv"
)

type FyndiqV2 struct {
	tr Transport
}

type FyndiqV2Settings struct {
	ProductFeedURL             *string `json:"product_feed_url"`
	ProductFeedNotificationURL *string `json:"product_feed_notification_url"`
	OrderNotificationURL       *string `json:"order_notification_url"`
}

type paginated struct {
	Next     *string `json:"next"`
	Previous *string `json:"previous"`
}

type FyndiqV2ProductInfoItem struct {
	ProductID string `json:"product_id"`
	ForSale   string `json:"for_sale"`
}

type FyndiqV2ProductInfo struct {
	Results []FyndiqV2ProductInfoItem `json:"results"`
	paginated
}

type FyndiqV2OrderItem struct {
	Sku               string `json:"sku"`
	Quantity          int    `json:"quantity"`
	UnitPriceAmount   string `json:"unit_price_amount"`
	UnitPriceCurrency string `json:"unit_price_currency"`
	VatPercent        string `json:"vat_percent"`
}

type FyndiqV2Order struct {
	ID                  int                 `json:"id"`
	Created             string              `json:"created"`
	DeliveryPhone       string              `json:"delivery_phone"`
	DeliveryCompany     string              `json:"delivery_company"`
	DeliveryFirstname   string              `json:"delivery_firstname"`
	DeliveryLastname    string              `json:"delivery_lastname"`
	DeliveryAddress     string              `json:"delivery_address"`
	DeliveryCo          string              `json:"delivery_co"`
	DeliveryPostalcode  string              `json:"delivery_postalcode"`
	DeliveryCity        string              `json:"delivery_city"`
	DeliveryCountry     string              `json:"delivery_country"`
	DeliveryCountryCode string              `json:"delivery_country_code"`
	OrderRows           []FyndiqV2OrderItem `json:"order_rows"`
	DeliveryNote        string              `json:"delivery_note"`
}

type FyndiqV2Orders struct {
	Results []FyndiqV2Order `json:"results"`
	paginated
}

func NewFyndiqV2(tr Transport) *FyndiqV2 {
	return &FyndiqV2{tr}
}

// GetSettings returns the current API v2 settings
func (f *FyndiqV2) GetSettings() (*FyndiqV2Settings, error) {
	var settings FyndiqV2Settings
	var err error
	var url string
	if url, err = f.tr.URL("settings/", map[string]string{}); err != nil {
		return nil, err
	}
	var b []byte
	if b, err = f.tr.Get(url); err != nil {
		return nil, err
	}
	if err = json.Unmarshal(b, settings); err != nil {
		return &settings, err
	}
	return &settings, nil
}

// UpdateSettings updates API v2 settings
func (f *FyndiqV2) UpdateSettings(settings *FyndiqV2Settings) error {
	var url string
	var err error
	if url, err = f.tr.URL("settings/", map[string]string{}); err != nil {
		return err
	}
	var b []byte
	if b, err = json.Marshal(settings); err != nil {
		return err
	}
	return f.tr.Patch(url, bytes.NewReader(b))
}

// GetProductInfo returns product status information
func (f *FyndiqV2) GetProductInfo(url string) (*FyndiqV2ProductInfo, error) {
	var productInfo FyndiqV2ProductInfo
	var err error
	var b []byte
	if b, err = f.tr.Get(url); err != nil {
		return nil, err
	}
	if err = json.Unmarshal(b, productInfo); err != nil {
		return &productInfo, err
	}
	return &productInfo, nil
}

func (f *FyndiqV2) GetOrder(orderId int) (*FyndiqV2Order, error) {
	var order FyndiqV2Order
	var err error
	var url string
	if url, err = f.tr.URL("order/"+strconv.Itoa(orderId), map[string]string{}); err != nil {
		return nil, err
	}
	var b []byte
	if b, err = f.tr.Get(url); err != nil {
		return nil, err
	}
	if err = json.Unmarshal(b, order); err != nil {
		return &order, err
	}
	return &order, nil
}

func (f *FyndiqV2) GetOrders(url string) (*FyndiqV2Orders, error) {
	var orders FyndiqV2Orders
	var err error
	var b []byte
	if b, err = f.tr.Get(url); err != nil {
		return nil, err
	}
	if err = json.Unmarshal(b, orders); err != nil {
		return &orders, err
	}
	return &orders, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package hpack

import (
	"bytes"
	"errors"
	"io"
	"sync"
)

var bufPool = sync.Pool{
	New: func() interface{} { return new(bytes.Buffer) },
}

// HuffmanDecode decodes the string in v and writes the expanded
// result to w, returning the number of bytes written to w and the
// Write call's return value. At most one Write call is made.
func HuffmanDecode(w io.Writer, v []byte) (int, error) {
	buf := bufPool.Get().(*bytes.Buffer)
	buf.Reset()
	defer bufPool.Put(buf)
	if err := huffmanDecode(buf, 0, v); err != nil {
		return 0, err
	}
	return w.Write(buf.Bytes())
}

// HuffmanDecodeToString decodes the string in v.
func HuffmanDecodeToString(v []byte) (string, error) {
	buf := bufPool.Get().(*bytes.Buffer)
	buf.Reset()
	defer bufPool.Put(buf)
	if err := huffmanDecode(buf, 0, v); err != nil {
		return "", err
	}
	return buf.String(), nil
}

// ErrInvalidHuffman is returned for errors found decoding
// Huffman-encoded strings.
var ErrInvalidHuffman = errors.New("hpack: invalid Huffman-encoded data")

// huffmanDecode decodes v to buf.
// If maxLen is greater than 0, attempts to write more to buf than
// maxLen bytes will return ErrStringLength.
func huffmanDecode(buf *bytes.Buffer, maxLen int, v []byte) error {
	rootHuffmanNode := getRootHuffmanNode()
	n := rootHuffmanNode
	// cur is the bit buffer that has not been fed into n.
	// cbits is the number of low order bits in cur that are valid.
	// sbits is the number of bits of the symbol prefix being decoded.
	cur, cbits, sbits := uint(0), uint8(0), uint8(0)
	for _, b := range v {
		cur = cur<<8 | uint(b)
		cbits += 8
		sbits += 8
		for cbits >= 8 {
			idx := byte(cur >> (cbits - 8))
			n = n.children[idx]
			if n == nil {
				return ErrInvalidHuffman
			}
			if n.children == nil {
				if maxLen != 0 && buf.Len() == maxLen {
					return ErrStringLength
				}
				buf.WriteByte(n.sym)
				cbits -= n.codeLen
				n = rootHuffmanNode
				sbits = cbits
			} else {
				cbits -= 8
			}
		}
	}
	for cbits > 0 {
		n = n.children[byte(cur<<(8-cbits))]
		if n == nil {
			return ErrInvalidHuffman
		}
		if n.children != nil || n.codeLen > cbits {
			break
		}
		if maxLen != 0 && buf.Len() == maxLen {
			return ErrStringLength
		}
		buf.WriteByte(n.sym)
		cbits -= n.codeLen
		n = rootHuffmanNode
		sbits = cbits
	}
	if sbits > 7 {
		// Either there was an incomplete symbol, or overlong padding.
		// Both are decoding errors per RFC 7541 section 5.2.
		return ErrInvalidHuffman
	}
	if mask := uint(1<<cbits - 1); cur&mask != mask {
		// Trailing bits must be a prefix of EOS per RFC 7541 section 5.2.
		return ErrInvalidHuffman
	}

	return nil
}

// incomparable is a zero-width, non-comparable type. Adding it to a struct
// makes that struct also non-comparable, and generally doesn't add
// any size (as long as it's first).
type incomparable [0]func()

type node struct {
	_ incomparable

	// children is non-nil for internal nodes
	children *[256]*node

	// The following are only valid if children is nil:
	codeLen uint8 // number of bits that led to the output of sym
	sym     byte  // output symbol
}

func newInternalNode() *node {
	return &node{children: new([256]*node)}
}

var (
	buildRootOnce       sync.Once
	lazyRootHuffmanNode *node
)

func getRootHuffmanNode() *node {
	buildRootOnce.Do(buildRootHuffmanNode)
	return lazyRootHuffmanNode
}

func buildRootHuffmanNode() {
	if len(huffmanCodes) != 256 {
		panic("unexpected size")
	}
	lazyRootHuffmanNode = newInternalNode()
	for i, code := range huffmanCodes {
		addDecoderNode(byte(i), code, huffmanCodeLen[i])
	}
}

func addDecoderNode(sym byte, code uint32, codeLen uint8) {
	cur := lazyRootHuffmanNode
	for codeLen > 8 {
		codeLen -= 8
		i := uint8(code >> codeLen)
		if cur.children[i] == nil {
			cur.children[i] = newInternalNode()
		}
		cur = cur.children[i]
	}
	shift := 8 - codeLen
	start, end := int(uint8(code<<shift)), int(1<<shift)
	for i := start; i < start+end; i++ {
		cur.children[i] = &node{sym: sym, codeLen: codeLen}
	}
}

// AppendHuffmanString appends s, as encoded in Huffman codes, to dst
// and returns the extended buffer.
func AppendHuffmanString(dst []byte, s string) []byte {
	rembits := uint8(8)

	for i := 0; i < len(s); i++ {
		if rembits == 8 {
			dst = append(dst, 0)
		}
		dst, rembits = appendByteToHuffmanCode(dst, rembits, s[i])
	}

	if rembits < 8 {
		// special EOS symbol
		code := uint32(0x3fffffff)
		nbits := uint8(30)

		t := uint8(code >> (nbits - rembits))
		dst[len(dst)-1] |= t
	}

	return dst
}

// HuffmanEncodeLength returns the number of bytes required to encode
// s in Huffman codes. The result is round up to byte boundary.
func HuffmanEncodeLength(s string) uint64 {
	n := uint64(0)
	for i := 0; i < len(s); i++ {
		n += uint64(huffmanCodeLen[s[i]])
	}
	return (n + 7) / 8
}

// appendByteToHuffmanCode appends Huffman code for c to dst and
// returns the extended buffer and the remaining bits in the last
// element. The appending is not byte aligned and the remaining bits
// in the last element of dst is given in rembits.
func appendByteToHuffmanCode(dst []byte, rembits uint8, c byte) ([]byte, uint8) {
	code := huffmanCodes[c]
	nbits := huffmanCodeLen[c]

	for {
		if rembits > nbits {
			t := uint8(code << (rembits - nbits))
			dst[len(dst)-1] |= t
			rembits -= nbits
			break
		}

		t := uint8(code >> (nbits - rembits))
		dst[len(dst)-1] |= t

		nbits -= rembits
		rembits = 8

		if nbits == 0 {
			break
		}

		dst = append(dst, 0)
	}

	return dst, rembits
}
<commit_msg>http2: number of allocs reduced from 3852 to 31<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package hpack

import (
	"bytes"
	"errors"
	"io"
	"sync"
)

var bufPool = sync.Pool{
	New: func() interface{} { return new(bytes.Buffer) },
}

// HuffmanDecode decodes the string in v and writes the expanded
// result to w, returning the number of bytes written to w and the
// Write call's return value. At most one Write call is made.
func HuffmanDecode(w io.Writer, v []byte) (int, error) {
	buf := bufPool.Get().(*bytes.Buffer)
	buf.Reset()
	defer bufPool.Put(buf)
	if err := huffmanDecode(buf, 0, v); err != nil {
		return 0, err
	}
	return w.Write(buf.Bytes())
}

// HuffmanDecodeToString decodes the string in v.
func HuffmanDecodeToString(v []byte) (string, error) {
	buf := bufPool.Get().(*bytes.Buffer)
	buf.Reset()
	defer bufPool.Put(buf)
	if err := huffmanDecode(buf, 0, v); err != nil {
		return "", err
	}
	return buf.String(), nil
}

// ErrInvalidHuffman is returned for errors found decoding
// Huffman-encoded strings.
var ErrInvalidHuffman = errors.New("hpack: invalid Huffman-encoded data")

// huffmanDecode decodes v to buf.
// If maxLen is greater than 0, attempts to write more to buf than
// maxLen bytes will return ErrStringLength.
func huffmanDecode(buf *bytes.Buffer, maxLen int, v []byte) error {
	rootHuffmanNode := getRootHuffmanNode()
	n := rootHuffmanNode
	// cur is the bit buffer that has not been fed into n.
	// cbits is the number of low order bits in cur that are valid.
	// sbits is the number of bits of the symbol prefix being decoded.
	cur, cbits, sbits := uint(0), uint8(0), uint8(0)
	for _, b := range v {
		cur = cur<<8 | uint(b)
		cbits += 8
		sbits += 8
		for cbits >= 8 {
			idx := byte(cur >> (cbits - 8))
			n = n.children[idx]
			if n == nil {
				return ErrInvalidHuffman
			}
			if n.children == nil {
				if maxLen != 0 && buf.Len() == maxLen {
					return ErrStringLength
				}
				buf.WriteByte(n.sym)
				cbits -= n.codeLen
				n = rootHuffmanNode
				sbits = cbits
			} else {
				cbits -= 8
			}
		}
	}
	for cbits > 0 {
		n = n.children[byte(cur<<(8-cbits))]
		if n == nil {
			return ErrInvalidHuffman
		}
		if n.children != nil || n.codeLen > cbits {
			break
		}
		if maxLen != 0 && buf.Len() == maxLen {
			return ErrStringLength
		}
		buf.WriteByte(n.sym)
		cbits -= n.codeLen
		n = rootHuffmanNode
		sbits = cbits
	}
	if sbits > 7 {
		// Either there was an incomplete symbol, or overlong padding.
		// Both are decoding errors per RFC 7541 section 5.2.
		return ErrInvalidHuffman
	}
	if mask := uint(1<<cbits - 1); cur&mask != mask {
		// Trailing bits must be a prefix of EOS per RFC 7541 section 5.2.
		return ErrInvalidHuffman
	}

	return nil
}

// incomparable is a zero-width, non-comparable type. Adding it to a struct
// makes that struct also non-comparable, and generally doesn't add
// any size (as long as it's first).
type incomparable [0]func()

type node struct {
	_ incomparable

	// children is non-nil for internal nodes
	children *[256]*node

	// The following are only valid if children is nil:
	codeLen uint8 // number of bits that led to the output of sym
	sym     byte  // output symbol
}

func newInternalNode() *node {
	return &node{children: new([256]*node)}
}

var (
	buildRootOnce       sync.Once
	lazyRootHuffmanNode *node
)

func getRootHuffmanNode() *node {
	buildRootOnce.Do(buildRootHuffmanNode)
	return lazyRootHuffmanNode
}

func buildRootHuffmanNode() {
	if len(huffmanCodes) != 256 {
		panic("unexpected size")
	}
	lazyRootHuffmanNode = newInternalNode()
	// allocate a leaf node for each of the 256 symbols
	leaves := new([256]node)

	for sym, code := range huffmanCodes {
		codeLen := huffmanCodeLen[sym]

		cur := lazyRootHuffmanNode
		for codeLen > 8 {
			codeLen -= 8
			i := uint8(code >> codeLen)
			if cur.children[i] == nil {
				cur.children[i] = newInternalNode()
			}
			cur = cur.children[i]
		}
		shift := 8 - codeLen
		start, end := int(uint8(code<<shift)), int(1<<shift)

		leaves[sym].sym = byte(sym)
		leaves[sym].codeLen = codeLen
		for i := start; i < start+end; i++ {
			cur.children[i] = &leaves[sym]
		}
	}
}

// AppendHuffmanString appends s, as encoded in Huffman codes, to dst
// and returns the extended buffer.
func AppendHuffmanString(dst []byte, s string) []byte {
	rembits := uint8(8)

	for i := 0; i < len(s); i++ {
		if rembits == 8 {
			dst = append(dst, 0)
		}
		dst, rembits = appendByteToHuffmanCode(dst, rembits, s[i])
	}

	if rembits < 8 {
		// special EOS symbol
		code := uint32(0x3fffffff)
		nbits := uint8(30)

		t := uint8(code >> (nbits - rembits))
		dst[len(dst)-1] |= t
	}

	return dst
}

// HuffmanEncodeLength returns the number of bytes required to encode
// s in Huffman codes. The result is round up to byte boundary.
func HuffmanEncodeLength(s string) uint64 {
	n := uint64(0)
	for i := 0; i < len(s); i++ {
		n += uint64(huffmanCodeLen[s[i]])
	}
	return (n + 7) / 8
}

// appendByteToHuffmanCode appends Huffman code for c to dst and
// returns the extended buffer and the remaining bits in the last
// element. The appending is not byte aligned and the remaining bits
// in the last element of dst is given in rembits.
func appendByteToHuffmanCode(dst []byte, rembits uint8, c byte) ([]byte, uint8) {
	code := huffmanCodes[c]
	nbits := huffmanCodeLen[c]

	for {
		if rembits > nbits {
			t := uint8(code << (rembits - nbits))
			dst[len(dst)-1] |= t
			rembits -= nbits
			break
		}

		t := uint8(code >> (nbits - rembits))
		dst[len(dst)-1] |= t

		nbits -= rembits
		rembits = 8

		if nbits == 0 {
			break
		}

		dst = append(dst, 0)
	}

	return dst, rembits
}
<|endoftext|>


<commit_before>package worker

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"testing"
	"time"

	gocontext "context"

	"github.com/stretchr/testify/assert"
)

func TestHTTPJobQueue(t *testing.T) {
	hjq, err := NewHTTPJobQueue(nil, "test", "fake", "fake", nil)
	assert.Nil(t, err)
	assert.NotNil(t, hjq)
}

func TestHTTPJobQueue_Jobs(t *testing.T) {
	mux := http.NewServeMux()
	mux.HandleFunc(`/jobs`, func(w http.ResponseWriter, req *http.Request) {
		fmt.Fprintf(w, `{"jobs":["100001"]}`)
	})
	mux.HandleFunc(`/jobs/100001`, func(w http.ResponseWriter, req *http.Request) {
		w.WriteHeader(http.StatusOK)
		fmt.Fprintf(w, strings.Replace(`{
			"data": {
				"type": "job",
				"job": {
					"id": 100001,
					"number": "42.1",
					"queued_at": "2011-04-01T11:05:55Z"
				},
				"source": {
					"id": 100001,
					"number": "42"
				},
				"repository": {
					"id": 8490324,
					"slug": "travis-ci/nonexistent-repository"
				},
				"uuid": "fafafaf",
				"config": {},
				"vm_type": "test",
				"meta": {
					"state_update_count": 0
				}
			}
		}`, "\t", "  ", -1))
	})

	mux.HandleFunc(`/`, func(w http.ResponseWriter, req *http.Request) {
		t.Fatalf("unknown URL requested: %#v", req.URL.Path)
	})
	jobBoardServer := httptest.NewServer(mux)
	defer jobBoardServer.Close()

	jobBoardURL, _ := url.Parse(jobBoardServer.URL)
	hjq, err := NewHTTPJobQueue(jobBoardURL, "test", "fake", "fake", nil)
	assert.Nil(t, err)
	assert.NotNil(t, hjq)

	ctx := gocontext.TODO()
	buildJobChan, err := hjq.Jobs(ctx)
	assert.Nil(t, err)
	assert.NotNil(t, buildJobChan)

	select {
	case job := <-buildJobChan:
		assert.NotNil(t, job)
	case <-time.After(time.Second):
		t.Fatalf("failed to recv job")
	}
}

func TestHTTPJobQueue_Name(t *testing.T) {
	hjq, err := NewHTTPJobQueue(nil, "test", "fake", "fake", nil)
	assert.Nil(t, err)
	assert.Equal(t, "http", hjq.Name())
}

func TestHTTPJobQueue_Cleanup(t *testing.T) {
	hjq, err := NewHTTPJobQueue(nil, "test", "fake", "fake", nil)
	assert.Nil(t, err)
	assert.Nil(t, hjq.Cleanup())
}
<commit_msg>Adjust http job queue test per changed job-board API<commit_after>package worker

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"testing"
	"time"

	gocontext "context"

	"github.com/stretchr/testify/assert"
)

func TestHTTPJobQueue(t *testing.T) {
	hjq, err := NewHTTPJobQueue(nil, "test", "fake", "fake", nil)
	assert.Nil(t, err)
	assert.NotNil(t, hjq)
}

func TestHTTPJobQueue_Jobs(t *testing.T) {
	mux := http.NewServeMux()
	mux.HandleFunc(`/jobs/pop`, func(w http.ResponseWriter, req *http.Request) {
		w.WriteHeader(http.StatusOK)
		fmt.Fprintf(w, `{"job_id":"100001"}`)
	})
	mux.HandleFunc(`/jobs/100001/claim`, func(w http.ResponseWriter, req *http.Request) {
		w.WriteHeader(http.StatusOK)
	})
	mux.HandleFunc(`/jobs/100001`, func(w http.ResponseWriter, req *http.Request) {
		w.WriteHeader(http.StatusOK)
		fmt.Fprintf(w, strings.Replace(`{
			"data": {
				"type": "job",
				"job": {
					"id": 100001,
					"number": "42.1",
					"queued_at": "2011-04-01T11:05:55Z"
				},
				"source": {
					"id": 100001,
					"number": "42"
				},
				"repository": {
					"id": 8490324,
					"slug": "travis-ci/nonexistent-repository"
				},
				"uuid": "fafafaf",
				"config": {},
				"vm_type": "test",
				"meta": {
					"state_update_count": 0
				}
			}
		}`, "\t", "  ", -1))
	})

	mux.HandleFunc(`/`, func(w http.ResponseWriter, req *http.Request) {
		t.Fatalf("unknown URL requested: %#v", req.URL.Path)
	})
	jobBoardServer := httptest.NewServer(mux)
	defer jobBoardServer.Close()

	jobBoardURL, _ := url.Parse(jobBoardServer.URL)
	hjq, err := NewHTTPJobQueue(jobBoardURL, "test", "fake", "fake", nil)
	assert.Nil(t, err)
	assert.NotNil(t, hjq)

	ctx := gocontext.TODO()
	buildJobChan, err := hjq.Jobs(ctx)
	assert.Nil(t, err)
	assert.NotNil(t, buildJobChan)

	select {
	case job := <-buildJobChan:
		assert.NotNil(t, job)
	case <-time.After(time.Second):
		t.Fatalf("failed to recv job")
	}
}

func TestHTTPJobQueue_Name(t *testing.T) {
	hjq, err := NewHTTPJobQueue(nil, "test", "fake", "fake", nil)
	assert.Nil(t, err)
	assert.Equal(t, "http", hjq.Name())
}

func TestHTTPJobQueue_Cleanup(t *testing.T) {
	hjq, err := NewHTTPJobQueue(nil, "test", "fake", "fake", nil)
	assert.Nil(t, err)
	assert.Nil(t, hjq.Cleanup())
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux netbsd openbsd dragonfly nacl

package os

import (
	"errors"
	"runtime"
)

// We query the executable path at init time to avoid the problem of
// readlink returns a path appended with " (deleted)" when the original
// binary gets deleted.
var executablePath, executablePathErr = func() (string, error) {
	var procfn string
	switch runtime.GOOS {
	default:
		return "", errors.New("Executable not implemented for " + runtime.GOOS)
	case "linux":
		procfn = "/proc/self/exe"
	case "netbsd":
		procfn = "/proc/curproc/exe"
	case "openbsd":
		procfn = "/proc/curproc/file"
	case "dragonfly":
		procfn = "/proc/curproc/file"
	}
	return Readlink(procfn)
}()

func executable() (string, error) {
	return executablePath, executablePathErr
}
<commit_msg>os: Executable can use /proc/self/exe on android<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux netbsd openbsd dragonfly nacl

package os

import (
	"errors"
	"runtime"
)

// We query the executable path at init time to avoid the problem of
// readlink returns a path appended with " (deleted)" when the original
// binary gets deleted.
var executablePath, executablePathErr = func() (string, error) {
	var procfn string
	switch runtime.GOOS {
	default:
		return "", errors.New("Executable not implemented for " + runtime.GOOS)
	case "linux", "android":
		procfn = "/proc/self/exe"
	case "netbsd":
		procfn = "/proc/curproc/exe"
	case "openbsd":
		procfn = "/proc/curproc/file"
	case "dragonfly":
		procfn = "/proc/curproc/file"
	}
	return Readlink(procfn)
}()

func executable() (string, error) {
	return executablePath, executablePathErr
}
<|endoftext|>


<commit_before>package kutil

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/golang/glog"
	"k8s.io/kube-deploy/upup/pkg/fi"
	"k8s.io/kube-deploy/upup/pkg/fi/cloudup"
	"k8s.io/kube-deploy/upup/pkg/fi/cloudup/awsup"
)

// UpgradeCluster performs an upgrade of a k8s cluster
type UpgradeCluster struct {
	OldClusterName string
	NewClusterName string
	Cloud          fi.Cloud

	StateStore fi.StateStore

	Config *cloudup.CloudConfig
}

func (x *UpgradeCluster) Upgrade() error {
	awsCloud := x.Cloud.(*awsup.AWSCloud)

	config := x.Config

	newClusterName := x.NewClusterName
	if newClusterName == "" {
		return fmt.Errorf("NewClusterName must be specified")
	}
	oldClusterName := x.OldClusterName
	if oldClusterName == "" {
		return fmt.Errorf("OldClusterName must be specified")
	}

	newTags := awsCloud.Tags()
	newTags["KubernetesCluster"] = newClusterName

	// Try to pre-query as much as possible before doing anything destructive
	instances, err := findInstances(awsCloud)
	if err != nil {
		return fmt.Errorf("error finding instances: %v", err)
	}

	volumes, err := DescribeVolumes(x.Cloud)
	if err != nil {
		return err
	}

	dhcpOptions, err := DescribeDhcpOptions(x.Cloud)
	if err != nil {
		return nil, err
	}

	// Find masters
	var masters []*ec2.Instance
	for _, instance := range instances {
		role, _ := awsup.FindEC2Tag(instance.Tags, "Role")
		if role == oldClusterName+"-master" {
			masters = append(masters, instance)
		}
	}
	if len(masters) == 0 {
		return fmt.Errorf("could not find masters")
	}

	// Stop masters
	for _, master := range masters {
		masterInstanceID := aws.StringValue(master.InstanceId)
		glog.Infof("Stopping master: %q", masterInstanceID)

		request := &ec2.StopInstancesInput{
			InstanceIds: []*string{master.InstanceId},
		}

		_, err := awsCloud.EC2.StopInstances(request)
		if err != nil {
			return fmt.Errorf("error stopping master instance: %v", err)
		}
	}

	// TODO: Wait for master to stop & detach volumes?

	//subnets, err := DescribeSubnets(x.Cloud)
	//if err != nil {
	//	return fmt.Errorf("error finding subnets: %v", err)
	//}
	//for _, s := range subnets {
	//	id := aws.StringValue(s.SubnetId)
	//	err := awsCloud.AddAWSTags(id, newTags)
	//	if err != nil {
	//		return fmt.Errorf("error re-tagging subnet %q: %v", id, err)
	//	}
	//}

	// Retag VPC
	// We have to be careful because VPCs can be shared
	{
		vpcID := config.NetworkID
		if vpcID != "" {
			tags, err := awsCloud.GetTags(vpcID)
			if err != nil {
				return fmt.Errorf("error getting VPC tags: %v", err)
			}

			clusterTag := tags[awsup.TagClusterName]
			if clusterTag != "" {
				if clusterTag != oldClusterName {
					return fmt.Errorf("VPC is tagged with a different cluster: %v", clusterTag)
				}
				replaceTags := make(map[string]string)
				replaceTags[awsup.TagClusterName] = newClusterName
				err := awsCloud.CreateTags(vpcID, replaceTags)
				if err != nil {
					return fmt.Errorf("error re-tagging VPC: %v", err)
				}
			}

		}
	}

	// Retag DHCP options
	// We have to be careful because DHCP options can be shared
	for _, dhcpOption := range dhcpOptions {
		clusterTag := dhcpOption.Tags[awsup.TagClusterName]
		if clusterTag != "" {
			if clusterTag != oldClusterName {
				return fmt.Errorf("DHCP options are tagged with a different cluster: %v", clusterTag)
			}
			replaceTags := make(map[string]string)
			replaceTags[awsup.TagClusterName] = newClusterName
			err := awsCloud.CreateTags(*dhcpOption.DhcpOptionsId, replaceTags)
			if err != nil {
				return fmt.Errorf("error re-tagging DHCP options: %v", err)
			}
		}

	}

	// TODO: Retag internet gateway (may not be tagged at all though...)
	// TODO: Share more code with cluste deletion?

	// TODO: Adopt LoadBalancers & LoadBalancer Security Groups

	// Adopt Volumes
	for _, volume := range volumes {
		id := aws.StringValue(volume.VolumeId)

		// TODO: Batch re-tag?
		replaceTags := make(map[string]string)
		replaceTags[awsup.TagClusterName] = newClusterName

		name, _ := awsup.FindEC2Tag(volume.Tags, "Name")
		if name == oldClusterName+"-master-pd" {
			glog.Infof("Found master volume %q: %s", id, name)
			replaceTags["Name"] = "kubernetes.master." + aws.StringValue(volume.AvailabilityZone) + "." + newClusterName
		}
		err := awsCloud.CreateTags(id, replaceTags)
		if err != nil {
			return fmt.Errorf("error re-tagging volume %q: %v", id, err)
		}
	}

	config.ClusterName = newClusterName
	err = x.StateStore.WriteConfig(config)
	if err != nil {
		return fmt.Errorf("error writing updated configuration: %v", err)
	}

	return nil
}
<commit_msg>Fix upgrade<commit_after>package kutil

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/golang/glog"
	"k8s.io/kube-deploy/upup/pkg/fi"
	"k8s.io/kube-deploy/upup/pkg/fi/cloudup"
	"k8s.io/kube-deploy/upup/pkg/fi/cloudup/awsup"
)

// UpgradeCluster performs an upgrade of a k8s cluster
type UpgradeCluster struct {
	OldClusterName string
	NewClusterName string
	Cloud          fi.Cloud

	StateStore fi.StateStore

	Config *cloudup.CloudConfig
}

func (x *UpgradeCluster) Upgrade() error {
	awsCloud := x.Cloud.(*awsup.AWSCloud)

	config := x.Config

	newClusterName := x.NewClusterName
	if newClusterName == "" {
		return fmt.Errorf("NewClusterName must be specified")
	}
	oldClusterName := x.OldClusterName
	if oldClusterName == "" {
		return fmt.Errorf("OldClusterName must be specified")
	}

	newTags := awsCloud.Tags()
	newTags["KubernetesCluster"] = newClusterName

	// Try to pre-query as much as possible before doing anything destructive
	instances, err := findInstances(awsCloud)
	if err != nil {
		return fmt.Errorf("error finding instances: %v", err)
	}

	volumes, err := DescribeVolumes(x.Cloud)
	if err != nil {
		return err
	}

	dhcpOptions, err := DescribeDhcpOptions(x.Cloud)
	if err != nil {
		return err
	}

	// Find masters
	var masters []*ec2.Instance
	for _, instance := range instances {
		role, _ := awsup.FindEC2Tag(instance.Tags, "Role")
		if role == oldClusterName+"-master" {
			masters = append(masters, instance)
		}
	}
	if len(masters) == 0 {
		return fmt.Errorf("could not find masters")
	}

	// Stop masters
	for _, master := range masters {
		masterInstanceID := aws.StringValue(master.InstanceId)
		glog.Infof("Stopping master: %q", masterInstanceID)

		request := &ec2.StopInstancesInput{
			InstanceIds: []*string{master.InstanceId},
		}

		_, err := awsCloud.EC2.StopInstances(request)
		if err != nil {
			return fmt.Errorf("error stopping master instance: %v", err)
		}
	}

	// TODO: Wait for master to stop & detach volumes?

	//subnets, err := DescribeSubnets(x.Cloud)
	//if err != nil {
	//	return fmt.Errorf("error finding subnets: %v", err)
	//}
	//for _, s := range subnets {
	//	id := aws.StringValue(s.SubnetId)
	//	err := awsCloud.AddAWSTags(id, newTags)
	//	if err != nil {
	//		return fmt.Errorf("error re-tagging subnet %q: %v", id, err)
	//	}
	//}

	// Retag VPC
	// We have to be careful because VPCs can be shared
	{
		vpcID := config.NetworkID
		if vpcID != "" {
			tags, err := awsCloud.GetTags(vpcID)
			if err != nil {
				return fmt.Errorf("error getting VPC tags: %v", err)
			}

			clusterTag := tags[awsup.TagClusterName]
			if clusterTag != "" {
				if clusterTag != oldClusterName {
					return fmt.Errorf("VPC is tagged with a different cluster: %v", clusterTag)
				}
				replaceTags := make(map[string]string)
				replaceTags[awsup.TagClusterName] = newClusterName
				err := awsCloud.CreateTags(vpcID, replaceTags)
				if err != nil {
					return fmt.Errorf("error re-tagging VPC: %v", err)
				}
			}

		}
	}

	// Retag DHCP options
	// We have to be careful because DHCP options can be shared
	for _, dhcpOption := range dhcpOptions {
		clusterTag, _ := awsup.FindEC2Tag(dhcpOption.Tags, awsup.TagClusterName)
		if clusterTag != "" {
			if clusterTag != oldClusterName {
				return fmt.Errorf("DHCP options are tagged with a different cluster: %v", clusterTag)
			}
			replaceTags := make(map[string]string)
			replaceTags[awsup.TagClusterName] = newClusterName
			err := awsCloud.CreateTags(*dhcpOption.DhcpOptionsId, replaceTags)
			if err != nil {
				return fmt.Errorf("error re-tagging DHCP options: %v", err)
			}
		}

	}

	// TODO: Retag internet gateway (may not be tagged at all though...)
	// TODO: Share more code with cluste deletion?

	// TODO: Adopt LoadBalancers & LoadBalancer Security Groups

	// Adopt Volumes
	for _, volume := range volumes {
		id := aws.StringValue(volume.VolumeId)

		// TODO: Batch re-tag?
		replaceTags := make(map[string]string)
		replaceTags[awsup.TagClusterName] = newClusterName

		name, _ := awsup.FindEC2Tag(volume.Tags, "Name")
		if name == oldClusterName+"-master-pd" {
			glog.Infof("Found master volume %q: %s", id, name)
			replaceTags["Name"] = "kubernetes.master." + aws.StringValue(volume.AvailabilityZone) + "." + newClusterName
		}
		err := awsCloud.CreateTags(id, replaceTags)
		if err != nil {
			return fmt.Errorf("error re-tagging volume %q: %v", id, err)
		}
	}

	config.ClusterName = newClusterName
	err = x.StateStore.WriteConfig(config)
	if err != nil {
		return fmt.Errorf("error writing updated configuration: %v", err)
	}

	return nil
}
<|endoftext|>