<commit_before>package common

import (
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/defaults"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/hashicorp/packer/template/interpolate"
)

// AccessConfig is for common configuration related to AWS access
type AccessConfig struct {
	AccessKey         string `mapstructure:"access_key"`
	AssumeRoleArn     string `mapstructure:"assume_role_arn"`
	CustomEndpointEc2 string `mapstructure:"custom_endpoint_ec2"`
	ExternalID        string `mapstructure:"external_id"`
	MFACode           string `mapstructure:"mfa_code"`
	MFASerial         string `mapstructure:"mfa_serial"`
	ProfileName       string `mapstructure:"profile"`
	RawRegion         string `mapstructure:"region"`
	SecretKey         string `mapstructure:"secret_key"`
	SkipValidation    bool   `mapstructure:"skip_region_validation"`
	Token             string `mapstructure:"token"`
}

// Config returns a valid aws.Config object for access to AWS services, or
// an error if the authentication and region couldn't be resolved
func (c *AccessConfig) Config() (*aws.Config, error) {
	var creds *credentials.Credentials

	region, err := c.Region()
	if err != nil {
		return nil, err
	}

	config := aws.NewConfig().WithRegion(region).WithMaxRetries(11)
	if c.CustomEndpointEc2 != "" {
		config.Endpoint = &c.CustomEndpointEc2
	}
	creds = credentials.NewChainCredentials(
		[]credentials.Provider{
			&credentials.StaticProvider{
				Value: credentials.Value{
					AccessKeyID:     c.AccessKey,
					SecretAccessKey: c.SecretKey,
					SessionToken:    c.Token,
				},
			},
			&credentials.EnvProvider{},
			&credentials.SharedCredentialsProvider{
				Profile: c.ProfileName,
			},
			defaults.RemoteCredProvider(*(defaults.Config()), defaults.Handlers()),
		})

	if c.AssumeRoleArn != "" {
		var mfa func(*stscreds.AssumeRoleProvider)
		if c.MFACode != "" {
			mfa = func(p *stscreds.AssumeRoleProvider) {
				p.SerialNumber = aws.String(c.MFASerial)
				p.TokenProvider = func() (string, error) {
					return c.MFACode, nil
				}
			}
		}

		sess := session.Must(session.NewSession(config.WithCredentials(creds)))
		creds = stscreds.NewCredentials(sess, c.AssumeRoleArn, mfa, func(p *stscreds.AssumeRoleProvider) {
			p.Duration = time.Duration(60) * time.Minute
			if len(c.ExternalID) > 0 {
				p.ExternalID = aws.String(c.ExternalID)
			}
		})

	}
	return config.WithCredentials(creds), nil
}

// Region returns the aws.Region object for access to AWS services, requesting
// the region from the instance metadata if possible.
func (c *AccessConfig) Region() (string, error) {
	if c.RawRegion != "" {
		if !c.SkipValidation {
			if valid := ValidateRegion(c.RawRegion); !valid {
				return "", fmt.Errorf("Not a valid region: %s", c.RawRegion)
			}
		}
		return c.RawRegion, nil
	}

	sess := session.New()
	ec2meta := ec2metadata.New(sess)
	identity, err := ec2meta.GetInstanceIdentityDocument()
	if err != nil {
		return "", err
	}
	return identity.Region, nil
}

func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error {
	var errs []error
	if c.RawRegion != "" && !c.SkipValidation {
		if valid := ValidateRegion(c.RawRegion); !valid {
			errs = append(errs, fmt.Errorf("Unknown region: %s", c.RawRegion))
		}
	}

	hasAssumeRoleArn := len(c.AssumeRoleArn) > 0
	hasMFASerial := len(c.MFASerial) > 0
	hasMFACode := len(c.MFACode) > 0
	if hasAssumeRoleArn && ((hasMFACode && !hasMFASerial) || (!hasMFACode && !hasMFACode)) {
		errs = append(errs, fmt.Errorf("Both mfa_serial and mfa_code must be specified."))

	}

	if len(errs) > 0 {
		return errs
	}

	return nil
}
<commit_msg>fix go vet issue<commit_after>package common

import (
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/defaults"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/hashicorp/packer/template/interpolate"
)

// AccessConfig is for common configuration related to AWS access
type AccessConfig struct {
	AccessKey         string `mapstructure:"access_key"`
	AssumeRoleArn     string `mapstructure:"assume_role_arn"`
	CustomEndpointEc2 string `mapstructure:"custom_endpoint_ec2"`
	ExternalID        string `mapstructure:"external_id"`
	MFACode           string `mapstructure:"mfa_code"`
	MFASerial         string `mapstructure:"mfa_serial"`
	ProfileName       string `mapstructure:"profile"`
	RawRegion         string `mapstructure:"region"`
	SecretKey         string `mapstructure:"secret_key"`
	SkipValidation    bool   `mapstructure:"skip_region_validation"`
	Token             string `mapstructure:"token"`
}

// Config returns a valid aws.Config object for access to AWS services, or
// an error if the authentication and region couldn't be resolved
func (c *AccessConfig) Config() (*aws.Config, error) {
	var creds *credentials.Credentials

	region, err := c.Region()
	if err != nil {
		return nil, err
	}

	config := aws.NewConfig().WithRegion(region).WithMaxRetries(11)
	if c.CustomEndpointEc2 != "" {
		config.Endpoint = &c.CustomEndpointEc2
	}
	creds = credentials.NewChainCredentials(
		[]credentials.Provider{
			&credentials.StaticProvider{
				Value: credentials.Value{
					AccessKeyID:     c.AccessKey,
					SecretAccessKey: c.SecretKey,
					SessionToken:    c.Token,
				},
			},
			&credentials.EnvProvider{},
			&credentials.SharedCredentialsProvider{
				Profile: c.ProfileName,
			},
			defaults.RemoteCredProvider(*(defaults.Config()), defaults.Handlers()),
		})

	if c.AssumeRoleArn != "" {
		var mfa func(*stscreds.AssumeRoleProvider)
		if c.MFACode != "" {
			mfa = func(p *stscreds.AssumeRoleProvider) {
				p.SerialNumber = aws.String(c.MFASerial)
				p.TokenProvider = func() (string, error) {
					return c.MFACode, nil
				}
			}
		}

		sess := session.Must(session.NewSession(config.WithCredentials(creds)))
		creds = stscreds.NewCredentials(sess, c.AssumeRoleArn, mfa, func(p *stscreds.AssumeRoleProvider) {
			p.Duration = time.Duration(60) * time.Minute
			if len(c.ExternalID) > 0 {
				p.ExternalID = aws.String(c.ExternalID)
			}
		})

	}
	return config.WithCredentials(creds), nil
}

// Region returns the aws.Region object for access to AWS services, requesting
// the region from the instance metadata if possible.
func (c *AccessConfig) Region() (string, error) {
	if c.RawRegion != "" {
		if !c.SkipValidation {
			if valid := ValidateRegion(c.RawRegion); !valid {
				return "", fmt.Errorf("Not a valid region: %s", c.RawRegion)
			}
		}
		return c.RawRegion, nil
	}

	sess := session.New()
	ec2meta := ec2metadata.New(sess)
	identity, err := ec2meta.GetInstanceIdentityDocument()
	if err != nil {
		return "", err
	}
	return identity.Region, nil
}

func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error {
	var errs []error
	if c.RawRegion != "" && !c.SkipValidation {
		if valid := ValidateRegion(c.RawRegion); !valid {
			errs = append(errs, fmt.Errorf("Unknown region: %s", c.RawRegion))
		}
	}

	hasAssumeRoleArn := len(c.AssumeRoleArn) > 0
	hasMFASerial := len(c.MFASerial) > 0
	hasMFACode := len(c.MFACode) > 0
	if hasAssumeRoleArn && (!hasMFACode || !hasMFASerial) {
		errs = append(errs, fmt.Errorf("Both mfa_serial and mfa_code must be specified."))

	}

	if len(errs) > 0 {
		return errs
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"reflect"
	"testing"

	pb "./genproto"
)

func Test_convert(t *testing.T) {
	type args struct {
		m    pb.MoneyAmount
		rate float64
	}
	tests := []struct {
		name string
		args args
		want *pb.MoneyAmount
	}{
		{
			"0.33*3", args{pb.MoneyAmount{Decimal: 0, Fractional: 330}, 3}, pb.MoneyAmount{Decimal: 0, Fractional: 99},
		},
		{
			"10.00*0.5", args{pb.MoneyAmount{Decimal: 10}, 0.5}, pb.MoneyAmount{Decimal: 5},
		},
		{
			"10.00*1.5", args{pb.MoneyAmount{Decimal: 10}, 1.5}, pb.MoneyAmount{Decimal: 15},
		},
		{
			"32.320*0.5 (trailing zero removed)", args{pb.MoneyAmount{Decimal: 32, Fractional: 32}, 0.5}, pb.MoneyAmount{Decimal: 16, Fractional: 16},
		},
		{
			"33.33*(1/3) (trailing zero removed)", args{pb.MoneyAmount{Decimal: 33, Fractional: 33}, 1.0 / 3}, pb.MoneyAmount{Decimal: 11, Fractional: 11},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := convert(tt.args.m, tt.args.rate); !reflect.DeepEqual(got, tt.want) {
				t.Errorf("convert([%v]*%f) = %v, want=[%v]", tt.args.m, tt.args.rate, got, tt.want)
			}
		})
	}
}
<commit_msg>currencyservice: fix test, add test for fractional result<commit_after>package main

import (
	"reflect"
	"testing"

	pb "./genproto"
)

func Test_convert(t *testing.T) {
	type args struct {
		m    pb.MoneyAmount
		rate float64
	}
	tests := []struct {
		name string
		args args
		want pb.MoneyAmount
	}{
		{
			"0.33*3", args{pb.MoneyAmount{Decimal: 0, Fractional: 330}, 3}, pb.MoneyAmount{Decimal: 0, Fractional: 99},
		},
		{
			"10.00*0.5", args{pb.MoneyAmount{Decimal: 10}, 0.5}, pb.MoneyAmount{Decimal: 5},
		},
		{
			"10.00*1.5", args{pb.MoneyAmount{Decimal: 10}, 1.5}, pb.MoneyAmount{Decimal: 15},
		},
		{
			"10.00*1/3", args{pb.MoneyAmount{Decimal: 10}, 1.0 / 3}, pb.MoneyAmount{Decimal: 3, Fractional: 3},
		},
		{
			"32.320*0.5 (trailing zero removed)", args{pb.MoneyAmount{Decimal: 32, Fractional: 32}, 0.5}, pb.MoneyAmount{Decimal: 16, Fractional: 16},
		},
		{
			"33.33*(1/3) (trailing zero removed)", args{pb.MoneyAmount{Decimal: 33, Fractional: 33}, 1.0 / 3}, pb.MoneyAmount{Decimal: 11, Fractional: 11},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := convert(tt.args.m, tt.args.rate); !reflect.DeepEqual(got, tt.want) {
				t.Errorf("convert([%v]*%f) = %v, want=[%v]", tt.args.m, tt.args.rate, got, tt.want)
			}
		})
	}
}
<|endoftext|>


<commit_before>package actions

import (
	"testing"
	"time"

	"github.com/gaia-adm/pumba/container"
	"github.com/gaia-adm/pumba/container/mockclient"
	"github.com/samalba/dockerclient"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
)

func TestPattern_Filter(t *testing.T) {
	c1 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "c1",
			Config: &dockerclient.ContainerConfig{},
		},
		nil,
	)
	c2 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "c2",
			Config: &dockerclient.ContainerConfig{},
		},
		nil,
	)
	cc := &dockerclient.ContainerConfig{
		Labels: map[string]string{"com.gaiaadm.pumba": "true"},
	}
	c3 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "c1",
			Config: cc,
		},
		nil,
	)
	cf := regexContainerFilter("*")
	assert.True(t, cf(c1))
	assert.True(t, cf(c2))
	assert.False(t, cf(c3))
}

func TestStopByPattern_All(t *testing.T) {
	c1 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name: "c1",
		},
		nil,
	)
	c2 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name: "c2",
		},
		nil,
	)
	cs := []container.Container{c1, c2}

	client := &mockclient.MockClient{}
	client.On("ListContainers", mock.AnythingOfType("container.Filter")).Return(cs, nil)
	client.On("StopContainer", c1, time.Duration(10)).Return(nil)
	client.On("StopContainer", c2, time.Duration(10)).Return(nil)

	err := StopByPattern(client, "*")

	assert.NoError(t, err)
	client.AssertExpectations(t)
}
<commit_msg>test for regex filter<commit_after>package actions

import (
	"testing"
	"time"

	"github.com/gaia-adm/pumba/container"
	"github.com/gaia-adm/pumba/container/mockclient"
	"github.com/samalba/dockerclient"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
)

func TestPattern_StarFilter(t *testing.T) {
	c1 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "c1",
			Config: &dockerclient.ContainerConfig{},
		},
		nil,
	)
	c2 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "c2",
			Config: &dockerclient.ContainerConfig{},
		},
		nil,
	)
	cc := &dockerclient.ContainerConfig{
		Labels: map[string]string{"com.gaiaadm.pumba": "true"},
	}
	c3 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "c1",
			Config: cc,
		},
		nil,
	)
	cf := regexContainerFilter("*")
	assert.True(t, cf(c1))
	assert.True(t, cf(c2))
	assert.False(t, cf(c3))
}

func TestPattern_NameFilter(t *testing.T) {
	c1 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "AbcEFG",
			Config: &dockerclient.ContainerConfig{},
		},
		nil,
	)
	c2 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "AbcHKL",
			Config: &dockerclient.ContainerConfig{},
		},
		nil,
	)
	cc := &dockerclient.ContainerConfig{
		Labels: map[string]string{"com.gaiaadm.pumba": "true"},
	}
	c3 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "AbcPumba",
			Config: cc,
		},
		nil,
	)
	cf := regexContainerFilter("^Abc")
	assert.True(t, cf(c1))
	assert.True(t, cf(c2))
	assert.False(t, cf(c3))
}

func TestStopByPattern_All(t *testing.T) {
	c1 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name: "c1",
		},
		nil,
	)
	c2 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name: "c2",
		},
		nil,
	)
	cs := []container.Container{c1, c2}

	client := &mockclient.MockClient{}
	client.On("ListContainers", mock.AnythingOfType("container.Filter")).Return(cs, nil)
	client.On("StopContainer", c1, time.Duration(10)).Return(nil)
	client.On("StopContainer", c2, time.Duration(10)).Return(nil)

	err := StopByPattern(client, "*")

	assert.NoError(t, err)
	client.AssertExpectations(t)
}
<|endoftext|>


<commit_before>package edn

import (
	. "testing"
)

func parse(s string, t *T) (val Value) {
	val, err := ParseString(s)

	if err != nil {
		t.Errorf("Expected parsing %+v to succeed. %v", s, err)
	}

	return
}

func assertValueEqual(actual, expected Value, t *T) {
	if !expected.Equals(actual) {
		t.Errorf("Expected %v, got %v", expected, actual)
	}
}

func TestDoesNotParseEmptyInput(t *T) {
	_, err := ParseString("")

	if err == nil {
		t.Error("Expected parsing empty input to fail")
	}
}

func TestParseList(t *T) {
	val := parse("()", t)

	if !new(List).Equals(val) {
		t.Errorf("Expected parsing \"()\" to return an empty list, got %+v", val)
	}

	str := `(() "abc" [] "def")`
	val = parse(str, t)

	l := new(List)
	ll := l.raw()
	ll.PushBack(new(List))
	ll.PushBack(String("abc"))
	ll.PushBack(Vector{})
	ll.PushBack(String("def"))

	if !l.Equals(val) {
		t.Errorf("Expected %v, got %v", l, val)
	}
}

func TestParseVector(t *T) {
	assertValueEqual(parse(`[]`, t), Vector{}, t)
	assertValueEqual(parse(`[[]]`, t), Vector{Vector{}}, t)
	l := new(List)
	l.raw().PushBack(String("abc"))
	assertValueEqual(parse(`[[("abc")] "def"]`, t), Vector{Vector{l}, String("def")}, t)
}

func TestParseString(t *T) {
	val := parse(`""`, t)
	if val == nil || !val.Equals(String("")) {
		t.Errorf("Expected \"\", got %v", val)
	}

	val = parse(`"abc"`, t)
	if val == nil || !val.Equals(String("abc")) {
		t.Errorf("Expected \"abc\", got %v", val)
	}
}
<commit_msg>Add test for parsing non-trivial EDN<commit_after>package edn

import (
	. "testing"
)

func parse(s string, t *T) (val Value) {
	val, err := ParseString(s)

	if err != nil {
		t.Errorf("Expected parsing %+v to succeed. %v", s, err)
		t.FailNow()
	}

	return
}

func assertValueEqual(actual, expected Value, t *T) {
	if !expected.Equals(actual) {
		t.Errorf("Expected %+v, got %+v", expected, actual)
	}
}

func TestDoesNotParseEmptyInput(t *T) {
	_, err := ParseString("")

	if err == nil {
		t.Error("Expected parsing empty input to fail")
	}
}

func TestParseList(t *T) {
	val := parse("()", t)

	if !new(List).Equals(val) {
		t.Errorf("Expected parsing \"()\" to return an empty list, got %+v", val)
	}

	str := `(() "abc" [] "def")`
	val = parse(str, t)

	l := new(List)
	ll := l.raw()
	ll.PushBack(new(List))
	ll.PushBack(String("abc"))
	ll.PushBack(Vector{})
	ll.PushBack(String("def"))

	if !l.Equals(val) {
		t.Errorf("Expected %v, got %v", l, val)
	}
}

func TestParseVector(t *T) {
	assertValueEqual(parse(`[]`, t), Vector{}, t)
	assertValueEqual(parse(`[[]]`, t), Vector{Vector{}}, t)
	l := new(List)
	l.raw().PushBack(String("abc"))
	assertValueEqual(parse(`[[("abc")] "def"]`, t), Vector{Vector{l}, String("def")}, t)
}

func TestParseString(t *T) {
	val := parse(`""`, t)
	if val == nil || !val.Equals(String("")) {
		t.Errorf("Expected \"\", got %v", val)
	}

	val = parse(`"abc"`, t)
	if val == nil || !val.Equals(String("abc")) {
		t.Errorf("Expected \"abc\", got %v", val)
	}
}

func TestParse(t *T) {
	expected := new(List).Insert(
		String("abc"),
		new(List).Insert(String("spaced")),
		Vector{
			String("vec"),
			new(List).Insert(
				String("an"),
				String("inner"),
				String("list"),
			),
		},
		new(List),
		Vector{},
		new(List),
		String(""),
		Vector{},
		Map{},
		Set{}.Insert(String("set")),
	)

	actual := parse(`
	("abc" ( "spaced" )
	    ["vec"( "an"	"inner""list",)]
		(),[]()""[]
		{}#{"set"}
	)
	`, t)

	assertValueEqual(actual, expected, t)
}
<|endoftext|>


<commit_before>package types

import (
	"bytes"
	"errors"
	"math"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/go-graphite/carbonapi/expr/consolidations"
	"github.com/go-graphite/carbonapi/expr/tags"
	pbv2 "github.com/go-graphite/protocol/carbonapi_v2_pb"
	pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
	pickle "github.com/lomik/og-rek"
)

var (
	// ErrWildcardNotAllowed is an eval error returned when a wildcard/glob argument is found where a single series is required.
	ErrWildcardNotAllowed = errors.New("found wildcard where series expected")
	// ErrTooManyArguments is an eval error returned when too many arguments are provided.
	ErrTooManyArguments = errors.New("too many arguments")
)

// MetricData contains necessary data to represent parsed metric (ready to be send out or drawn)
type MetricData struct {
	pb.FetchResponse

	GraphOptions

	ValuesPerPoint    int
	aggregatedValues  []float64
	Tags              map[string]string
	AggregateFunction func([]float64) float64 `json:"-"`
}

// MarshalCSV marshals metric data to CSV
func MarshalCSV(results []*MetricData) []byte {

	var b []byte

	for _, r := range results {

		step := r.StepTime
		t := r.StartTime
		for _, v := range r.Values {
			b = append(b, "\""+r.Name+"\","+time.Unix(t, 0).Format("2006-01-02 15:04:05")+","...)
			if !math.IsNaN(v) {
				b = strconv.AppendFloat(b, v, 'f', -1, 64)
			}
			b = append(b, '\n')
			t += step
		}
	}
	return b
}

// ConsolidateJSON consolidates values to maxDataPoints size
func ConsolidateJSON(maxDataPoints int64, results []*MetricData) {
	if len(results) == 0 {
		return
	}
	startTime := results[0].StartTime
	endTime := results[0].StopTime
	for _, r := range results {
		t := r.StartTime
		if startTime > t {
			startTime = t
		}
		t = r.StopTime
		if endTime < t {
			endTime = t
		}
	}

	timeRange := endTime - startTime

	if timeRange <= 0 {
		return
	}

	for _, r := range results {
		numberOfDataPoints := math.Floor(float64(timeRange) / float64(r.StepTime))
		if numberOfDataPoints > float64(maxDataPoints) {
			valuesPerPoint := math.Ceil(numberOfDataPoints / float64(maxDataPoints))
			r.SetValuesPerPoint(int(valuesPerPoint))
		}
	}
}

// MarshalJSON marshals metric data to JSON
func MarshalJSON(results []*MetricData, timestampMultiplier int64, noNullPoints bool) []byte {
	var b []byte
	b = append(b, '[')

	var topComma bool
	for _, r := range results {
		if r == nil {
			continue
		}

		if topComma {
			b = append(b, ',')
		}
		topComma = true

		b = append(b, `{"target":`...)
		b = strconv.AppendQuoteToASCII(b, r.Name)
		b = append(b, `,"datapoints":[`...)

		var innerComma bool
		t := r.StartTime * timestampMultiplier
		for _, v := range r.AggregatedValues() {
			if noNullPoints && math.IsNaN(v) {
				t += r.AggregatedTimeStep() * timestampMultiplier
			} else {
				if innerComma {
					b = append(b, ',')
				}
				innerComma = true

				b = append(b, '[')

				if math.IsNaN(v) || math.IsInf(v, 1) || math.IsInf(v, -1) {
					b = append(b, "null"...)
				} else {
					b = strconv.AppendFloat(b, v, 'f', -1, 64)
				}

				b = append(b, ',')

				b = strconv.AppendInt(b, t, 10)

				b = append(b, ']')

				t += r.AggregatedTimeStep() * timestampMultiplier
			}
		}

		b = append(b, `],"tags":{`...)
		notFirstTag := false
		responseTags := make([]string, 0, len(r.Tags))
		for tag := range r.Tags {
			responseTags = append(responseTags, tag)
		}
		sort.Strings(responseTags)
		for _, tag := range responseTags {
			v := r.Tags[tag]
			if notFirstTag {
				b = append(b, ',')
			}
			b = strconv.AppendQuoteToASCII(b, tag)
			b = append(b, ':')
			b = strconv.AppendQuoteToASCII(b, v)
			notFirstTag = true
		}

		b = append(b, `}}`...)
	}

	b = append(b, ']')

	return b
}

// MarshalPickle marshals metric data to pickle format
func MarshalPickle(results []*MetricData) []byte {

	var p []map[string]interface{}

	for _, r := range results {
		values := make([]interface{}, len(r.Values))
		for i, v := range r.Values {
			if math.IsNaN(v) {
				values[i] = pickle.None{}
			} else {
				values[i] = v
			}

		}
		p = append(p, map[string]interface{}{
			"name":              r.Name,
			"pathExpression":    r.PathExpression,
			"consolidationFunc": r.ConsolidationFunc,
			"start":             r.StartTime,
			"end":               r.StopTime,
			"step":              r.StepTime,
			"xFilesFactor":      r.XFilesFactor,
			"values":            values,
		})
	}

	var buf bytes.Buffer

	penc := pickle.NewEncoder(&buf)
	_ = penc.Encode(p)

	return buf.Bytes()
}

// MarshalProtobufV3 marshals metric data to protobuf
func MarshalProtobufV2(results []*MetricData) ([]byte, error) {
	response := pbv2.MultiFetchResponse{}
	for _, metric := range results {
		fmv3 := (*metric).FetchResponse
		v := make([]float64, len(fmv3.Values))
		isAbsent := make([]bool, len(fmv3.Values))
		for i := range fmv3.Values {
			if math.IsNaN(fmv3.Values[i]) {
				v[i] = 0
				isAbsent[i] = true
			} else {
				v[i] = fmv3.Values[i]
			}
		}
		fm := pbv2.FetchResponse{
			Name:      fmv3.Name,
			StartTime: int32(fmv3.StartTime),
			StopTime:  int32(fmv3.StopTime),
			StepTime:  int32(fmv3.StepTime),
			Values:    v,
			IsAbsent:  isAbsent,
		}
		response.Metrics = append(response.Metrics, fm)
	}
	b, err := response.Marshal()
	if err != nil {
		return nil, err
	}

	return b, nil
}

// MarshalProtobufV3 marshals metric data to protobuf
func MarshalProtobufV3(results []*MetricData) ([]byte, error) {
	response := pb.MultiFetchResponse{}
	for _, metric := range results {
		response.Metrics = append(response.Metrics, (*metric).FetchResponse)
	}
	b, err := response.Marshal()
	if err != nil {
		return nil, err
	}

	return b, nil
}

// MarshalRaw marshals metric data to graphite's internal format, called 'raw'
func MarshalRaw(results []*MetricData) []byte {

	var b []byte

	for _, r := range results {

		b = append(b, r.Name...)

		b = append(b, ',')
		b = strconv.AppendInt(b, r.StartTime, 10)
		b = append(b, ',')
		b = strconv.AppendInt(b, r.StopTime, 10)
		b = append(b, ',')
		b = strconv.AppendInt(b, r.StepTime, 10)
		b = append(b, '|')

		var comma bool
		for _, v := range r.Values {
			if comma {
				b = append(b, ',')
			}
			comma = true
			if math.IsNaN(v) {
				b = append(b, "None"...)
			} else {
				b = strconv.AppendFloat(b, v, 'f', -1, 64)
			}
		}

		b = append(b, '\n')
	}
	return b
}

// SetValuesPerPoint sets value per point coefficient.
func (r *MetricData) SetValuesPerPoint(v int) {
	r.ValuesPerPoint = v
	r.aggregatedValues = nil
}

// AggregatedTimeStep aggregates time step
func (r *MetricData) AggregatedTimeStep() int64 {
	if r.ValuesPerPoint == 1 || r.ValuesPerPoint == 0 {
		return r.StepTime
	}

	return r.StepTime * int64(r.ValuesPerPoint)
}

// GetAggregateFunction returns MetricData.AggregateFunction and set it, if it's not yet
func (r *MetricData) GetAggregateFunction() func([]float64) float64 {
	if r.AggregateFunction == nil {
		var ok bool
		if r.AggregateFunction, ok = consolidations.ConsolidationToFunc[strings.ToLower(r.ConsolidationFunc)]; !ok {
			// if consolidation function is not known, we should fall back to average
			r.AggregateFunction = consolidations.AvgValue
		}
	}

	return r.AggregateFunction
}

// AggregatedValues aggregates values (with cache)
func (r *MetricData) AggregatedValues() []float64 {
	if r.aggregatedValues == nil {
		r.AggregateValues()
	}
	return r.aggregatedValues
}

// AggregateValues aggregates values
func (r *MetricData) AggregateValues() {
	if r.ValuesPerPoint == 1 || r.ValuesPerPoint == 0 {
		r.aggregatedValues = make([]float64, len(r.Values))
		copy(r.aggregatedValues, r.Values)
		return
	}
	aggFunc := r.GetAggregateFunction()

	n := len(r.Values)/r.ValuesPerPoint + 1
	aggV := make([]float64, 0, n)

	v := r.Values

	for len(v) >= r.ValuesPerPoint {
		val := aggFunc(v[:r.ValuesPerPoint])
		aggV = append(aggV, val)
		v = v[r.ValuesPerPoint:]
	}

	if len(v) > 0 {
		val := aggFunc(v)
		aggV = append(aggV, val)
	}

	r.aggregatedValues = aggV
}

// MakeMetricData creates new metrics data with given metric timeseries
func MakeMetricData(name string, values []float64, step, start int64) *MetricData {
	return makeMetricDataWithTags(name, values, step, start, tags.ExtractTags(name))
}

// MakeMetricDataWithTags creates new metrics data with given metric Time Series (with tags)
func makeMetricDataWithTags(name string, values []float64, step, start int64, tags map[string]string) *MetricData {
	stop := start + int64(len(values))*step

	return &MetricData{FetchResponse: pb.FetchResponse{
		Name:      name,
		Values:    values,
		StartTime: start,
		StepTime:  step,
		StopTime:  stop,
	},
		Tags: tags,
	}
}
<commit_msg>Implement MetricData.Copy function<commit_after>package types

import (
	"bytes"
	"errors"
	"math"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/go-graphite/carbonapi/expr/consolidations"
	"github.com/go-graphite/carbonapi/expr/tags"
	pbv2 "github.com/go-graphite/protocol/carbonapi_v2_pb"
	pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
	pickle "github.com/lomik/og-rek"
)

var (
	// ErrWildcardNotAllowed is an eval error returned when a wildcard/glob argument is found where a single series is required.
	ErrWildcardNotAllowed = errors.New("found wildcard where series expected")
	// ErrTooManyArguments is an eval error returned when too many arguments are provided.
	ErrTooManyArguments = errors.New("too many arguments")
)

// MetricData contains necessary data to represent parsed metric (ready to be send out or drawn)
type MetricData struct {
	pb.FetchResponse

	GraphOptions

	ValuesPerPoint    int
	aggregatedValues  []float64
	Tags              map[string]string
	AggregateFunction func([]float64) float64 `json:"-"`
}

// MarshalCSV marshals metric data to CSV
func MarshalCSV(results []*MetricData) []byte {

	var b []byte

	for _, r := range results {

		step := r.StepTime
		t := r.StartTime
		for _, v := range r.Values {
			b = append(b, "\""+r.Name+"\","+time.Unix(t, 0).Format("2006-01-02 15:04:05")+","...)
			if !math.IsNaN(v) {
				b = strconv.AppendFloat(b, v, 'f', -1, 64)
			}
			b = append(b, '\n')
			t += step
		}
	}
	return b
}

// ConsolidateJSON consolidates values to maxDataPoints size
func ConsolidateJSON(maxDataPoints int64, results []*MetricData) {
	if len(results) == 0 {
		return
	}
	startTime := results[0].StartTime
	endTime := results[0].StopTime
	for _, r := range results {
		t := r.StartTime
		if startTime > t {
			startTime = t
		}
		t = r.StopTime
		if endTime < t {
			endTime = t
		}
	}

	timeRange := endTime - startTime

	if timeRange <= 0 {
		return
	}

	for _, r := range results {
		numberOfDataPoints := math.Floor(float64(timeRange) / float64(r.StepTime))
		if numberOfDataPoints > float64(maxDataPoints) {
			valuesPerPoint := math.Ceil(numberOfDataPoints / float64(maxDataPoints))
			r.SetValuesPerPoint(int(valuesPerPoint))
		}
	}
}

// MarshalJSON marshals metric data to JSON
func MarshalJSON(results []*MetricData, timestampMultiplier int64, noNullPoints bool) []byte {
	var b []byte
	b = append(b, '[')

	var topComma bool
	for _, r := range results {
		if r == nil {
			continue
		}

		if topComma {
			b = append(b, ',')
		}
		topComma = true

		b = append(b, `{"target":`...)
		b = strconv.AppendQuoteToASCII(b, r.Name)
		b = append(b, `,"datapoints":[`...)

		var innerComma bool
		t := r.StartTime * timestampMultiplier
		for _, v := range r.AggregatedValues() {
			if noNullPoints && math.IsNaN(v) {
				t += r.AggregatedTimeStep() * timestampMultiplier
			} else {
				if innerComma {
					b = append(b, ',')
				}
				innerComma = true

				b = append(b, '[')

				if math.IsNaN(v) || math.IsInf(v, 1) || math.IsInf(v, -1) {
					b = append(b, "null"...)
				} else {
					b = strconv.AppendFloat(b, v, 'f', -1, 64)
				}

				b = append(b, ',')

				b = strconv.AppendInt(b, t, 10)

				b = append(b, ']')

				t += r.AggregatedTimeStep() * timestampMultiplier
			}
		}

		b = append(b, `],"tags":{`...)
		notFirstTag := false
		responseTags := make([]string, 0, len(r.Tags))
		for tag := range r.Tags {
			responseTags = append(responseTags, tag)
		}
		sort.Strings(responseTags)
		for _, tag := range responseTags {
			v := r.Tags[tag]
			if notFirstTag {
				b = append(b, ',')
			}
			b = strconv.AppendQuoteToASCII(b, tag)
			b = append(b, ':')
			b = strconv.AppendQuoteToASCII(b, v)
			notFirstTag = true
		}

		b = append(b, `}}`...)
	}

	b = append(b, ']')

	return b
}

// MarshalPickle marshals metric data to pickle format
func MarshalPickle(results []*MetricData) []byte {

	var p []map[string]interface{}

	for _, r := range results {
		values := make([]interface{}, len(r.Values))
		for i, v := range r.Values {
			if math.IsNaN(v) {
				values[i] = pickle.None{}
			} else {
				values[i] = v
			}

		}
		p = append(p, map[string]interface{}{
			"name":              r.Name,
			"pathExpression":    r.PathExpression,
			"consolidationFunc": r.ConsolidationFunc,
			"start":             r.StartTime,
			"end":               r.StopTime,
			"step":              r.StepTime,
			"xFilesFactor":      r.XFilesFactor,
			"values":            values,
		})
	}

	var buf bytes.Buffer

	penc := pickle.NewEncoder(&buf)
	_ = penc.Encode(p)

	return buf.Bytes()
}

// MarshalProtobufV3 marshals metric data to protobuf
func MarshalProtobufV2(results []*MetricData) ([]byte, error) {
	response := pbv2.MultiFetchResponse{}
	for _, metric := range results {
		fmv3 := (*metric).FetchResponse
		v := make([]float64, len(fmv3.Values))
		isAbsent := make([]bool, len(fmv3.Values))
		for i := range fmv3.Values {
			if math.IsNaN(fmv3.Values[i]) {
				v[i] = 0
				isAbsent[i] = true
			} else {
				v[i] = fmv3.Values[i]
			}
		}
		fm := pbv2.FetchResponse{
			Name:      fmv3.Name,
			StartTime: int32(fmv3.StartTime),
			StopTime:  int32(fmv3.StopTime),
			StepTime:  int32(fmv3.StepTime),
			Values:    v,
			IsAbsent:  isAbsent,
		}
		response.Metrics = append(response.Metrics, fm)
	}
	b, err := response.Marshal()
	if err != nil {
		return nil, err
	}

	return b, nil
}

// MarshalProtobufV3 marshals metric data to protobuf
func MarshalProtobufV3(results []*MetricData) ([]byte, error) {
	response := pb.MultiFetchResponse{}
	for _, metric := range results {
		response.Metrics = append(response.Metrics, (*metric).FetchResponse)
	}
	b, err := response.Marshal()
	if err != nil {
		return nil, err
	}

	return b, nil
}

// MarshalRaw marshals metric data to graphite's internal format, called 'raw'
func MarshalRaw(results []*MetricData) []byte {

	var b []byte

	for _, r := range results {

		b = append(b, r.Name...)

		b = append(b, ',')
		b = strconv.AppendInt(b, r.StartTime, 10)
		b = append(b, ',')
		b = strconv.AppendInt(b, r.StopTime, 10)
		b = append(b, ',')
		b = strconv.AppendInt(b, r.StepTime, 10)
		b = append(b, '|')

		var comma bool
		for _, v := range r.Values {
			if comma {
				b = append(b, ',')
			}
			comma = true
			if math.IsNaN(v) {
				b = append(b, "None"...)
			} else {
				b = strconv.AppendFloat(b, v, 'f', -1, 64)
			}
		}

		b = append(b, '\n')
	}
	return b
}

// SetValuesPerPoint sets value per point coefficient.
func (r *MetricData) SetValuesPerPoint(v int) {
	r.ValuesPerPoint = v
	r.aggregatedValues = nil
}

// AggregatedTimeStep aggregates time step
func (r *MetricData) AggregatedTimeStep() int64 {
	if r.ValuesPerPoint == 1 || r.ValuesPerPoint == 0 {
		return r.StepTime
	}

	return r.StepTime * int64(r.ValuesPerPoint)
}

// GetAggregateFunction returns MetricData.AggregateFunction and set it, if it's not yet
func (r *MetricData) GetAggregateFunction() func([]float64) float64 {
	if r.AggregateFunction == nil {
		var ok bool
		if r.AggregateFunction, ok = consolidations.ConsolidationToFunc[strings.ToLower(r.ConsolidationFunc)]; !ok {
			// if consolidation function is not known, we should fall back to average
			r.AggregateFunction = consolidations.AvgValue
		}
	}

	return r.AggregateFunction
}

// AggregatedValues aggregates values (with cache)
func (r *MetricData) AggregatedValues() []float64 {
	if r.aggregatedValues == nil {
		r.AggregateValues()
	}
	return r.aggregatedValues
}

// AggregateValues aggregates values
func (r *MetricData) AggregateValues() {
	if r.ValuesPerPoint == 1 || r.ValuesPerPoint == 0 {
		r.aggregatedValues = make([]float64, len(r.Values))
		copy(r.aggregatedValues, r.Values)
		return
	}
	aggFunc := r.GetAggregateFunction()

	n := len(r.Values)/r.ValuesPerPoint + 1
	aggV := make([]float64, 0, n)

	v := r.Values

	for len(v) >= r.ValuesPerPoint {
		val := aggFunc(v[:r.ValuesPerPoint])
		aggV = append(aggV, val)
		v = v[r.ValuesPerPoint:]
	}

	if len(v) > 0 {
		val := aggFunc(v)
		aggV = append(aggV, val)
	}

	r.aggregatedValues = aggV
}

// Copy returns the copy of r. If includeValues set to true, it copies values as well.
func (r *MetricData) Copy(includeValues bool) *MetricData {
	var values, aggregatedValues []float64
	values = make([]float64, 0)
	aggregatedValues = nil

	if includeValues {
		values = make([]float64, len(r.Values))
		copy(values, r.Values)
		if r.aggregatedValues != nil {
			aggregatedValues = make([]float64, len(r.aggregatedValues))
			copy(aggregatedValues, r.aggregatedValues)
		}
	}

	return &MetricData{
		FetchResponse: pb.FetchResponse{
			Name:                    r.Name,
			PathExpression:          r.PathExpression,
			ConsolidationFunc:       r.ConsolidationFunc,
			StartTime:               r.StartTime,
			StopTime:                r.StopTime,
			StepTime:                r.StepTime,
			XFilesFactor:            r.XFilesFactor,
			HighPrecisionTimestamps: r.HighPrecisionTimestamps,
			Values:                  values,
			AppliedFunctions:        r.AppliedFunctions,
			RequestStartTime:        r.RequestStartTime,
			RequestStopTime:         r.RequestStopTime,
		},
		GraphOptions:      r.GraphOptions,
		ValuesPerPoint:    r.ValuesPerPoint,
		aggregatedValues:  aggregatedValues,
		Tags:              r.Tags,
		AggregateFunction: r.AggregateFunction,
	}
}

// MakeMetricData creates new metrics data with given metric timeseries
func MakeMetricData(name string, values []float64, step, start int64) *MetricData {
	return makeMetricDataWithTags(name, values, step, start, tags.ExtractTags(name))
}

// MakeMetricDataWithTags creates new metrics data with given metric Time Series (with tags)
func makeMetricDataWithTags(name string, values []float64, step, start int64, tags map[string]string) *MetricData {
	stop := start + int64(len(values))*step

	return &MetricData{
		FetchResponse: pb.FetchResponse{
			Name:      name,
			Values:    values,
			StartTime: start,
			StepTime:  step,
			StopTime:  stop,
		},
		Tags: tags,
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sources

import (
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/pkg/errors"
	v1 "k8s.io/api/core/v1"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/color"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
)

const (
	initContainer = "kaniko-init-container"
)

// LocalDir refers to kaniko using a local directory as a buildcontext
// skaffold copies the buildcontext into the local directory via kubectl cp
type LocalDir struct {
	cfg     *latest.KanikoBuild
	tarPath string
}

// Setup for LocalDir creates a tarball of the buildcontext and stores it in /tmp
func (g *LocalDir) Setup(ctx context.Context, out io.Writer, artifact *latest.Artifact, initialTag string) (string, error) {
	g.tarPath = filepath.Join(os.TempDir(), fmt.Sprintf("context-%s.tar.gz", initialTag))
	color.Default.Fprintln(out, "Storing build context at", g.tarPath)

	f, err := os.Create(g.tarPath)
	if err != nil {
		return "", errors.Wrap(err, "creating temporary buildcontext tarball")
	}
	defer f.Close()

	err = docker.CreateDockerTarGzContext(ctx, f, artifact.Workspace, artifact.DockerArtifact)

	context := fmt.Sprintf("dir://%s", constants.DefaultKanikoEmptyDirMountPath)
	return context, err
}

// Pod returns the pod template to ModifyPod
func (g *LocalDir) Pod(args []string) *v1.Pod {
	p := podTemplate(g.cfg, args)
	// Include the emptyDir volume and volume source in both containers
	v := v1.Volume{
		Name: constants.DefaultKanikoEmptyDirName,
		VolumeSource: v1.VolumeSource{
			EmptyDir: &v1.EmptyDirVolumeSource{},
		},
	}
	vm := v1.VolumeMount{
		Name:      constants.DefaultKanikoEmptyDirName,
		MountPath: constants.DefaultKanikoEmptyDirMountPath,
	}
	// Generate the init container, which will run until the /tmp/complete file is created
	ic := v1.Container{
		Name:  initContainer,
		Image: constants.DefaultAlpineImage,
		Args: []string{"sh", "-c", `while true; do
	sleep 1; if [ -f /tmp/complete ]; then break; fi
done`},
		VolumeMounts: []v1.VolumeMount{vm},
	}

	p.Spec.InitContainers = []v1.Container{ic}
	p.Spec.Containers[0].VolumeMounts = append(p.Spec.Containers[0].VolumeMounts, vm)
	p.Spec.Volumes = append(p.Spec.Volumes, v)
	return p
}

// ModifyPod first copies over the buildcontext tarball into the init container tmp dir via kubectl cp
// Via kubectl exec, we extract the tarball to the empty dir
// Then, via kubectl exec, create the /tmp/complete file via kubectl exec to complete the init container
func (g *LocalDir) ModifyPod(ctx context.Context, p *v1.Pod) error {
	client, err := kubernetes.GetClientset()
	if err != nil {
		return errors.Wrap(err, "getting clientset")
	}
	if err := kubernetes.WaitForPodInitialized(ctx, client.CoreV1().Pods(p.Namespace), p.Name); err != nil {
		return errors.Wrap(err, "waiting for pod to initialize")
	}
	// Copy over the buildcontext tarball into the init container
	tarCopyPath := fmt.Sprintf("/tmp/%s", filepath.Base(g.tarPath))
	copy := exec.CommandContext(ctx, "kubectl", "cp", g.tarPath, fmt.Sprintf("%s:%s", p.Name, tarCopyPath), "-c", initContainer, "-n", p.Namespace)
	if err := util.RunCmd(copy); err != nil {
		return errors.Wrap(err, "copying buildcontext into init container")
	}
	// Next, extract the buildcontext to the empty dir
	extract := exec.CommandContext(ctx, "kubectl", "exec", p.Name, "-c", initContainer, "-n", p.Namespace, "--", "tar", "-xzf", tarCopyPath, "-C", constants.DefaultKanikoEmptyDirMountPath)
	if err := util.RunCmd(extract); err != nil {
		return errors.Wrap(err, "extracting buildcontext to empty dir")
	}
	// Generate a file to successfully terminate the init container
	file := exec.CommandContext(ctx, "kubectl", "exec", p.Name, "-c", initContainer, "-n", p.Namespace, "--", "touch", "/tmp/complete")
	return util.RunCmd(file)
}

// Cleanup deletes the buidcontext tarball stored on the local filesystem
func (g *LocalDir) Cleanup(ctx context.Context) error {
	return os.Remove(g.tarPath)
}
<commit_msg>Fixed localDir build context copy on Windows (#1438)<commit_after>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sources

import (
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/pkg/errors"
	v1 "k8s.io/api/core/v1"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/color"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
)

const (
	initContainer = "kaniko-init-container"
)

// LocalDir refers to kaniko using a local directory as a buildcontext
// skaffold copies the buildcontext into the local directory via kubectl cp
type LocalDir struct {
	cfg     *latest.KanikoBuild
	tarPath string
}

// Setup for LocalDir creates a tarball of the buildcontext and stores it in /tmp
func (g *LocalDir) Setup(ctx context.Context, out io.Writer, artifact *latest.Artifact, initialTag string) (string, error) {
	g.tarPath = filepath.Join(os.TempDir(), fmt.Sprintf("context-%s.tar.gz", initialTag))
	color.Default.Fprintln(out, "Storing build context at", g.tarPath)

	f, err := os.Create(g.tarPath)
	if err != nil {
		return "", errors.Wrap(err, "creating temporary buildcontext tarball")
	}
	defer f.Close()

	err = docker.CreateDockerTarGzContext(ctx, f, artifact.Workspace, artifact.DockerArtifact)

	context := fmt.Sprintf("dir://%s", constants.DefaultKanikoEmptyDirMountPath)
	return context, err
}

// Pod returns the pod template to ModifyPod
func (g *LocalDir) Pod(args []string) *v1.Pod {
	p := podTemplate(g.cfg, args)
	// Include the emptyDir volume and volume source in both containers
	v := v1.Volume{
		Name: constants.DefaultKanikoEmptyDirName,
		VolumeSource: v1.VolumeSource{
			EmptyDir: &v1.EmptyDirVolumeSource{},
		},
	}
	vm := v1.VolumeMount{
		Name:      constants.DefaultKanikoEmptyDirName,
		MountPath: constants.DefaultKanikoEmptyDirMountPath,
	}
	// Generate the init container, which will run until the /tmp/complete file is created
	ic := v1.Container{
		Name:  initContainer,
		Image: constants.DefaultAlpineImage,
		Args: []string{"sh", "-c", `while true; do
	sleep 1; if [ -f /tmp/complete ]; then break; fi
done`},
		VolumeMounts: []v1.VolumeMount{vm},
	}

	p.Spec.InitContainers = []v1.Container{ic}
	p.Spec.Containers[0].VolumeMounts = append(p.Spec.Containers[0].VolumeMounts, vm)
	p.Spec.Volumes = append(p.Spec.Volumes, v)
	return p
}

// ModifyPod first copies over the buildcontext tarball into the init container tmp dir via kubectl cp
// Via kubectl exec, we extract the tarball to the empty dir
// Then, via kubectl exec, create the /tmp/complete file via kubectl exec to complete the init container
func (g *LocalDir) ModifyPod(ctx context.Context, p *v1.Pod) error {
	client, err := kubernetes.GetClientset()
	if err != nil {
		return errors.Wrap(err, "getting clientset")
	}
	if err := kubernetes.WaitForPodInitialized(ctx, client.CoreV1().Pods(p.Namespace), p.Name); err != nil {
		return errors.Wrap(err, "waiting for pod to initialize")
	}

	f, err := os.Open(g.tarPath)
	if err != nil {
		return errors.Wrap(err, "opening context tar")
	}
	defer f.Close()

	// Copy the context to the empty dir and extract it
	copyAndExtract := exec.CommandContext(ctx, "kubectl", "exec", "-i", p.Name, "-c", initContainer, "-n", p.Namespace, "--", "tar", "-xzf", "-", "-C", constants.DefaultKanikoEmptyDirMountPath)
	copyAndExtract.Stdin = f
	if err := util.RunCmd(copyAndExtract); err != nil {
		return errors.Wrap(err, "copying and extracting buildcontext to empty dir")
	}
	// Generate a file to successfully terminate the init container
	file := exec.CommandContext(ctx, "kubectl", "exec", p.Name, "-c", initContainer, "-n", p.Namespace, "--", "touch", "/tmp/complete")
	return util.RunCmd(file)
}

// Cleanup deletes the buildcontext tarball stored on the local filesystem
func (g *LocalDir) Cleanup(ctx context.Context) error {
	return os.Remove(g.tarPath)
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"
)

// Signature and API related constants.
const (
	signV2Algorithm = "AWS"
)

// TODO add post policy signature.

// doesPresignV2SignatureMatch - Verify query headers with presigned signature
//     - http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationQueryStringAuth
// returns ErrNone if matches. S3 errors otherwise.
func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
	// Access credentials.
	cred := serverConfig.GetCredential()

	// Copy request
	req := *r

	// Validate if we do have query params.
	if req.URL.Query().Encode() == "" {
		return ErrInvalidQueryParams
	}

	// Validate if access key id same.
	if req.URL.Query().Get("AWSAccessKeyId") != cred.AccessKeyID {
		return ErrInvalidAccessKeyID
	}

	// Parse expires param into its native form.
	expired, err := strconv.ParseInt(req.URL.Query().Get("Expires"), 10, 64)
	if err != nil {
		errorIf(err, "Unable to parse expires query param")
		return ErrMalformedExpires
	}

	// Validate if the request has already expired.
	if expired < time.Now().UTC().Unix() {
		return ErrExpiredPresignRequest
	}

	// Get presigned string to sign.
	stringToSign := preStringifyHTTPReq(req)
	hm := hmac.New(sha1.New, []byte(cred.SecretAccessKey))
	hm.Write([]byte(stringToSign))

	// Calculate signature and validate.
	signature := base64.StdEncoding.EncodeToString(hm.Sum(nil))
	if req.URL.Query().Get("Signature") != signature {
		return ErrSignatureDoesNotMatch
	}

	// Success.
	return ErrNone
}

// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature;
// Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID, UTF-8-Encoding-Of( StringToSign ) ) );
//
// StringToSign = HTTP-Verb + "\n" +
//  	Content-Md5 + "\n" +
//  	Content-Type + "\n" +
//  	Date + "\n" +
//  	CanonicalizedProtocolHeaders +
//  	CanonicalizedResource;
//
// CanonicalizedResource = [ "/" + Bucket ] +
//  	<HTTP-Request-URI, from the protocol name up to the query string> +
//  	[ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
//
// CanonicalizedProtocolHeaders = <described below>

// doesSignV2Match - Verify authorization header with calculated header in accordance with
//     - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html
// returns true if matches, false otherwise. if error is not nil then it is always false
func doesSignV2Match(r *http.Request) APIErrorCode {
	// Access credentials.
	cred := serverConfig.GetCredential()

	// Copy request.
	req := *r

	// Save authorization header.
	v2Auth := req.Header.Get("Authorization")
	if v2Auth == "" {
		return ErrAuthHeaderEmpty
	}

	// Add date if not present.
	if date := req.Header.Get("Date"); date == "" {
		if date = req.Header.Get("X-Amz-Date"); date == "" {
			return ErrMissingDateHeader
		}
	}

	// Calculate HMAC for secretAccessKey.
	stringToSign := stringifyHTTPReq(req)
	hm := hmac.New(sha1.New, []byte(cred.SecretAccessKey))
	hm.Write([]byte(stringToSign))

	// Prepare auth header.
	authHeader := new(bytes.Buffer)
	authHeader.WriteString(fmt.Sprintf("%s %s:", signV2Algorithm, cred.AccessKeyID))
	encoder := base64.NewEncoder(base64.StdEncoding, authHeader)
	encoder.Write(hm.Sum(nil))
	encoder.Close()

	// Verify if signature match.
	if authHeader.String() != v2Auth {
		return ErrSignatureDoesNotMatch
	}

	return ErrNone
}

// From the Amazon docs:
//
// StringToSign = HTTP-Verb + "\n" +
// 	 Content-Md5 + "\n" +
//	 Content-Type + "\n" +
//	 Expires + "\n" +
//	 CanonicalizedProtocolHeaders +
//	 CanonicalizedResource;
func preStringifyHTTPReq(req http.Request) string {
	buf := new(bytes.Buffer)
	// Write standard headers.
	writePreSignV2Headers(buf, req)
	// Write canonicalized protocol headers if any.
	writeCanonicalizedHeaders(buf, req)
	// Write canonicalized Query resources if any.
	isPreSign := true
	writeCanonicalizedResource(buf, req, isPreSign)
	return buf.String()
}

// writePreSignV2Headers - write preSign v2 required headers.
func writePreSignV2Headers(buf *bytes.Buffer, req http.Request) {
	buf.WriteString(req.Method + "\n")
	buf.WriteString(req.Header.Get("Content-Md5") + "\n")
	buf.WriteString(req.Header.Get("Content-Type") + "\n")
	buf.WriteString(req.Header.Get("Expires") + "\n")
}

// From the Amazon docs:
//
// StringToSign = HTTP-Verb + "\n" +
// 	 Content-Md5 + "\n" +
//	 Content-Type + "\n" +
//	 Date + "\n" +
//	 CanonicalizedProtocolHeaders +
//	 CanonicalizedResource;
func stringifyHTTPReq(req http.Request) string {
	buf := new(bytes.Buffer)
	// Write standard headers.
	writeSignV2Headers(buf, req)
	// Write canonicalized protocol headers if any.
	writeCanonicalizedHeaders(buf, req)
	// Write canonicalized Query resources if any.
	isPreSign := false
	writeCanonicalizedResource(buf, req, isPreSign)
	return buf.String()
}

// writeSignV2Headers - write signV2 required headers.
func writeSignV2Headers(buf *bytes.Buffer, req http.Request) {
	buf.WriteString(req.Method + "\n")
	buf.WriteString(req.Header.Get("Content-Md5") + "\n")
	buf.WriteString(req.Header.Get("Content-Type") + "\n")
	buf.WriteString(req.Header.Get("Date") + "\n")
}

// writeCanonicalizedHeaders - write canonicalized headers.
func writeCanonicalizedHeaders(buf *bytes.Buffer, req http.Request) {
	var protoHeaders []string
	vals := make(map[string][]string)
	for k, vv := range req.Header {
		// All the AMZ headers should be lowercase
		lk := strings.ToLower(k)
		if strings.HasPrefix(lk, "x-amz") {
			protoHeaders = append(protoHeaders, lk)
			vals[lk] = vv
		}
	}
	sort.Strings(protoHeaders)
	for _, k := range protoHeaders {
		buf.WriteString(k)
		buf.WriteByte(':')
		for idx, v := range vals[k] {
			if idx > 0 {
				buf.WriteByte(',')
			}
			if strings.Contains(v, "\n") {
				// TODO: "Unfold" long headers that
				// span multiple lines (as allowed by
				// RFC 2616, section 4.2) by replacing
				// the folding white-space (including
				// new-line) by a single space.
				buf.WriteString(v)
			} else {
				buf.WriteString(v)
			}
		}
		buf.WriteByte('\n')
	}
}

// The following list is already sorted and should always be, otherwise we could
// have signature-related issues
var resourceList = []string{
	"acl",
	"delete",
	"location",
	"logging",
	"notification",
	"partNumber",
	"policy",
	"requestPayment",
	"torrent",
	"uploadId",
	"uploads",
	"versionId",
	"versioning",
	"versions",
	"website",
}

// From the Amazon docs:
//
// CanonicalizedResource = [ "/" + Bucket ] +
// 	  <HTTP-Request-URI, from the protocol name up to the query string> +
// 	  [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request, isPreSign bool) {
	// Save request URL.
	requestURL := req.URL
	// Get encoded URL path.
	path := getURLEncodedName(requestURL.Path)
	if isPreSign {
		// Get encoded URL path.
		if len(requestURL.Query()) > 0 {
			// Keep the usual queries unescaped for string to sign.
			query, _ := url.QueryUnescape(queryEncode(requestURL.Query()))
			path = path + "?" + query
		}
		buf.WriteString(path)
		return
	}
	buf.WriteString(path)
	if requestURL.RawQuery != "" {
		var n int
		vals, _ := url.ParseQuery(requestURL.RawQuery)
		// Verify if any sub resource queries are present, if yes
		// canonicallize them.
		for _, resource := range resourceList {
			if vv, ok := vals[resource]; ok && len(vv) > 0 {
				n++
				// First element
				switch n {
				case 1:
					buf.WriteByte('?')
				// The rest
				default:
					buf.WriteByte('&')
				}
				buf.WriteString(resource)
				// Request parameters
				if len(vv[0]) > 0 {
					buf.WriteByte('=')
					buf.WriteString(strings.Replace(url.QueryEscape(vv[0]), "+", "%20", -1))
				}
			}
		}
	}
}
<commit_msg>signature/v2: Fix presigned requests.<commit_after>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"
)

// Signature and API related constants.
const (
	signV2Algorithm = "AWS"
)

// TODO add post policy signature.

// doesPresignV2SignatureMatch - Verify query headers with presigned signature
//     - http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationQueryStringAuth
// returns ErrNone if matches. S3 errors otherwise.
func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
	// Access credentials.
	cred := serverConfig.GetCredential()

	// Copy request
	req := *r

	// Validate if we do have query params.
	if req.URL.Query().Encode() == "" {
		return ErrInvalidQueryParams
	}

	// Validate if access key id same.
	if req.URL.Query().Get("AWSAccessKeyId") != cred.AccessKeyID {
		return ErrInvalidAccessKeyID
	}

	// Parse expires param into its native form.
	expired, err := strconv.ParseInt(req.URL.Query().Get("Expires"), 10, 64)
	if err != nil {
		errorIf(err, "Unable to parse expires query param")
		return ErrMalformedExpires
	}

	// Validate if the request has already expired.
	if expired < time.Now().UTC().Unix() {
		return ErrExpiredPresignRequest
	}

	// Save incoming siganture to be validated later.
	incomingSignature := req.URL.Query().Get("Signature")

	// Set the expires header for string to sign.
	req.Header.Set("Expires", strconv.FormatInt(expired, 10))

	/// Empty out the query params, we only need to validate signature.
	query := req.URL.Query()
	// Remove all the query params added for signature alone, we need
	// a proper URL for string to sign.
	query.Del("Expires")
	query.Del("AWSAccessKeyId")
	query.Del("Signature")
	// Query encode whatever is left back to RawQuery.
	req.URL.RawQuery = queryEncode(query)

	// Get presigned string to sign.
	stringToSign := preStringifyHTTPReq(req)
	hm := hmac.New(sha1.New, []byte(cred.SecretAccessKey))
	hm.Write([]byte(stringToSign))

	// Calculate signature and validate.
	signature := base64.StdEncoding.EncodeToString(hm.Sum(nil))
	if incomingSignature != signature {
		return ErrSignatureDoesNotMatch
	}

	// Success.
	return ErrNone
}

// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature;
// Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID, UTF-8-Encoding-Of( StringToSign ) ) );
//
// StringToSign = HTTP-Verb + "\n" +
//  	Content-Md5 + "\n" +
//  	Content-Type + "\n" +
//  	Date + "\n" +
//  	CanonicalizedProtocolHeaders +
//  	CanonicalizedResource;
//
// CanonicalizedResource = [ "/" + Bucket ] +
//  	<HTTP-Request-URI, from the protocol name up to the query string> +
//  	[ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
//
// CanonicalizedProtocolHeaders = <described below>

// doesSignV2Match - Verify authorization header with calculated header in accordance with
//     - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html
// returns true if matches, false otherwise. if error is not nil then it is always false
func doesSignV2Match(r *http.Request) APIErrorCode {
	// Access credentials.
	cred := serverConfig.GetCredential()

	// Copy request.
	req := *r

	// Save authorization header.
	v2Auth := req.Header.Get("Authorization")
	if v2Auth == "" {
		return ErrAuthHeaderEmpty
	}

	// Add date if not present.
	if date := req.Header.Get("Date"); date == "" {
		if date = req.Header.Get("X-Amz-Date"); date == "" {
			return ErrMissingDateHeader
		}
	}

	// Calculate HMAC for secretAccessKey.
	stringToSign := stringifyHTTPReq(req)
	hm := hmac.New(sha1.New, []byte(cred.SecretAccessKey))
	hm.Write([]byte(stringToSign))

	// Prepare auth header.
	authHeader := new(bytes.Buffer)
	authHeader.WriteString(fmt.Sprintf("%s %s:", signV2Algorithm, cred.AccessKeyID))
	encoder := base64.NewEncoder(base64.StdEncoding, authHeader)
	encoder.Write(hm.Sum(nil))
	encoder.Close()

	// Verify if signature match.
	if authHeader.String() != v2Auth {
		return ErrSignatureDoesNotMatch
	}

	return ErrNone
}

// From the Amazon docs:
//
// StringToSign = HTTP-Verb + "\n" +
// 	 Content-Md5 + "\n" +
//	 Content-Type + "\n" +
//	 Expires + "\n" +
//	 CanonicalizedProtocolHeaders +
//	 CanonicalizedResource;
func preStringifyHTTPReq(req http.Request) string {
	buf := new(bytes.Buffer)
	// Write standard headers.
	writePreSignV2Headers(buf, req)
	// Write canonicalized protocol headers if any.
	writeCanonicalizedHeaders(buf, req)
	// Write canonicalized Query resources if any.
	isPreSign := true
	writeCanonicalizedResource(buf, req, isPreSign)
	return buf.String()
}

// writePreSignV2Headers - write preSign v2 required headers.
func writePreSignV2Headers(buf *bytes.Buffer, req http.Request) {
	buf.WriteString(req.Method + "\n")
	buf.WriteString(req.Header.Get("Content-Md5") + "\n")
	buf.WriteString(req.Header.Get("Content-Type") + "\n")
	buf.WriteString(req.Header.Get("Expires") + "\n")
}

// From the Amazon docs:
//
// StringToSign = HTTP-Verb + "\n" +
// 	 Content-Md5 + "\n" +
//	 Content-Type + "\n" +
//	 Date + "\n" +
//	 CanonicalizedProtocolHeaders +
//	 CanonicalizedResource;
func stringifyHTTPReq(req http.Request) string {
	buf := new(bytes.Buffer)
	// Write standard headers.
	writeSignV2Headers(buf, req)
	// Write canonicalized protocol headers if any.
	writeCanonicalizedHeaders(buf, req)
	// Write canonicalized Query resources if any.
	isPreSign := false
	writeCanonicalizedResource(buf, req, isPreSign)
	return buf.String()
}

// writeSignV2Headers - write signV2 required headers.
func writeSignV2Headers(buf *bytes.Buffer, req http.Request) {
	buf.WriteString(req.Method + "\n")
	buf.WriteString(req.Header.Get("Content-Md5") + "\n")
	buf.WriteString(req.Header.Get("Content-Type") + "\n")
	buf.WriteString(req.Header.Get("Date") + "\n")
}

// writeCanonicalizedHeaders - write canonicalized headers.
func writeCanonicalizedHeaders(buf *bytes.Buffer, req http.Request) {
	var protoHeaders []string
	vals := make(map[string][]string)
	for k, vv := range req.Header {
		// All the AMZ headers should be lowercase
		lk := strings.ToLower(k)
		if strings.HasPrefix(lk, "x-amz") {
			protoHeaders = append(protoHeaders, lk)
			vals[lk] = vv
		}
	}
	sort.Strings(protoHeaders)
	for _, k := range protoHeaders {
		buf.WriteString(k)
		buf.WriteByte(':')
		for idx, v := range vals[k] {
			if idx > 0 {
				buf.WriteByte(',')
			}
			if strings.Contains(v, "\n") {
				// TODO: "Unfold" long headers that
				// span multiple lines (as allowed by
				// RFC 2616, section 4.2) by replacing
				// the folding white-space (including
				// new-line) by a single space.
				buf.WriteString(v)
			} else {
				buf.WriteString(v)
			}
		}
		buf.WriteByte('\n')
	}
}

// The following list is already sorted and should always be, otherwise we could
// have signature-related issues
var resourceList = []string{
	"acl",
	"delete",
	"location",
	"logging",
	"notification",
	"partNumber",
	"policy",
	"requestPayment",
	"torrent",
	"uploadId",
	"uploads",
	"versionId",
	"versioning",
	"versions",
	"website",
}

// From the Amazon docs:
//
// CanonicalizedResource = [ "/" + Bucket ] +
// 	  <HTTP-Request-URI, from the protocol name up to the query string> +
// 	  [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request, isPreSign bool) {
	// Save request URL.
	requestURL := req.URL
	// Get encoded URL path.
	path := getURLEncodedName(requestURL.Path)
	if isPreSign {
		// Get encoded URL path.
		if len(requestURL.Query()) > 0 {
			// Keep the usual queries unescaped for string to sign.
			query, _ := url.QueryUnescape(queryEncode(requestURL.Query()))
			path = path + "?" + query
		}
		buf.WriteString(path)
		return
	}
	buf.WriteString(path)
	if requestURL.RawQuery != "" {
		var n int
		vals, _ := url.ParseQuery(requestURL.RawQuery)
		// Verify if any sub resource queries are present, if yes
		// canonicallize them.
		for _, resource := range resourceList {
			if vv, ok := vals[resource]; ok && len(vv) > 0 {
				n++
				// First element
				switch n {
				case 1:
					buf.WriteByte('?')
				// The rest
				default:
					buf.WriteByte('&')
				}
				buf.WriteString(resource)
				// Request parameters
				if len(vv[0]) > 0 {
					buf.WriteByte('=')
					buf.WriteString(strings.Replace(url.QueryEscape(vv[0]), "+", "%20", -1))
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package director

import (
	"errors"
	"fmt"
	"io"
	"os/exec"
)

var defaultBoshArgs = []string{"--non-interactive", "--tty", "--no-color"}

// RunAuthenticatedCommand runs a command against the bosh director, after authenticating
func (client *Client) RunAuthenticatedCommand(stdout, stderr io.Writer, detach bool, args ...string) error {
	if detach {
		return errors.New("detach mode not yet implemented")
	}
	if err := client.ensureBinaryDownloaded(); err != nil {
		return err
	}
	args = append([]string{
		"--environment",
		fmt.Sprintf("https://%s", client.creds.Host),
		"--ca-cert",
		client.caCertPath,
		"--client",
		client.creds.Username,
		"--client-secret",
		client.creds.Password,
	}, args...)

	return client.RunCommand(stdout, stderr, args...)
}

// RunCommand runs a command against the bosh director
// https://github.com/cloudfoundry/bosh-cli/blob/master/main.go
func (client *Client) RunCommand(stdout, stderr io.Writer, args ...string) error {
	if err := client.ensureBinaryDownloaded(); err != nil {
		return err
	}

	cmd := exec.Command(client.tempDir.Path("bosh-cli"), args...)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	return cmd.Run()
}
<commit_msg>ensure default bosh args are set<commit_after>package director

import (
	"errors"
	"fmt"
	"io"
	"os/exec"
)

var defaultBoshArgs = []string{"--non-interactive", "--tty", "--no-color"}

// RunAuthenticatedCommand runs a command against the bosh director, after authenticating
func (client *Client) RunAuthenticatedCommand(stdout, stderr io.Writer, detach bool, args ...string) error {
	if detach {
		return errors.New("detach mode not yet implemented")
	}
	if err := client.ensureBinaryDownloaded(); err != nil {
		return err
	}
	args = append([]string{
		"--environment",
		fmt.Sprintf("https://%s", client.creds.Host),
		"--ca-cert",
		client.caCertPath,
		"--client",
		client.creds.Username,
		"--client-secret",
		client.creds.Password,
	}, args...)

	return client.RunCommand(stdout, stderr, args...)
}

// RunCommand runs a command against the bosh director
// https://github.com/cloudfoundry/bosh-cli/blob/master/main.go
func (client *Client) RunCommand(stdout, stderr io.Writer, args ...string) error {
	if err := client.ensureBinaryDownloaded(); err != nil {
		return err
	}

	args = append(defaultBoshArgs, args...)

	cmd := exec.Command(client.tempDir.Path("bosh-cli"), args...)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	return cmd.Run()
}
<|endoftext|>


<commit_before>package heroku

import (
	"fmt"
	"strings"

	"github.com/satori/uuid"
)

var validPipelineStageNames = []string{
	"review",
	"development",
	"staging",
	"production",
}

func validatePipelineStageName(v interface{}, k string) (ws []string, errors []error) {
	for _, s := range validPipelineStageNames {
		if v == s {
			return
		}
	}

	err := fmt.Errorf(
		"%s is an invalid pipeline stage, must be one of [%s]",
		v,
		strings.Join(validPipelineStageNames, ", "),
	)
	errors = append(errors, err)
	return
}

func validateUUID(v interface{}, k string) (ws []string, errors []error) {
	if _, err := uuid.FromString(v.(string)); err != nil {
		errors = append(errors, fmt.Errorf("%q is an invalid UUID: %s", k, err))
	}
	return
}
<commit_msg>provider/heroku: scope valid stage names to func<commit_after>package heroku

import (
	"fmt"
	"strings"

	"github.com/satori/uuid"
)

func validatePipelineStageName(v interface{}, k string) (ws []string, errors []error) {
	validPipelineStageNames := []string{
		"review",
		"development",
		"staging",
		"production",
	}

	for _, s := range validPipelineStageNames {
		if v == s {
			return
		}
	}

	err := fmt.Errorf(
		"%s is an invalid pipeline stage, must be one of [%s]",
		v,
		strings.Join(validPipelineStageNames, ", "),
	)
	errors = append(errors, err)
	return
}

func validateUUID(v interface{}, k string) (ws []string, errors []error) {
	if _, err := uuid.FromString(v.(string)); err != nil {
		errors = append(errors, fmt.Errorf("%q is an invalid UUID: %s", k, err))
	}
	return
}
<|endoftext|>


<commit_before>// +build !appengine

package orm

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"testing"

	_ "gnd.la/orm/driver/mysql"
	_ "gnd.la/orm/driver/postgres"
	_ "gnd.la/orm/driver/sqlite"
)

// This file has tests which run all the tests for
// every driver.

type sqliteOpener struct {
}

func (o *sqliteOpener) Open(t testing.TB) (*Orm, interface{}) {
	f, err := ioutil.TempFile("", "sqlite-")
	if err != nil {
		t.Fatal(err)
	}
	f.Close()
	orm := newOrm(t, "sqlite://"+f.Name(), true)
	orm.SqlDB().Exec("PRAGMA journal_mode = WAL")
	orm.SqlDB().Exec("PRAGMA foreign_keys = on")
	return orm, f.Name()
}

func (o *sqliteOpener) Close(data interface{}) {
	os.Remove(data.(string))
}

type postgresOpener struct {
}

func (o *postgresOpener) Open(t testing.TB) (*Orm, interface{}) {
	u, err := user.Current()
	if err != nil {
		t.Fatal(err)
	}
	exec.Command("dropdb", "gotest").Run()
	if err := exec.Command("createdb", "gotest").Run(); err != nil {
		t.Skip("cannot create gotest postgres database, skipping test")
	}
	return newOrm(t, fmt.Sprintf("postgres://dbname=gotest user=%v password=%v", u.Username, u.Username), true), nil
}

func (o *postgresOpener) Close(_ interface{}) {}

type mysqlOpener struct {
}

func (o *mysqlOpener) Open(t testing.TB) (*Orm, interface{}) {
	orm := newOrm(t, "mysql://gotest:gotest@/test", true)
	db := orm.SqlDB()
	if _, err := db.Exec("DROP DATABASE IF EXISTS gotest"); err != nil {
		t.Skipf("cannot connect to mysql database, skipping test: %s", err)
	}
	if _, err := db.Exec("CREATE DATABASE gotest"); err != nil {
		t.Fatal(err)
	}
	if err := orm.Close(); err != nil {
		t.Fatal(err)
	}
	return newOrm(t, "mysql://gotest:gotest@/gotest", true), nil
}

func (o *mysqlOpener) Close(_ interface{}) {}

func TestSqlite(t *testing.T) {
	runAllTests(t, &sqliteOpener{})
}

func TestPostgres(t *testing.T) {
	runAllTests(t, &postgresOpener{})
}

func TestMysql(t *testing.T) {
	runAllTests(t, &mysqlOpener{})
}

func init() {
	openers["default"] = &sqliteOpener{}
	openers["sqlite"] = &sqliteOpener{}
	openers["postgres"] = &postgresOpener{}
	openers["mysql"] = &mysqlOpener{}
}
<commit_msg>Skip MySQL tests if MySQL is not running locally<commit_after>// +build !appengine

package orm

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"testing"

	"net"

	_ "gnd.la/orm/driver/mysql"
	_ "gnd.la/orm/driver/postgres"
	_ "gnd.la/orm/driver/sqlite"
)

// This file has tests which run all the tests for
// every driver.

type sqliteOpener struct {
}

func (o *sqliteOpener) Open(t testing.TB) (*Orm, interface{}) {
	f, err := ioutil.TempFile("", "sqlite-")
	if err != nil {
		t.Fatal(err)
	}
	f.Close()
	orm := newOrm(t, "sqlite://"+f.Name(), true)
	orm.SqlDB().Exec("PRAGMA journal_mode = WAL")
	orm.SqlDB().Exec("PRAGMA foreign_keys = on")
	return orm, f.Name()
}

func (o *sqliteOpener) Close(data interface{}) {
	os.Remove(data.(string))
}

type postgresOpener struct {
}

func (o *postgresOpener) Open(t testing.TB) (*Orm, interface{}) {
	u, err := user.Current()
	if err != nil {
		t.Fatal(err)
	}
	exec.Command("dropdb", "gotest").Run()
	if err := exec.Command("createdb", "gotest").Run(); err != nil {
		t.Skip("cannot create gotest postgres database, skipping test")
	}
	return newOrm(t, fmt.Sprintf("postgres://dbname=gotest user=%v password=%v", u.Username, u.Username), true), nil
}

func (o *postgresOpener) Close(_ interface{}) {}

type mysqlOpener struct {
}

func (o *mysqlOpener) Open(t testing.TB) (*Orm, interface{}) {
	// Check if MySQL is running
	conn, err := net.Dial("tcp", "localhost:3306")
	if err != nil {
		t.Skipf("MySQL is not running, skipping test (%v)", err)
	}
	conn.Close()
	orm := newOrm(t, "mysql://gotest:gotest@/test", true)
	db := orm.SqlDB()
	if _, err := db.Exec("DROP DATABASE IF EXISTS gotest"); err != nil {
		t.Skipf("cannot connect to mysql database, skipping test: %s", err)
	}
	if _, err := db.Exec("CREATE DATABASE gotest"); err != nil {
		t.Fatal(err)
	}
	if err := orm.Close(); err != nil {
		t.Fatal(err)
	}
	return newOrm(t, "mysql://gotest:gotest@/gotest", true), nil
}

func (o *mysqlOpener) Close(_ interface{}) {}

func TestSqlite(t *testing.T) {
	runAllTests(t, &sqliteOpener{})
}

func TestPostgres(t *testing.T) {
	runAllTests(t, &postgresOpener{})
}

func TestMysql(t *testing.T) {
	runAllTests(t, &mysqlOpener{})
}

func init() {
	openers["default"] = &sqliteOpener{}
	openers["sqlite"] = &sqliteOpener{}
	openers["postgres"] = &postgresOpener{}
	openers["mysql"] = &mysqlOpener{}
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import "github.com/coreos/etcd/Godeps/_workspace/src/github.com/boltdb/bolt"

// TODO: support syscall.MAP_POPULATE in windows.
// Need upstream patch from boltdb/bolt.
var boltOpenOptions *bolt.Option = nil
<commit_msg>storage/backend: fixes Windows compile error<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import "github.com/coreos/etcd/Godeps/_workspace/src/github.com/boltdb/bolt"

// TODO: support syscall.MAP_POPULATE in windows.
// Need upstream patch from boltdb/bolt.
var boltOpenOptions *bolt.Options = nil
<|endoftext|>


<commit_before>package elk

import (
	"errors"
	"fmt"
	"log"
	"log/syslog"
	"net"
	"os"
	"text/template"
	"time"

	"github.com/delectable/logspout/router"
)

func init() {
	router.AdapterFactories.Register(NewElkAdapter, "elk")
}

func getopt(name, dfault string) string {
	value := os.Getenv(name)
	if value == "" {
		value = dfault
	}
	return value
}

func NewElkAdapter(route *router.Route) (router.LogAdapter, error) {
	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
	if !found {
		return nil, errors.New("unable to find adapter: " + route.Adapter)
	}
	conn, err := transport.Dial(route.Address, route.Options)
	if err != nil {
		return nil, err
	}

	// priority := getopt("ELK_PRIORITY", "{{.Priority}}")
	// hostname := getopt("ELK_HOSTNAME", "{{.Container.Config.Hostname}}")
	// pid := getopt("ELK_PID", "{{.Container.State.Pid}}")
	// tag := getopt("ELK_TAG", "{{.ContainerName}}"+route.Options["append_tag"])
	// structuredData := getopt("ELK_STRUCTURED_DATA", "")
	// if route.Options["structured_data"] != "" {
	// 	structuredData = route.Options["structured_data"]
	// }
	data := getopt("ELK_DATA", "{{.Data}}")

	tmplStr := fmt.Sprintf("CRUNCHY BACON: <%d> {{.Timestamp}} %s %s %d - [%s] %s",
		priority, hostname, tag, pid, structuredData, data)

	// tmplStr := "LOG ENTRY FROM {{.Timestamp}}"

	fmt.Println("GOT A LOG ENTRY.")

	tmpl, err := template.New("elk").Parse(tmplStr)
	if err != nil {
		return nil, err
	}
	return &ElkAdapter{
		route: route,
		conn:  conn,
		tmpl:  tmpl,
	}, nil
}

type ElkAdapter struct {
	conn  net.Conn
	route *router.Route
	tmpl  *template.Template
}

func (a *ElkAdapter) Stream(logstream chan *router.Message) {
	for message := range logstream {
		err := a.tmpl.Execute(a.conn, &ElkMessage{message, a})
		fmt.Println("LOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOGGGG!")
		if err != nil {
			log.Println("syslog:", err)
			a.route.Close()
			return
		}
	}
}

type ElkMessage struct {
	*router.Message
	adapter *ElkAdapter
}

func (m *ElkMessage) Priority() syslog.Priority {
	switch m.Message.Source {
	case "stdout":
		return syslog.LOG_USER | syslog.LOG_INFO
	case "stderr":
		return syslog.LOG_USER | syslog.LOG_ERR
	default:
		return syslog.LOG_DAEMON | syslog.LOG_INFO
	}
}

func (m *ElkMessage) Hostname() string {
	h, _ := os.Hostname()
	return h
}

func (m *ElkMessage) LocalAddr() string {
	return m.adapter.conn.LocalAddr().String()
}

func (m *ElkMessage) Timestamp() string {
	return m.Message.Time.Format(time.RFC3339)
}

func (m *ElkMessage) ContainerName() string {
	return m.Message.Container.Name[1:]
}
<commit_msg>Testing ELK Adapter<commit_after>package elk

import (
	"errors"
	"fmt"
	"log"
	"log/syslog"
	"net"
	"os"
	"text/template"
	"time"

	"github.com/delectable/logspout/router"
)

func init() {
	router.AdapterFactories.Register(NewElkAdapter, "elk")
}

func getopt(name, dfault string) string {
	value := os.Getenv(name)
	if value == "" {
		value = dfault
	}
	return value
}

func NewElkAdapter(route *router.Route) (router.LogAdapter, error) {
	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
	if !found {
		return nil, errors.New("unable to find adapter: " + route.Adapter)
	}
	conn, err := transport.Dial(route.Address, route.Options)
	if err != nil {
		return nil, err
	}

	priority := getopt("ELK_PRIORITY", "{{.Priority}}")
	hostname := getopt("ELK_HOSTNAME", "{{.Container.Config.Hostname}}")
	pid := getopt("ELK_PID", "{{.Container.State.Pid}}")
	tag := getopt("ELK_TAG", "{{.ContainerName}}"+route.Options["append_tag"])
	structuredData := getopt("ELK_STRUCTURED_DATA", "")
	if route.Options["structured_data"] != "" {
		structuredData = route.Options["structured_data"]
	}
	data := getopt("ELK_DATA", "{{.Data}}")

	tmplStr := fmt.Sprintf("CRUNCHY BACON: <%d> {{.Timestamp}} %s %s %d - [%s] %s",
		priority, hostname, tag, pid, structuredData, data)

	// tmplStr := "LOG ENTRY FROM {{.Timestamp}}"

	fmt.Println("GOT A LOG ENTRY.")

	tmpl, err := template.New("elk").Parse(tmplStr)
	if err != nil {
		return nil, err
	}
	return &ElkAdapter{
		route: route,
		conn:  conn,
		tmpl:  tmpl,
	}, nil
}

type ElkAdapter struct {
	conn  net.Conn
	route *router.Route
	tmpl  *template.Template
}

func (a *ElkAdapter) Stream(logstream chan *router.Message) {
	for message := range logstream {
		err := a.tmpl.Execute(a.conn, &ElkMessage{message, a})
		fmt.Println("LOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOGGGG!")
		if err != nil {
			log.Println("syslog:", err)
			a.route.Close()
			return
		}
	}
}

type ElkMessage struct {
	*router.Message
	adapter *ElkAdapter
}

func (m *ElkMessage) Priority() syslog.Priority {
	switch m.Message.Source {
	case "stdout":
		return syslog.LOG_USER | syslog.LOG_INFO
	case "stderr":
		return syslog.LOG_USER | syslog.LOG_ERR
	default:
		return syslog.LOG_DAEMON | syslog.LOG_INFO
	}
}

func (m *ElkMessage) Hostname() string {
	h, _ := os.Hostname()
	return h
}

func (m *ElkMessage) LocalAddr() string {
	return m.adapter.conn.LocalAddr().String()
}

func (m *ElkMessage) Timestamp() string {
	return m.Message.Time.Format(time.RFC3339)
}

func (m *ElkMessage) ContainerName() string {
	return m.Message.Container.Name[1:]
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"path/filepath"

	"github.com/Smartling/api-sdk-go"
	"github.com/reconquest/hierr-go"
)

func doFilesPush(
	client *smartling.Client,
	config Config,
	args map[string]interface{},
) error {
	var (
		project     = config.ProjectID
		file, _     = args["<file>"].(string)
		uri, useURI = args["<uri>"].(string)
		branch, _   = args["--branch"].(string)
		locales, _  = args["--locale"].([]string)
		authorize   = args["--authorize"].(bool)
		directory   = args["--directory"].(string)
		fileType, _ = args["--type"].(string)
	)

	patterns := []string{}

	if file != "" {
		patterns = append(patterns, file)
	} else {
		for pattern, section := range config.Files {
			if section.Push.Type != "" {
				patterns = append(patterns, pattern)
			}
		}
	}

	files := []string{}

	for _, pattern := range patterns {
		chunk, err := globFilesLocally(directory, pattern)
		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to find matching files to upload`,
				),

				`Check, that specified pattern is valid and refer to help for`+
					` more information about glob patterns.`,
			)
		}

		files = append(files, chunk...)
	}

	if len(files) == 0 {
		return NewError(
			fmt.Errorf(`no files found by specified patterns`),

			`Check command line pattern if any and configuration file for`+
				` more patterns to search for.`,
		)
	}

	if uri != "" && len(files) > 1 {
		return NewError(
			fmt.Errorf(
				`more than one file is matching speciifed pattern and <uri>`+
					` is specified too`,
			),

			`Either remove <uri> argument or make sure that only one file`+
				` is matching mask.`,
		)
	}

	for _, file := range files {
		if !useURI {
			uri = file
		}

		fileConfig, err := config.GetFileConfig(file)
		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to retrieve file specific configuration`,
				),

				``,
			)
		}

		contents, err := ioutil.ReadFile(file)
		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to read file contents "%s"`,
					file,
				),

				`Check that file exists and readable by current user.`,
			)
		}

		request := smartling.FileUploadRequest{
			File:               contents,
			FileURI:            branch + uri,
			Authorize:          authorize,
			LocalesToAuthorize: locales,
		}

		if fileConfig.Push.Type == "" {
			if fileType == "" {
				request.FileType = smartling.GetFileTypeByExtension(
					filepath.Ext(file),
				)

				if request.FileType == smartling.FileTypeUnknown {
					return NewError(
						fmt.Errorf(
							"unable to deduce file type from extension: %q",
							filepath.Ext(file),
						),

						`You need to specify file type via --type option.`,
					)
				}
			} else {
				request.FileType = smartling.FileType(fileType)
			}
		} else {
			request.FileType = smartling.FileType(fileConfig.Push.Type)
		}

		request.Smartling.Directives = fileConfig.Push.Directives

		response, err := client.UploadFile(project, request)

		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to upload file "%s"`,
					file,
				),

				`Check, that you have enough permissions to upload file to`+
					` the specified project`,
			)
		}

		status := "new"
		if response.Overwritten {
			status = "overwritten"
		}

		fmt.Printf(
			"%s %s [strings %d words %d]\n",
			branch+file,
			status,
			response.StringCount,
			response.WordCount,
		)
	}

	return nil
}
<commit_msg>#17: add implicit / if branch is specified<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"

	"github.com/Smartling/api-sdk-go"
	"github.com/reconquest/hierr-go"
)

func doFilesPush(
	client *smartling.Client,
	config Config,
	args map[string]interface{},
) error {
	var (
		project     = config.ProjectID
		file, _     = args["<file>"].(string)
		uri, useURI = args["<uri>"].(string)
		branch, _   = args["--branch"].(string)
		locales, _  = args["--locale"].([]string)
		authorize   = args["--authorize"].(bool)
		directory   = args["--directory"].(string)
		fileType, _ = args["--type"].(string)
	)

	if branch != "" {
		branch = strings.TrimSuffix(branch, "/") + "/"
	}

	patterns := []string{}

	if file != "" {
		patterns = append(patterns, file)
	} else {
		for pattern, section := range config.Files {
			if section.Push.Type != "" {
				patterns = append(patterns, pattern)
			}
		}
	}

	files := []string{}

	for _, pattern := range patterns {
		chunk, err := globFilesLocally(directory, pattern)
		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to find matching files to upload`,
				),

				`Check, that specified pattern is valid and refer to help for`+
					` more information about glob patterns.`,
			)
		}

		files = append(files, chunk...)
	}

	if len(files) == 0 {
		return NewError(
			fmt.Errorf(`no files found by specified patterns`),

			`Check command line pattern if any and configuration file for`+
				` more patterns to search for.`,
		)
	}

	if uri != "" && len(files) > 1 {
		return NewError(
			fmt.Errorf(
				`more than one file is matching speciifed pattern and <uri>`+
					` is specified too`,
			),

			`Either remove <uri> argument or make sure that only one file`+
				` is matching mask.`,
		)
	}

	for _, file := range files {
		if !useURI {
			uri = file
		}

		fileConfig, err := config.GetFileConfig(file)
		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to retrieve file specific configuration`,
				),

				``,
			)
		}

		contents, err := ioutil.ReadFile(file)
		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to read file contents "%s"`,
					file,
				),

				`Check that file exists and readable by current user.`,
			)
		}

		request := smartling.FileUploadRequest{
			File:               contents,
			FileURI:            branch + uri,
			Authorize:          authorize,
			LocalesToAuthorize: locales,
		}

		if fileConfig.Push.Type == "" {
			if fileType == "" {
				request.FileType = smartling.GetFileTypeByExtension(
					filepath.Ext(file),
				)

				if request.FileType == smartling.FileTypeUnknown {
					return NewError(
						fmt.Errorf(
							"unable to deduce file type from extension: %q",
							filepath.Ext(file),
						),

						`You need to specify file type via --type option.`,
					)
				}
			} else {
				request.FileType = smartling.FileType(fileType)
			}
		} else {
			request.FileType = smartling.FileType(fileConfig.Push.Type)
		}

		request.Smartling.Directives = fileConfig.Push.Directives

		response, err := client.UploadFile(project, request)

		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to upload file "%s"`,
					file,
				),

				`Check, that you have enough permissions to upload file to`+
					` the specified project`,
			)
		}

		status := "new"
		if response.Overwritten {
			status = "overwritten"
		}

		fmt.Printf(
			"%s %s [strings %d words %d]\n",
			branch+file,
			status,
			response.StringCount,
			response.WordCount,
		)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/mozillazg/go-pinyin"
	"golang.org/x/crypto/ssh/terminal"
)

func main() {
	heteronym := flag.Bool("e", false, "启用多音字模式")
	flag.Parse()
	hans := flag.Args()
	args := pinyin.NewArgs()
	args.Style = pinyin.Tone
	stdin := []byte{}
	if !terminal.IsTerminal(0) {
		stdin, _ = ioutil.ReadAll(os.Stdin)
	}
	if len(stdin) > 0 {
		hans = append(hans, string(stdin))
	}

	if len(hans) == 0 {
		fmt.Println("请至少输入一个汉字: pinyin HANS [HANS ...]")
		os.Exit(1)
	}
	if *heteronym {
		args.Heteronym = true
	}
	pys := pinyin.Pinyin(strings.Join(hans, ""), args)
	for _, s := range pys {
		fmt.Print(strings.Join(s, ","), " ")
	}
	if len(pys) > 0 {
		fmt.Println()
	}
}
<commit_msg>change to use github.com/mattn/go-isatty<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/mattn/go-isatty"
	"github.com/mozillazg/go-pinyin"
)

func main() {
	heteronym := flag.Bool("e", false, "启用多音字模式")
	flag.Parse()
	hans := flag.Args()
	args := pinyin.NewArgs()
	args.Style = pinyin.Tone
	stdin := []byte{}
	if !isatty.IsTerminal(os.Stdin.Fd()) {
		stdin, _ = ioutil.ReadAll(os.Stdin)
	}
	if len(stdin) > 0 {
		hans = append(hans, string(stdin))
	}

	if len(hans) == 0 {
		fmt.Println("请至少输入一个汉字: pinyin HANS [HANS ...]")
		os.Exit(1)
	}
	if *heteronym {
		args.Heteronym = true
	}
	pys := pinyin.Pinyin(strings.Join(hans, ""), args)
	for _, s := range pys {
		fmt.Print(strings.Join(s, ","), " ")
	}
	if len(pys) > 0 {
		fmt.Println()
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Dorival Pedroso. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"strings"
	"time"

	"github.com/cpmech/goga"
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/plt"
)

// main function
func main() {

	// flags
	benchmark := true
	ncpuMax := 16

	// benchmarking
	if benchmark {
		var nsol, tf int
		var et time.Duration
		X := make([]float64, ncpuMax)
		T := make([]float64, ncpuMax)
		S := make([]float64, ncpuMax) // speedup
		S[0] = 1
		for i := 0; i < ncpuMax; i++ {
			io.Pf("\n\n")
			nsol, tf, et = runone(i + 1)
			io.PfYel("elaspsedTime = %v\n", et)
			X[i] = float64(i + 1)
			T[i] = et.Seconds()
			if i > 0 {
				S[i] = T[0] / T[i] // Told / Tnew
			}
		}

		plt.SetForEps(0.75, 250)
		plt.Plot(X, S, io.Sf("'b-',marker='.', label='speedup: $N_{sol}=%d,\\,t_f=%d$', clip_on=0, zorder=100", nsol, tf))
		plt.Plot([]float64{1, 16}, []float64{1, 16}, "'k--',zorder=50")
		plt.Gll("$N_{cpu}:\\;$ number of groups", "speedup", "leg_out=1")
		plt.DoubleYscale("$T_{sys}:\\;$ system time [s]")
		plt.Plot(X, T, "'k-',color='gray', clip_on=0")
		plt.SaveD("/tmp/goga", "topology-speedup.eps")
		return
	}

	// normal run
	runone(-1)
}

func runone(ncpu int) (nsol, tf int, elaspsedTime time.Duration) {

	// input filename
	fn, fnkey := io.ArgToFilename(0, "ground10", ".sim", true)

	// GA parameters
	var opt goga.Optimiser
	opt.Read("ga-" + fnkey + ".json")
	opt.GenType = "rnd"
	nsol, tf = opt.Nsol, opt.Tf
	postproc := true
	if ncpu > 0 {
		opt.Ncpu = ncpu
		postproc = false
	}

	// FEM
	data := make([]*FemData, opt.Ncpu)
	for i := 0; i < opt.Ncpu; i++ {
		data[i] = NewData(fn, fnkey, i)
	}
	io.Pforan("MaxWeight = %v\n", data[0].MaxWeight)

	// set integers
	if data[0].Opt.BinInt {
		opt.CxInt = goga.CxInt
		opt.MtInt = goga.MtIntBin
		opt.BinInt = data[0].Ncells
	}

	// set floats
	opt.FltMin = make([]float64, data[0].Nareas)
	opt.FltMax = make([]float64, data[0].Nareas)
	for i := 0; i < data[0].Nareas; i++ {
		opt.FltMin[i] = data[0].Opt.Amin
		opt.FltMax[i] = data[0].Opt.Amax
	}

	// initialise optimiser
	opt.Nova = 2 // weight and deflection
	opt.Noor = 4 // mobility, feasibility, maxdeflection, stress
	opt.Init(goga.GenTrialSolutions, func(sol *goga.Solution, cpu int) {
		mob, fail, weight, umax, _, errU, errS := data[cpu].RunFEM(sol.Int, sol.Flt, 0, false)
		sol.Ova[0] = weight
		sol.Ova[1] = umax
		sol.Oor[0] = mob
		sol.Oor[1] = fail
		sol.Oor[2] = errU
		sol.Oor[3] = errS
	}, nil, 0, 0, 0)

	// initial solutions
	var sols0 []*goga.Solution
	if false {
		sols0 = opt.GetSolutionsCopy()
	}

	// benchmark
	initialTime := time.Now()
	defer func() {
		elaspsedTime = time.Now().Sub(initialTime)
	}()

	// solve
	opt.Verbose = true
	opt.Solve()
	goga.SortByOva(opt.Solutions, 0)

	// post processing
	if !postproc {
		return
	}

	// check
	front0 := make([]*goga.Solution, 0)
	var nfailed, nsuccess int
	for _, sol := range opt.Solutions {
		var failed bool
		for _, oor := range sol.Oor {
			if oor > 0 {
				failed = true
				break
			}
		}
		if failed {
			nfailed++
		} else {
			nsuccess++
			if sol.FrontId == 0 {
				front0 = append(front0, sol)
			}
		}
	}
	if nfailed > 0 {
		io.PfRed("N failed = %d out of %d\n", nfailed, opt.Nsol)
	} else {
		io.PfGreen("N success = %d out of %d\n", nsuccess, opt.Nsol)
		io.PfGreen("N front 0 = %d\n", len(front0))
	}

	// save results
	var log, res bytes.Buffer
	io.Ff(&log, opt.LogParams())
	io.Ff(&res, PrintSolutions(data[0], opt.Solutions))
	io.Ff(&res, io.Sf("\n\nnfailed = %d\n", nfailed))
	io.WriteFileVD("/tmp/goga", fnkey+".log", &log)
	io.WriteFileVD("/tmp/goga", fnkey+".res", &res)

	// plot Pareto-optimal front
	feasibleOnly := true
	plt.SetForEps(0.8, 355)
	if strings.HasPrefix(fnkey, "ground10") {
		_, ref, _ := io.ReadTable("p460_fig300.dat")
		plt.Plot(ref["w"], ref["u"], "'b-'")
	}
	fmtAll := &plt.Fmt{L: "final solutions", M: ".", C: "orange", Ls: "none", Ms: 3}
	fmtFront := &plt.Fmt{L: "final Pareto front", C: "r", M: "o", Ms: 3, Ls: "none"}
	goga.PlotOvaOvaPareto(&opt, sols0, 0, 1, feasibleOnly, fmtAll, fmtFront)
	plt.Gll("weight ($f_0$)", "deflection ($f_1)$", "leg_out=1, leg_ncol=4, leg_hlen=1.5")
	if strings.HasPrefix(fnkey, "ground10") {
		plt.AxisRange(1800, 14000, 1, 6)
	}

	// plot selected results
	nfront0 := len(front0)
	if nfront0 > 2 {
		m := nfront0 / 2
		l := nfront0 - 1
		io.Pforan("nfront0=%d m=%d l=%v\n", nfront0, m, l)
		_, _, weight, umax, _, _, _ := data[0].RunFEM(front0[0].Int, front0[0].Flt, 0, false)
		plt.Text(weight, umax, "1", "size=7")
		plt.PlotOne(weight, umax, "'g*', zorder=100")
		_, _, weight, umax, _, _, _ = data[0].RunFEM(front0[m].Int, front0[m].Flt, 0, false)
		plt.Text(weight, umax, "2", "size=7")
		plt.PlotOne(weight, umax, "'g*', zorder=100")
		_, _, weight, umax, _, _, _ = data[0].RunFEM(front0[l].Int, front0[l].Flt, 0, false)
		plt.Text(weight, umax, "3", "size=7")
		plt.PlotOne(weight, umax, "'g*', zorder=100")
		plt.PyCmds(`
from pylab import axes, setp
a = axes([0.2, 0.75, 0.20, 0.10], axisbg='#dcdcdc')
setp(a, xticks=[0,720], yticks=[0,360])
axis('equal')
axis('off')
`)
		data[0].RunFEM(front0[0].Int, front0[0].Flt, 1, false)
		plt.PyCmds(`
a = axes([0.40, 0.28, 0.20, 0.10], axisbg='#dcdcdc')
setp(a, xticks=[0,720], yticks=[0,360])
axis('equal')
axis('off')
`)
		data[0].RunFEM(front0[m].Int, front0[m].Flt, 2, false)
		plt.PyCmds(`
a = axes([0.7, 0.18, 0.20, 0.10], axisbg='#dcdcdc')
setp(a, xticks=[0,720], yticks=[0,360])
axis('equal')
axis('off')
`)
		data[0].RunFEM(front0[l].Int, front0[l].Flt, 3, false)
	}

	// save
	plt.SaveD("/tmp/goga", fnkey+".eps")
	return
}

type FltFormatter []float64

func (o FltFormatter) String() (l string) {
	for _, val := range o {
		if val < 1e-9 {
			l += "       "
		} else {
			l += io.Sf("%7.2f", val)
		}
	}
	return l
}

func PrintSolutions(fed *FemData, sols []*goga.Solution) (l string) {
	goga.SortByOva(sols, 0)
	l = io.Sf("%8s%6s%6s |%s\n", "weight", "umax", "smax", "areas")
	for _, sol := range sols {
		mob, fail, weight, umax, smax, errU, errS := fed.RunFEM(sol.Int, sol.Flt, 0, false)
		if mob > 0 || fail > 0 || errU > 0 || errS > 0 {
			l += io.Sf("%20s |%s\n", "unfeasible    ", FltFormatter(sol.Flt))
			continue
		}
		l += io.Sf("%8.1f%6.2f%6.2f |%s\n", weight, umax, smax, FltFormatter(sol.Flt))
	}
	return
}
<commit_msg>example: topology: normal mode activated<commit_after>// Copyright 2015 Dorival Pedroso. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"strings"
	"time"

	"github.com/cpmech/goga"
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/plt"
)

// main function
func main() {

	// flags
	benchmark := false
	ncpuMax := 16

	// benchmarking
	if benchmark {
		var nsol, tf int
		var et time.Duration
		X := make([]float64, ncpuMax)
		T := make([]float64, ncpuMax)
		S := make([]float64, ncpuMax) // speedup
		S[0] = 1
		for i := 0; i < ncpuMax; i++ {
			io.Pf("\n\n")
			nsol, tf, et = runone(i + 1)
			io.PfYel("elaspsedTime = %v\n", et)
			X[i] = float64(i + 1)
			T[i] = et.Seconds()
			if i > 0 {
				S[i] = T[0] / T[i] // Told / Tnew
			}
		}

		plt.SetForEps(0.75, 250)
		plt.Plot(X, S, io.Sf("'b-',marker='.', label='speedup: $N_{sol}=%d,\\,t_f=%d$', clip_on=0, zorder=100", nsol, tf))
		plt.Plot([]float64{1, 16}, []float64{1, 16}, "'k--',zorder=50")
		plt.Gll("$N_{cpu}:\\;$ number of groups", "speedup", "leg_out=1")
		plt.DoubleYscale("$T_{sys}:\\;$ system time [s]")
		plt.Plot(X, T, "'k-',color='gray', clip_on=0")
		plt.SaveD("/tmp/goga", "topology-speedup.eps")
		return
	}

	// normal run
	runone(-1)
}

func runone(ncpu int) (nsol, tf int, elaspsedTime time.Duration) {

	// input filename
	fn, fnkey := io.ArgToFilename(0, "ground10", ".sim", true)

	// GA parameters
	var opt goga.Optimiser
	opt.Read("ga-" + fnkey + ".json")
	opt.GenType = "rnd"
	nsol, tf = opt.Nsol, opt.Tf
	postproc := true
	if ncpu > 0 {
		opt.Ncpu = ncpu
		postproc = false
	}

	// FEM
	data := make([]*FemData, opt.Ncpu)
	for i := 0; i < opt.Ncpu; i++ {
		data[i] = NewData(fn, fnkey, i)
	}
	io.Pforan("MaxWeight = %v\n", data[0].MaxWeight)

	// set integers
	if data[0].Opt.BinInt {
		opt.CxInt = goga.CxInt
		opt.MtInt = goga.MtIntBin
		opt.BinInt = data[0].Ncells
	}

	// set floats
	opt.FltMin = make([]float64, data[0].Nareas)
	opt.FltMax = make([]float64, data[0].Nareas)
	for i := 0; i < data[0].Nareas; i++ {
		opt.FltMin[i] = data[0].Opt.Amin
		opt.FltMax[i] = data[0].Opt.Amax
	}

	// initialise optimiser
	opt.Nova = 2 // weight and deflection
	opt.Noor = 4 // mobility, feasibility, maxdeflection, stress
	opt.Init(goga.GenTrialSolutions, func(sol *goga.Solution, cpu int) {
		mob, fail, weight, umax, _, errU, errS := data[cpu].RunFEM(sol.Int, sol.Flt, 0, false)
		sol.Ova[0] = weight
		sol.Ova[1] = umax
		sol.Oor[0] = mob
		sol.Oor[1] = fail
		sol.Oor[2] = errU
		sol.Oor[3] = errS
	}, nil, 0, 0, 0)

	// initial solutions
	var sols0 []*goga.Solution
	if false {
		sols0 = opt.GetSolutionsCopy()
	}

	// benchmark
	initialTime := time.Now()
	defer func() {
		elaspsedTime = time.Now().Sub(initialTime)
	}()

	// solve
	opt.Verbose = true
	opt.Solve()
	goga.SortByOva(opt.Solutions, 0)

	// post processing
	if !postproc {
		return
	}

	// check
	front0 := make([]*goga.Solution, 0)
	var nfailed, nsuccess int
	for _, sol := range opt.Solutions {
		var failed bool
		for _, oor := range sol.Oor {
			if oor > 0 {
				failed = true
				break
			}
		}
		if failed {
			nfailed++
		} else {
			nsuccess++
			if sol.FrontId == 0 {
				front0 = append(front0, sol)
			}
		}
	}
	if nfailed > 0 {
		io.PfRed("N failed = %d out of %d\n", nfailed, opt.Nsol)
	} else {
		io.PfGreen("N success = %d out of %d\n", nsuccess, opt.Nsol)
		io.PfGreen("N front 0 = %d\n", len(front0))
	}

	// save results
	var log, res bytes.Buffer
	io.Ff(&log, opt.LogParams())
	io.Ff(&res, PrintSolutions(data[0], opt.Solutions))
	io.Ff(&res, io.Sf("\n\nnfailed = %d\n", nfailed))
	io.WriteFileVD("/tmp/goga", fnkey+".log", &log)
	io.WriteFileVD("/tmp/goga", fnkey+".res", &res)

	// plot Pareto-optimal front
	feasibleOnly := true
	plt.SetForEps(0.8, 355)
	if strings.HasPrefix(fnkey, "ground10") {
		_, ref, _ := io.ReadTable("p460_fig300.dat")
		plt.Plot(ref["w"], ref["u"], "'b-'")
	}
	fmtAll := &plt.Fmt{L: "final solutions", M: ".", C: "orange", Ls: "none", Ms: 3}
	fmtFront := &plt.Fmt{L: "final Pareto front", C: "r", M: "o", Ms: 3, Ls: "none"}
	goga.PlotOvaOvaPareto(&opt, sols0, 0, 1, feasibleOnly, fmtAll, fmtFront)
	plt.Gll("weight ($f_0$)", "deflection ($f_1)$", "leg_out=1, leg_ncol=4, leg_hlen=1.5")
	if strings.HasPrefix(fnkey, "ground10") {
		plt.AxisRange(1800, 14000, 1, 6)
	}

	// plot selected results
	nfront0 := len(front0)
	if nfront0 > 2 {
		m := nfront0 / 2
		l := nfront0 - 1
		io.Pforan("nfront0=%d m=%d l=%v\n", nfront0, m, l)
		_, _, weight, umax, _, _, _ := data[0].RunFEM(front0[0].Int, front0[0].Flt, 0, false)
		plt.Text(weight, umax, "1", "size=7")
		plt.PlotOne(weight, umax, "'g*', zorder=100")
		_, _, weight, umax, _, _, _ = data[0].RunFEM(front0[m].Int, front0[m].Flt, 0, false)
		plt.Text(weight, umax, "2", "size=7")
		plt.PlotOne(weight, umax, "'g*', zorder=100")
		_, _, weight, umax, _, _, _ = data[0].RunFEM(front0[l].Int, front0[l].Flt, 0, false)
		plt.Text(weight, umax, "3", "size=7")
		plt.PlotOne(weight, umax, "'g*', zorder=100")
		plt.PyCmds(`
from pylab import axes, setp
a = axes([0.2, 0.75, 0.20, 0.10], axisbg='#dcdcdc')
setp(a, xticks=[0,720], yticks=[0,360])
axis('equal')
axis('off')
`)
		data[0].RunFEM(front0[0].Int, front0[0].Flt, 1, false)
		plt.PyCmds(`
a = axes([0.40, 0.28, 0.20, 0.10], axisbg='#dcdcdc')
setp(a, xticks=[0,720], yticks=[0,360])
axis('equal')
axis('off')
`)
		data[0].RunFEM(front0[m].Int, front0[m].Flt, 2, false)
		plt.PyCmds(`
a = axes([0.7, 0.18, 0.20, 0.10], axisbg='#dcdcdc')
setp(a, xticks=[0,720], yticks=[0,360])
axis('equal')
axis('off')
`)
		data[0].RunFEM(front0[l].Int, front0[l].Flt, 3, false)
	}

	// save
	plt.SaveD("/tmp/goga", fnkey+".eps")
	return
}

type FltFormatter []float64

func (o FltFormatter) String() (l string) {
	for _, val := range o {
		if val < 1e-9 {
			l += "       "
		} else {
			l += io.Sf("%7.2f", val)
		}
	}
	return l
}

func PrintSolutions(fed *FemData, sols []*goga.Solution) (l string) {
	goga.SortByOva(sols, 0)
	l = io.Sf("%8s%6s%6s |%s\n", "weight", "umax", "smax", "areas")
	for _, sol := range sols {
		mob, fail, weight, umax, smax, errU, errS := fed.RunFEM(sol.Int, sol.Flt, 0, false)
		if mob > 0 || fail > 0 || errU > 0 || errS > 0 {
			l += io.Sf("%20s |%s\n", "unfeasible    ", FltFormatter(sol.Flt))
			continue
		}
		l += io.Sf("%8.1f%6.2f%6.2f |%s\n", weight, umax, smax, FltFormatter(sol.Flt))
	}
	return
}
<|endoftext|>