<commit_before>package apns

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"math/rand"
	"strconv"
)

// Push commands always start with command value 2.
const pushCommandValue = 2

// Your total notification payload cannot exceed 2 KB.
const MaxPayloadSizeBytes = 2048

// Constants related to the payload fields and their lengths.
const (
	deviceTokenItemid            = 1
	payloadItemid                = 2
	notificationIdentifierItemid = 3
	expirationDateItemid         = 4
	priorityItemid               = 5
	deviceTokenLength            = 32
	notificationIdentifierLength = 4
	expirationDateLength         = 4
	priorityLength               = 1
)

// Payload contains the notification data for your request.
//
// Alert is an interface here because it supports either a string
// or a dictionary, represented within by an AlertDictionary struct.
type Payload struct {
	Alert            interface{} `json:"alert,omitempty"`
	Badge            int         `json:"badge,omitempty"`
	Sound            string      `json:"sound,omitempty"`
	ContentAvailable int         `json:"content-available,omitempty"`
	Category         string      `json:"category,omitempty"`
}

// NewPayload creates and returns a Payload structure.
func NewPayload() *Payload {
	return new(Payload)
}

// AlertDictionary is a more complex notification payload.
//
// From the APN docs: "Use the ... alert dictionary in general only if you absolutely need to."
// The AlertDictionary is suitable for specific localization needs.
type AlertDictionary struct {
	Body         string   `json:"body,omitempty"`
	ActionLocKey string   `json:"action-loc-key,omitempty"`
	LocKey       string   `json:"loc-key,omitempty"`
	LocArgs      []string `json:"loc-args,omitempty"`
	LaunchImage  string   `json:"launch-image,omitempty"`
}

// NewAlertDictionary creates and returns an AlertDictionary structure.
func NewAlertDictionary() *AlertDictionary {
	return new(AlertDictionary)
}

// PushNotification is the wrapper for the Payload.
// The length fields are computed in ToBytes() and aren't represented here.
type PushNotification struct {
	Identifier  uint32
	Expiry      uint32
	DeviceToken string
	payload     map[string]interface{}
	Priority    uint8
}

// NewPushNotification creates and returns a PushNotification structure.
// It also initializes the pseudo-random identifier.
func NewPushNotification() (pn *PushNotification) {
	pn = new(PushNotification)
	pn.payload = make(map[string]interface{})
	pn.Identifier = rand.Uint32()
	pn.Priority = 10
	return
}

// AddPayload sets the "aps" payload section of the request. It also
// has a hack described within to deal with specific zero values.
func (pn *PushNotification) AddPayload(p *Payload) {
	// This deserves some explanation.
	//
	// Setting an exported field of type int to 0
	// triggers the omitempty behavior if you've set it.
	// Since the badge is optional, we should omit it if
	// it's not set. However, we want to include it if the
	// value is 0, so there's a hack in push_notification.go
	// that exploits the fact that Apple treats -1 for a
	// badge value as though it were 0 (i.e. it clears the
	// badge but doesn't stop the notification from going
	// through successfully.)
	//
	// Still a hack though :)
	if p.Badge == 0 {
		p.Badge = -1
	}
	pn.Set("aps", p)
}

// Get returns the value of a payload key, if it exists.
func (pn *PushNotification) Get(key string) interface{} {
	return pn.payload[key]
}

// Set defines the value of a payload key.
func (pn *PushNotification) Set(key string, value interface{}) {
	pn.payload[key] = value
}

// PayloadJSON returns the current payload in JSON format.
func (pn *PushNotification) PayloadJSON() ([]byte, error) {
	return json.Marshal(pn.payload)
}

// PayloadString returns the current payload in string format.
func (pn *PushNotification) PayloadString() (string, error) {
	j, err := pn.PayloadJSON()
	return string(j), err
}

// ToBytes returns a byte array of the complete PushNotification
// struct. This array is what should be transmitted to the APN Service.
func (pn *PushNotification) ToBytes() ([]byte, error) {
	token, err := hex.DecodeString(pn.DeviceToken)
	if err != nil {
		return nil, err
	}
	if len(token) != deviceTokenLength {
		return nil, errors.New("device token has incorrect length")
	}
	payload, err := pn.PayloadJSON()
	if err != nil {
		return nil, err
	}
	if len(payload) > MaxPayloadSizeBytes {
		return nil, errors.New("payload is larger than the " + strconv.Itoa(MaxPayloadSizeBytes) + " byte limit")
	}

	frameBuffer := new(bytes.Buffer)
	binary.Write(frameBuffer, binary.BigEndian, uint8(deviceTokenItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(deviceTokenLength))
	binary.Write(frameBuffer, binary.BigEndian, token)
	binary.Write(frameBuffer, binary.BigEndian, uint8(payloadItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(len(payload)))
	binary.Write(frameBuffer, binary.BigEndian, payload)
	binary.Write(frameBuffer, binary.BigEndian, uint8(notificationIdentifierItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(notificationIdentifierLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Identifier)
	binary.Write(frameBuffer, binary.BigEndian, uint8(expirationDateItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(expirationDateLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Expiry)
	binary.Write(frameBuffer, binary.BigEndian, uint8(priorityItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(priorityLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Priority)

	buffer := bytes.NewBuffer([]byte{})
	binary.Write(buffer, binary.BigEndian, uint8(pushCommandValue))
	binary.Write(buffer, binary.BigEndian, uint32(frameBuffer.Len()))
	binary.Write(buffer, binary.BigEndian, frameBuffer.Bytes())
	return buffer.Bytes(), nil
}
<commit_msg>It was a mistake to remove the seed for the random number generator; push notifications with the same ID were being generated. In the future though, maybe we should look into using sequential IDs rather than random ones.<commit_after>package apns

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"math/rand"
	"strconv"
	"time"
)

// Push commands always start with command value 2.
const pushCommandValue = 2

// Your total notification payload cannot exceed 2 KB.
const MaxPayloadSizeBytes = 2048

// Constants related to the payload fields and their lengths.
const (
	deviceTokenItemid            = 1
	payloadItemid                = 2
	notificationIdentifierItemid = 3
	expirationDateItemid         = 4
	priorityItemid               = 5
	deviceTokenLength            = 32
	notificationIdentifierLength = 4
	expirationDateLength         = 4
	priorityLength               = 1
)

// Payload contains the notification data for your request.
//
// Alert is an interface here because it supports either a string
// or a dictionary, represented within by an AlertDictionary struct.
type Payload struct {
	Alert            interface{} `json:"alert,omitempty"`
	Badge            int         `json:"badge,omitempty"`
	Sound            string      `json:"sound,omitempty"`
	ContentAvailable int         `json:"content-available,omitempty"`
	Category         string      `json:"category,omitempty"`
}

// NewPayload creates and returns a Payload structure.
func NewPayload() *Payload {
	return new(Payload)
}

// AlertDictionary is a more complex notification payload.
//
// From the APN docs: "Use the ... alert dictionary in general only if you absolutely need to."
// The AlertDictionary is suitable for specific localization needs.
type AlertDictionary struct {
	Body         string   `json:"body,omitempty"`
	ActionLocKey string   `json:"action-loc-key,omitempty"`
	LocKey       string   `json:"loc-key,omitempty"`
	LocArgs      []string `json:"loc-args,omitempty"`
	LaunchImage  string   `json:"launch-image,omitempty"`
}

// NewAlertDictionary creates and returns an AlertDictionary structure.
func NewAlertDictionary() *AlertDictionary {
	return new(AlertDictionary)
}

// PushNotification is the wrapper for the Payload.
// The length fields are computed in ToBytes() and aren't represented here.
type PushNotification struct {
	Identifier  uint32
	Expiry      uint32
	DeviceToken string
	payload     map[string]interface{}
	Priority    uint8
}

// NewPushNotification creates and returns a PushNotification structure.
// It also initializes the pseudo-random identifier.
func NewPushNotification() (pn *PushNotification) {
	pn = new(PushNotification)
	pn.payload = make(map[string]interface{})
	pn.Identifier = rand.New(rand.NewSource(time.Now().UnixNano())).Uint32()
	pn.Priority = 10
	return
}

// AddPayload sets the "aps" payload section of the request. It also
// has a hack described within to deal with specific zero values.
func (pn *PushNotification) AddPayload(p *Payload) {
	// This deserves some explanation.
	//
	// Setting an exported field of type int to 0
	// triggers the omitempty behavior if you've set it.
	// Since the badge is optional, we should omit it if
	// it's not set. However, we want to include it if the
	// value is 0, so there's a hack in push_notification.go
	// that exploits the fact that Apple treats -1 for a
	// badge value as though it were 0 (i.e. it clears the
	// badge but doesn't stop the notification from going
	// through successfully.)
	//
	// Still a hack though :)
	if p.Badge == 0 {
		p.Badge = -1
	}
	pn.Set("aps", p)
}

// Get returns the value of a payload key, if it exists.
func (pn *PushNotification) Get(key string) interface{} {
	return pn.payload[key]
}

// Set defines the value of a payload key.
func (pn *PushNotification) Set(key string, value interface{}) {
	pn.payload[key] = value
}

// PayloadJSON returns the current payload in JSON format.
func (pn *PushNotification) PayloadJSON() ([]byte, error) {
	return json.Marshal(pn.payload)
}

// PayloadString returns the current payload in string format.
func (pn *PushNotification) PayloadString() (string, error) {
	j, err := pn.PayloadJSON()
	return string(j), err
}

// ToBytes returns a byte array of the complete PushNotification
// struct. This array is what should be transmitted to the APN Service.
func (pn *PushNotification) ToBytes() ([]byte, error) {
	token, err := hex.DecodeString(pn.DeviceToken)
	if err != nil {
		return nil, err
	}
	if len(token) != deviceTokenLength {
		return nil, errors.New("device token has incorrect length")
	}
	payload, err := pn.PayloadJSON()
	if err != nil {
		return nil, err
	}
	if len(payload) > MaxPayloadSizeBytes {
		return nil, errors.New("payload is larger than the " + strconv.Itoa(MaxPayloadSizeBytes) + " byte limit")
	}

	frameBuffer := new(bytes.Buffer)
	binary.Write(frameBuffer, binary.BigEndian, uint8(deviceTokenItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(deviceTokenLength))
	binary.Write(frameBuffer, binary.BigEndian, token)
	binary.Write(frameBuffer, binary.BigEndian, uint8(payloadItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(len(payload)))
	binary.Write(frameBuffer, binary.BigEndian, payload)
	binary.Write(frameBuffer, binary.BigEndian, uint8(notificationIdentifierItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(notificationIdentifierLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Identifier)
	binary.Write(frameBuffer, binary.BigEndian, uint8(expirationDateItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(expirationDateLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Expiry)
	binary.Write(frameBuffer, binary.BigEndian, uint8(priorityItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(priorityLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Priority)

	buffer := bytes.NewBuffer([]byte{})
	binary.Write(buffer, binary.BigEndian, uint8(pushCommandValue))
	binary.Write(buffer, binary.BigEndian, uint32(frameBuffer.Len()))
	binary.Write(buffer, binary.BigEndian, frameBuffer.Bytes())
	return buffer.Bytes(), nil
}
<|endoftext|>


<commit_before>// Package logrus provides an adapter to the
// go-kit log.Logger interface.
package logrus

import (
	"errors"
	"fmt"

	"github.com/go-kit/kit/log"
	"github.com/sirupsen/logrus"
)

type Logger struct {
	field logrus.FieldLogger
	level logrus.Level
}

type Option func(*Logger)

var errMissingValue = errors.New("(MISSING)")

// NewLogger returns a go-kit log.Logger that sends log events to a Logrus logger.
func NewLogger(logger logrus.FieldLogger, options ...Option) log.Logger {
	l := &Logger{
		field: logger,
		level: logrus.InfoLevel,
	}

	for _, optFunc := range options {
		optFunc(l)
	}

	return l
}

// WithLevel configures a logrus logger to log at level for all events.
func WithLevel(level logrus.Level) Option {
	return func(c *Logger) {
		c.level = level
	}
}

func (l Logger) Log(keyvals ...interface{}) error {
	fields := logrus.Fields{}
	for i := 0; i < len(keyvals); i += 2 {
		if i+1 < len(keyvals) {
			fields[fmt.Sprint(keyvals[i])] = keyvals[i+1]
		} else {
			fields[fmt.Sprint(keyvals[i])] = errMissingValue
		}
	}

	switch l.level {
	case logrus.InfoLevel:
		l.field.WithFields(fields).Info()
	case logrus.ErrorLevel:
		l.field.WithFields(fields).Error()
	case logrus.DebugLevel:
		l.field.WithFields(fields).Debug()
	case logrus.WarnLevel:
		l.field.WithFields(fields).Warn()
	case logrus.TraceLevel:
		l.field.WithFields(fields).Trace()
	default:
		l.field.WithFields(fields).Print()
	}

	return nil
}
<commit_msg>update NewLogger comment<commit_after>// Package logrus provides an adapter to the
// go-kit log.Logger interface.
package logrus

import (
	"errors"
	"fmt"

	"github.com/go-kit/kit/log"
	"github.com/sirupsen/logrus"
)

type Logger struct {
	field logrus.FieldLogger
	level logrus.Level
}

type Option func(*Logger)

var errMissingValue = errors.New("(MISSING)")

// NewLogger returns a Go kit log.Logger that sends log events to a logrus.Logger.
func NewLogger(logger logrus.FieldLogger, options ...Option) log.Logger {
	l := &Logger{
		field: logger,
		level: logrus.InfoLevel,
	}

	for _, optFunc := range options {
		optFunc(l)
	}

	return l
}

// WithLevel configures a logrus logger to log at level for all events.
func WithLevel(level logrus.Level) Option {
	return func(c *Logger) {
		c.level = level
	}
}

func (l Logger) Log(keyvals ...interface{}) error {
	fields := logrus.Fields{}
	for i := 0; i < len(keyvals); i += 2 {
		if i+1 < len(keyvals) {
			fields[fmt.Sprint(keyvals[i])] = keyvals[i+1]
		} else {
			fields[fmt.Sprint(keyvals[i])] = errMissingValue
		}
	}

	switch l.level {
	case logrus.InfoLevel:
		l.field.WithFields(fields).Info()
	case logrus.ErrorLevel:
		l.field.WithFields(fields).Error()
	case logrus.DebugLevel:
		l.field.WithFields(fields).Debug()
	case logrus.WarnLevel:
		l.field.WithFields(fields).Warn()
	case logrus.TraceLevel:
		l.field.WithFields(fields).Trace()
	default:
		l.field.WithFields(fields).Print()
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/AdRoll/goamz/aws" //traditional library by Canonical + AdRoll
	"github.com/AdRoll/goamz/s3"
	sdkaws "github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session" // the new library by aws
	sdks3 "github.com/aws/aws-sdk-go/service/s3"
	"net/http"
	"os"
	//	"os/exec"
	//	"strings"
	"sync"
	"sync/atomic"
	"time"
)

func main() {
	context, _ := prepareContext()
	parallelity := 10

	names := make(chan string)

	context.wg.Add(parallelity)
	for gr := 1; gr <= parallelity; gr++ {
		go cpworker(&context, names)
	}

	input := bufio.NewScanner(os.Stdin)
	for input.Scan() {
		names <- input.Text()
	}
	close(names)

	context.wg.Wait()
	fmt.Printf("\nDone.\n")
}

type CopyContext struct {
	s3         *s3.Bucket
	s3svc      *sdks3.S3
	bucketname string
	newvalue   string

	copiedObjects   int64
	copiedBytes     int64
	expectedObjects int64
	start           time.Time
	statusLineMutex sync.Mutex
	lastStatusShown float64

	wg sync.WaitGroup
}

func prepareContext() (CopyContext, error) {
	auth, err := aws.EnvAuth()
	if err != nil {
		panic(err.Error())
	}
	// Using old library, TODO: switch everything to the new
	s := s3.New(auth, aws.EUCentral)
	s.Signature = aws.V4Signature

	// Session with the new library
	sess, err := session.NewSession(&sdkaws.Config{
		Region: sdkaws.String("eu-central-1")},
	)
	if err != nil {
		panic(fmt.Sprintf("Can not create AWS SDK session %s", err))
	}

	if len(os.Args) != 3 {
		panic("Please provide bucket name and desired Cache-Control setting")
	}
	return CopyContext{
		s3:              s.Bucket(os.Args[1]),
		s3svc:           sdks3.New(sess),
		bucketname:      os.Args[1],
		expectedObjects: 18000000,
		newvalue:        os.Args[2],
		start:           time.Now(),
	}, nil
}

func cpworker(context *CopyContext, names <-chan string) {
	for {
		name, more := <-names
		if more {
			// fmt.Printf("Starting copy %v\n", name)
			cp(context, name)
		} else {
			fmt.Printf("\nNo more data in names channel.\n")
			context.wg.Done()
			return
		}
	}
}

func cp(context *CopyContext, name string) {
	resp, err := context.s3.Head(name, make(http.Header))
	if err != nil {
		os.Stderr.WriteString(fmt.Sprintf("==> Head failed for '%s' Error: %s\n", name, err.Error()))
		filename := "error_keys.txt"
		os.Stderr.WriteString(fmt.Sprintf("Adding file to 'TODO' list '%s'", filename))
		f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			os.Stderr.WriteString(fmt.Sprintf("Could not write to 'TODO' file '%s'. Error: %s\n", filename, err.Error()))
			panic(err)
		}
		fmt.Fprintln(f, name)
		f.Close()
		return
	}

	contenttypes, contenttype_present := resp.Header["Content-Type"]
	cache, cache_present := resp.Header["Cache-Control"]

	var status, contenttype string
	// . - skip
	// X - type was not set, set to image/png
	// j - was image/jpeg; adjusted CacheControl
	// g - was image/png; adjusted CacheControl
	// P - pdf file; adjusted CacheControl
	// Y - other file type; adjusted CacheControl
	if cache_present && len(cache) == 1 && cache[0] == context.newvalue {
		status = "."
	} else {
		if !contenttype_present {
			status = "X"
			contenttype = "image/png"
		} else {
			contenttype = contenttypes[0] // theoretically, there can be multiple HTTP headers with the same key
			// but lets assume, there is at most one
			if contenttype == "image/png" {
				status = "g"
			} else if contenttype == "image/jpeg" {
				status = "j"
			} else if contenttype == "application/pdf" {
				status = "P"
			} else {
				status = "Y"
			}
		}

		src := fmt.Sprintf("%s/%s", context.bucketname, name)
		directive := "REPLACE"
		inp := sdks3.CopyObjectInput{
			Bucket:            sdkaws.String(context.bucketname),
			CopySource:        &src,
			Key:               &name,
			CacheControl:      &context.newvalue,
			ContentType:       &contenttype,
			MetadataDirective: &directive,
		}
		// fmt.Printf("%v", inp)
		/*resp*/ _, err := context.s3svc.CopyObject(&inp)
		if err != nil {
			fmt.Println("Failed changing (inplace-copying) object")
			fmt.Println(err.Error())
			return
		}
		// fmt.Println(resp)
	}
	fmt.Print(status)

	atomic.AddInt64(&context.copiedObjects, 1)
	sec := time.Since(context.start).Seconds()
	o_s := float64(context.copiedObjects) / sec
	expectedDuration := time.Duration(int(float64(context.expectedObjects-context.copiedObjects)/o_s)) * time.Second
	days := int(expectedDuration.Hours() / 24)
	andHours := expectedDuration.Hours() - float64(days)*24
	eta := fmt.Sprintf("%dd %.1fh", days, andHours)
	if days == 0 {
		hours := int(expectedDuration.Minutes() / 60)
		andMinutes := expectedDuration.Minutes() - float64(hours)*60
		eta = fmt.Sprintf("%dh %.1fm", hours, andMinutes)
	}
	if context.expectedObjects < context.copiedObjects {
		eta = "-"
	}

	const everySeconds = 10
	show := false
	context.statusLineMutex.Lock()
	if context.lastStatusShown < sec-everySeconds {
		show = true
		context.lastStatusShown = sec
	}
	context.statusLineMutex.Unlock()

	if show {
		fmt.Printf("\n%-30s Totals: %d/%d objects. Avg: %.2f obj/s. ETA: %v    \n",
			name, context.copiedObjects, context.expectedObjects, o_s, eta,
		)
	}
}
<commit_msg>Unify error handling on cp-single-object level<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/AdRoll/goamz/aws" //traditional library by Canonical + AdRoll
	"github.com/AdRoll/goamz/s3"
	sdkaws "github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session" // the new library by aws
	sdks3 "github.com/aws/aws-sdk-go/service/s3"
	"net/http"
	"os"
	//	"os/exec"
	//	"strings"
	"sync"
	"sync/atomic"
	"time"
)

func main() {
	context, _ := prepareContext()
	parallelity := 10

	names := make(chan string)

	context.wg.Add(parallelity)
	for gr := 1; gr <= parallelity; gr++ {
		go cpworker(&context, names)
	}

	input := bufio.NewScanner(os.Stdin)
	for input.Scan() {
		names <- input.Text()
	}
	close(names)

	context.wg.Wait()
	fmt.Printf("\nDone.\n")
}

type CopyContext struct {
	s3         *s3.Bucket
	s3svc      *sdks3.S3
	bucketname string
	newvalue   string

	copiedObjects   int64
	copiedBytes     int64
	expectedObjects int64
	start           time.Time
	statusLineMutex sync.Mutex
	lastStatusShown float64

	wg sync.WaitGroup
}

func prepareContext() (CopyContext, error) {
	auth, err := aws.EnvAuth()
	if err != nil {
		panic(err.Error())
	}
	// Using old library, TODO: switch everything to the new
	s := s3.New(auth, aws.EUCentral)
	s.Signature = aws.V4Signature

	// Session with the new library
	sess, err := session.NewSession(&sdkaws.Config{
		Region: sdkaws.String("eu-central-1")},
	)
	if err != nil {
		panic(fmt.Sprintf("Can not create AWS SDK session %s", err))
	}

	if len(os.Args) != 3 {
		panic("Please provide bucket name and desired Cache-Control setting")
	}
	return CopyContext{
		s3:              s.Bucket(os.Args[1]),
		s3svc:           sdks3.New(sess),
		bucketname:      os.Args[1],
		expectedObjects: 18000000,
		newvalue:        os.Args[2],
		start:           time.Now(),
	}, nil
}

func cpworker(context *CopyContext, names <-chan string) {
	for {
		name, more := <-names
		if more {
			// fmt.Printf("Starting copy %v\n", name)
			if err := cp(context, name); err != nil {
				os.Stderr.WriteString(fmt.Sprintf("==> Failed processing '%s': %v\n", name, err))
				filename := "error_keys.txt"
				os.Stderr.WriteString(fmt.Sprintf("Adding name to '%s' for later processing or reference", filename))
				f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
				if err != nil {
					os.Stderr.WriteString(fmt.Sprintf("Could not write to 'TODO' file '%s'. Error: %v\n", filename, err))
					panic(err)
				}
				fmt.Fprintln(f, name)
				f.Close()
			}
		} else {
			fmt.Printf("\nNo more data in names channel.\n")
			context.wg.Done()
			return
		}
	}
}

func cp(context *CopyContext, name string) error {
	resp, err := context.s3.Head(name, make(http.Header))
	if err != nil {
		return fmt.Errorf("HEAD failed for '%s' Error: %v", name, err)
	}

	contenttypes, contenttype_present := resp.Header["Content-Type"]
	cache, cache_present := resp.Header["Cache-Control"]

	var status, contenttype string
	// . - skip
	// X - type was not set, set to image/png
	// j - was image/jpeg; adjusted CacheControl
	// g - was image/png; adjusted CacheControl
	// P - pdf file; adjusted CacheControl
	// Y - other file type; adjusted CacheControl
	if cache_present && len(cache) == 1 && cache[0] == context.newvalue {
		status = "."
	} else {
		if !contenttype_present {
			status = "X"
			contenttype = "image/png"
		} else {
			contenttype = contenttypes[0] // theoretically, there can be multiple HTTP headers with the same key
			// but lets assume, there is at most one
			if contenttype == "image/png" {
				status = "g"
			} else if contenttype == "image/jpeg" {
				status = "j"
			} else if contenttype == "application/pdf" {
				status = "P"
			} else {
				status = "Y"
			}
		}

		src := fmt.Sprintf("%s/%s", context.bucketname, name)
		inp := sdks3.CopyObjectInput{
			Bucket:            sdkaws.String(context.bucketname),
			CopySource:        &src,
			Key:               &name,
			CacheControl:      &context.newvalue,
			ContentType:       &contenttype,
			MetadataDirective: sdkaws.String("REPLACE"),
		}
		_, err := context.s3svc.CopyObject(&inp)
		if err != nil {
			return fmt.Errorf("Failed changing (inplace-copying) object: %v", err)
		}
	}
	fmt.Print(status)

	atomic.AddInt64(&context.copiedObjects, 1)
	sec := time.Since(context.start).Seconds()
	o_s := float64(context.copiedObjects) / sec
	expectedDuration := time.Duration(int(float64(context.expectedObjects-context.copiedObjects)/o_s)) * time.Second
	days := int(expectedDuration.Hours() / 24)
	andHours := expectedDuration.Hours() - float64(days)*24
	eta := fmt.Sprintf("%dd %.1fh", days, andHours)
	if days == 0 {
		hours := int(expectedDuration.Minutes() / 60)
		andMinutes := expectedDuration.Minutes() - float64(hours)*60
		eta = fmt.Sprintf("%dh %.1fm", hours, andMinutes)
	}
	if context.expectedObjects < context.copiedObjects {
		eta = "-"
	}

	const everySeconds = 10
	show := false
	context.statusLineMutex.Lock()
	if context.lastStatusShown < sec-everySeconds {
		show = true
		context.lastStatusShown = sec
	}
	context.statusLineMutex.Unlock()

	if show {
		fmt.Printf("\n%-30s Totals: %d/%d objects. Avg: %.2f obj/s. ETA: %v    \n",
			name, context.copiedObjects, context.expectedObjects, o_s, eta,
		)
	}
	return nil
}
<|endoftext|>


<commit_before>package device

import (
	"fmt"
	"net"

	clusterConfig "github.com/lxc/lxd/lxd/cluster/config"
	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/network"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared/logger"
)

// deviceCommon represents the common struct for all devices.
type deviceCommon struct {
	logger      logger.Logger
	inst        instance.Instance
	name        string
	config      deviceConfig.Device
	state       *state.State
	volatileGet func() map[string]string
	volatileSet func(map[string]string) error
}

// init stores the Instance, daemon state, device name and config into device.
// It also needs to be provided with volatile get and set functions for the device to allow
// persistent data to be accessed. This is implemented as part of deviceCommon so that the majority
// of devices don't need to implement it and can just embed deviceCommon.
func (d *deviceCommon) init(inst instance.Instance, state *state.State, name string, conf deviceConfig.Device, volatileGet VolatileGetter, volatileSet VolatileSetter) {
	logCtx := logger.Ctx{"driver": conf["type"], "device": name}
	if inst != nil {
		logCtx["project"] = inst.Project()
		logCtx["instance"] = inst.Name()
	}

	d.logger = logger.AddContext(logger.Log, logCtx)
	d.inst = inst
	d.name = name
	d.config = conf
	d.state = state
	d.volatileGet = volatileGet
	d.volatileSet = volatileSet
}

// Name returns the name of the device.
func (d *deviceCommon) Name() string {
	return d.name
}

// Config returns the config for the device.
func (d *deviceCommon) Config() deviceConfig.Device {
	return d.config
}

// Add returns nil error as majority of devices don't need to do any host-side setup.
func (d *deviceCommon) Add() error {
	return nil
}

// Register returns nil error as majority of devices don't need to do any event registration.
func (d *deviceCommon) Register() error {
	return nil
}

// CanHotPlug returns whether the device can be managed whilst the instance is running,
// Returns true if instance type is container, as majority of devices can be started/stopped when
// instance is running. If instance type is VM then returns false as this is not currently supported.
func (d *deviceCommon) CanHotPlug() bool {
	return d.inst.Type() == instancetype.Container
}

// CanMigrate returns whether the device can be migrated to any other cluster member.
func (d *deviceCommon) CanMigrate() bool {
	return false
}

// UpdatableFields returns an empty list of updatable fields as most devices do not support updates.
func (d *deviceCommon) UpdatableFields(oldDevice Type) []string {
	return []string{}
}

// PreStartCheck indicates if the device is available for starting.
func (d *deviceCommon) PreStartCheck() error {
	return nil
}

// Update returns an ErrCannotUpdate error as most devices do not support updates.
func (d *deviceCommon) Update(oldDevices deviceConfig.Devices, isRunning bool) error {
	return ErrCannotUpdate
}

// Remove returns nil error as majority of devices don't need to do any host-side cleanup on delete.
func (d *deviceCommon) Remove() error {
	return nil
}

// generateHostName generates the name to use for the host side NIC interface based on the
// instances.nic.host_name setting.
// Accepts prefix argument to use with random interface generation.
// Accepts optional hwaddr MAC address to use for generating the interface name in mac mode.
// In mac mode the interface prefix is always "lxd".
func (d *deviceCommon) generateHostName(prefix string, hwaddr string) (string, error) {
	hostNameMode, err := clusterConfig.GetString(d.state.DB.Cluster, "instances.nic.host_name")
	if err != nil {
		return "", fmt.Errorf(`Failed getting "instances.nic.host_name" config: %w`, err)
	}

	// Handle instances.nic.host_name mac mode if a MAC address has been supplied.
	if hostNameMode == "mac" && hwaddr != "" {
		mac, err := net.ParseMAC(hwaddr)
		if err != nil {
			return "", fmt.Errorf("Failed parsing MAC address %q: %w", hwaddr, err)
		}

		return network.MACDevName(mac), nil
	}

	// Handle instances.nic.host_name random mode or where no MAC address supplied.
	return network.RandomDevName(prefix), nil
}
<commit_msg>lxd/device/device/common: Updates generateHostName to use d.state.GlobalConfig.InstancesNICHostname<commit_after>package device

import (
	"fmt"
	"net"

	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/network"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared/logger"
)

// deviceCommon represents the common struct for all devices.
type deviceCommon struct {
	logger      logger.Logger
	inst        instance.Instance
	name        string
	config      deviceConfig.Device
	state       *state.State
	volatileGet func() map[string]string
	volatileSet func(map[string]string) error
}

// init stores the Instance, daemon state, device name and config into device.
// It also needs to be provided with volatile get and set functions for the device to allow
// persistent data to be accessed. This is implemented as part of deviceCommon so that the majority
// of devices don't need to implement it and can just embed deviceCommon.
func (d *deviceCommon) init(inst instance.Instance, state *state.State, name string, conf deviceConfig.Device, volatileGet VolatileGetter, volatileSet VolatileSetter) {
	logCtx := logger.Ctx{"driver": conf["type"], "device": name}
	if inst != nil {
		logCtx["project"] = inst.Project()
		logCtx["instance"] = inst.Name()
	}

	d.logger = logger.AddContext(logger.Log, logCtx)
	d.inst = inst
	d.name = name
	d.config = conf
	d.state = state
	d.volatileGet = volatileGet
	d.volatileSet = volatileSet
}

// Name returns the name of the device.
func (d *deviceCommon) Name() string {
	return d.name
}

// Config returns the config for the device.
func (d *deviceCommon) Config() deviceConfig.Device {
	return d.config
}

// Add returns nil error as majority of devices don't need to do any host-side setup.
func (d *deviceCommon) Add() error {
	return nil
}

// Register returns nil error as majority of devices don't need to do any event registration.
func (d *deviceCommon) Register() error {
	return nil
}

// CanHotPlug returns whether the device can be managed whilst the instance is running,
// Returns true if instance type is container, as majority of devices can be started/stopped when
// instance is running. If instance type is VM then returns false as this is not currently supported.
func (d *deviceCommon) CanHotPlug() bool {
	return d.inst.Type() == instancetype.Container
}

// CanMigrate returns whether the device can be migrated to any other cluster member.
func (d *deviceCommon) CanMigrate() bool {
	return false
}

// UpdatableFields returns an empty list of updatable fields as most devices do not support updates.
func (d *deviceCommon) UpdatableFields(oldDevice Type) []string {
	return []string{}
}

// PreStartCheck indicates if the device is available for starting.
func (d *deviceCommon) PreStartCheck() error {
	return nil
}

// Update returns an ErrCannotUpdate error as most devices do not support updates.
func (d *deviceCommon) Update(oldDevices deviceConfig.Devices, isRunning bool) error {
	return ErrCannotUpdate
}

// Remove returns nil error as majority of devices don't need to do any host-side cleanup on delete.
func (d *deviceCommon) Remove() error {
	return nil
}

// generateHostName generates the name to use for the host side NIC interface based on the
// instances.nic.host_name setting.
// Accepts prefix argument to use with random interface generation.
// Accepts optional hwaddr MAC address to use for generating the interface name in mac mode.
// In mac mode the interface prefix is always "lxd".
func (d *deviceCommon) generateHostName(prefix string, hwaddr string) (string, error) {
	hostNameMode := d.state.GlobalConfig.InstancesNICHostname()

	// Handle instances.nic.host_name mac mode if a MAC address has been supplied.
	if hostNameMode == "mac" && hwaddr != "" {
		mac, err := net.ParseMAC(hwaddr)
		if err != nil {
			return "", fmt.Errorf("Failed parsing MAC address %q: %w", hwaddr, err)
		}

		return network.MACDevName(mac), nil
	}

	// Handle instances.nic.host_name random mode or where no MAC address supplied.
	return network.RandomDevName(prefix), nil
}
<|endoftext|>


<commit_before>package imgur

import (
	"errors"
	"strings"
)

// GenericInfo is returned from functions for which the final result type is not known beforehand.
// Only one pointer is != nil
type GenericInfo struct {
	Image  *ImageInfo
	Album  *AlbumInfo
	GImage *GalleryImageInfo
	GAlbum *GalleryAlbumInfo
	Limit  *RateLimit
}

// GetInfoFromURL tries to query imgur based on information identified in the URL.
// returns image/album info, status code of the request, error
func (client *Client) GetInfoFromURL(url string) (*GenericInfo, int, error) {
	url = strings.TrimSpace(url)
	var ret GenericInfo

	// https://i.imgur.com/<id>.jpg -> image
	if strings.Contains(url, "://i.imgur.com/") {
		start := strings.LastIndex(url, "/") + 1
		end := strings.LastIndex(url, ".")
		if start == -1 || end == -1 || start == end {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down i.imgur.com path.")
		}
		id := url[start:end]
		client.Log.Debugf("Detected imgur image ID %v. Was going down the i.imgur.com/ path.", id)
		gii, status, err := client.GetGalleryImageInfo(id)
		if status < 400 {
			ret.GImage = gii
		} else {
			var ii *ImageInfo
			ii, status, err = client.GetImageInfo(id)
			ret.Image = ii
		}
		return &ret, status, err
	}

	// https://imgur.com/a/<id> -> album
	if strings.Contains(url, "://imgur.com/a/") {
		start := strings.LastIndex(url, "/") + 1
		if start == -1 {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down imgur.com/a/ path.")
		}
		id := url[start:]
		client.Log.Debugf("Detected imgur album ID %v. Was going down the imgur.com/a/ path.", id)
		ai, status, err := client.GetAlbumInfo(id)
		ret.Album = ai
		return &ret, status, err
	}

	// https://imgur.com/gallery/<id> len(id) == 5 -> gallery album
	// https://imgur.com/gallery/<id> len(id) == 7 -> gallery image
	if strings.Contains(url, "://imgur.com/gallery/") {
		start := strings.LastIndex(url, "/") + 1
		if start == -1 {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down imgur.com/gallery/ path.")
		}
		id := url[start:]
		client.Log.Debugf("Detected imgur gallery ID %v. Was going down the imgur.com/gallery/ path.", id)
		if len(id) == 5 {
			client.Log.Debugf("Detected imgur gallery album.")
			ai, status, err := client.GetGalleryAlbumInfo(id)
			ret.GAlbum = ai
			return &ret, status, err
		}

		ii, status, err := client.GetGalleryImageInfo(id)
		ret.GImage = ii
		return &ret, status, err
	}

	// https://imgur.com/<id> -> image
	if strings.Contains(url, "://imgur.com/") {
		start := strings.LastIndex(url, "/") + 1
		if start == -1 {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down imgur.com/ path.")
		}
		id := url[start:]
		client.Log.Debugf("Detected imgur image ID %v. Was going down the imgur.com/ path.", id)
		ii, status, err := client.GetGalleryImageInfo(id)
		ret.GImage = ii
		return &ret, status, err
	}

	return nil, -1, errors.New("URL pattern matching for URL " + url + " failed.")
}
<commit_msg>Added support for mobile URLs.<commit_after>package imgur

import (
	"errors"
	"strings"
)

// GenericInfo is returned from functions for which the final result type is not known beforehand.
// Only one pointer is != nil
type GenericInfo struct {
	Image  *ImageInfo
	Album  *AlbumInfo
	GImage *GalleryImageInfo
	GAlbum *GalleryAlbumInfo
	Limit  *RateLimit
}

// GetInfoFromURL tries to query imgur based on information identified in the URL.
// returns image/album info, status code of the request, error
func (client *Client) GetInfoFromURL(url string) (*GenericInfo, int, error) {
	url = strings.TrimSpace(url)
	var ret GenericInfo

	// https://i.imgur.com/<id>.jpg -> image
	if strings.Contains(url, "://i.imgur.com/") {
		start := strings.LastIndex(url, "/") + 1
		end := strings.LastIndex(url, ".")
		if start == -1 || end == -1 || start == end {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down i.imgur.com path.")
		}
		id := url[start:end]
		client.Log.Debugf("Detected imgur image ID %v. Was going down the i.imgur.com/ path.", id)
		gii, status, err := client.GetGalleryImageInfo(id)
		if status < 400 {
			ret.GImage = gii
		} else {
			var ii *ImageInfo
			ii, status, err = client.GetImageInfo(id)
			ret.Image = ii
		}
		return &ret, status, err
	}

	// https://imgur.com/a/<id> -> album
	if strings.Contains(url, "://imgur.com/a/") || strings.Contains(url, "://m.imgur.com/a/") {
		start := strings.LastIndex(url, "/") + 1
		if start == -1 {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down imgur.com/a/ path.")
		}
		end := strings.LastIndex(url, "?")
		id := url[start:end]
		client.Log.Debugf("Detected imgur album ID %v. Was going down the imgur.com/a/ path.", id)
		ai, status, err := client.GetAlbumInfo(id)
		ret.Album = ai
		return &ret, status, err
	}

	// https://imgur.com/gallery/<id> len(id) == 5 -> gallery album
	// https://imgur.com/gallery/<id> len(id) == 7 -> gallery image
	if strings.Contains(url, "://imgur.com/gallery/") || strings.Contains(url, "://m.imgur.com/gallery/") {
		start := strings.LastIndex(url, "/") + 1
		if start == -1 {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down imgur.com/gallery/ path.")
		}
		end := strings.LastIndex(url, "?")
		id := url[start:end]
		client.Log.Debugf("Detected imgur gallery ID %v. Was going down the imgur.com/gallery/ path.", id)
		if len(id) == 5 {
			client.Log.Debugf("Detected imgur gallery album.")
			ai, status, err := client.GetGalleryAlbumInfo(id)
			ret.GAlbum = ai
			return &ret, status, err
		}

		ii, status, err := client.GetGalleryImageInfo(id)
		ret.GImage = ii
		return &ret, status, err
	}

	// https://imgur.com/<id> -> image
	if strings.Contains(url, "://imgur.com/") || strings.Contains(url, "://m.imgur.com/") {
		start := strings.LastIndex(url, "/") + 1
		if start == -1 {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down imgur.com/ path.")
		}
		end := strings.LastIndex(url, "?")
		id := url[start:end]
		client.Log.Debugf("Detected imgur image ID %v. Was going down the imgur.com/ path.", id)
		ii, status, err := client.GetGalleryImageInfo(id)
		ret.GImage = ii
		return &ret, status, err
	}

	return nil, -1, errors.New("URL pattern matching for URL " + url + " failed.")
}
<|endoftext|>


<commit_before>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package procfs

import "testing"

const (
	procTestFixtures = "fixtures/proc"
)

func TestNewFS(t *testing.T) {
	if _, err := NewFS("foobar"); err == nil {
		t.Error("want NewFS to fail for non-existing mount point")
	}

	if _, err := NewFS("procfs.go"); err == nil {
		t.Error("want NewFS to fail if mount point is not a directory")
	}
	getProcFixtures(t)
}

func getProcFixtures(t *testing.T) FS {
	fs, err := NewFS(procTestFixtures)
	if err != nil {
		t.Errorf("Creating pseudo fs from getProcFixtures failed at fixtures/proc")
	}
	return fs
}
<commit_msg>Improved error message in getProcFixtures()<commit_after>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package procfs

import "testing"

const (
	procTestFixtures = "fixtures/proc"
)

func TestNewFS(t *testing.T) {
	if _, err := NewFS("foobar"); err == nil {
		t.Error("want NewFS to fail for non-existing mount point")
	}

	if _, err := NewFS("procfs.go"); err == nil {
		t.Error("want NewFS to fail if mount point is not a directory")
	}
	getProcFixtures(t)
}

func getProcFixtures(t *testing.T) FS {
	fs, err := NewFS(procTestFixtures)
	if err != nil {
		t.Fatalf("Creating pseudo fs from getProcFixtures failed at fixtures/proc with error: %s", err)
	}
	return fs
}
<|endoftext|>


<commit_before>package boxstrapper_test

import (
	"testing"
	. "boxstrapper"
  	"github.com/stretchr/testify/assert"
)

func TestSinglePackage(t *testing.T) {
	s := "i3: system"

	packages := NewPackage(s)

	assert.Equal(t, 1, len(packages))
	assert.Equal(t, "i3", packages[0].Package)
	assert.Equal(t, 1, len(packages[0].Groups))
	assert.Equal(t, "system", packages[0].Groups[0])
}

func TestSinglePackageWithLeadingWhitespace(t *testing.T) {
	s := "i3:  system"

	packages := NewPackage(s)

	assert.Equal(t, 1, len(packages))
	assert.Equal(t, "i3", packages[0].Package)
	assert.Equal(t, 1, len(packages[0].Groups))
	assert.Equal(t, "system", packages[0].Groups[0])
}

func TestSinglePackageWithTrailingWhitespace(t *testing.T) {
	s := "i3: system "

	packages := NewPackage(s)

	assert.Equal(t, 1, len(packages))
	assert.Equal(t, "i3", packages[0].Package)
	assert.Equal(t, 1, len(packages[0].Groups))
	assert.Equal(t, "system", packages[0].Groups[0])
}

func TestSinglePackage_MultipleGroups(t *testing.T) {
	s := "i3: system, boxstrapper"

	packages := NewPackage(s)

	assert.Equal(t, 1, len(packages))
	assert.Equal(t, "i3", packages[0].Package)
	assert.Equal(t, 2, len(packages[0].Groups))
	assert.Equal(t, "system", packages[0].Groups[0])
	assert.Equal(t, "boxstrapper", packages[0].Groups[1])
}

func TestSinglePackage_MultipleGroups_PrefixWhitespace(t *testing.T) {
	s := "i3: system,  boxstrapper"

	packages := NewPackage(s)

	assert.Equal(t, 1, len(packages))
	assert.Equal(t, "i3", packages[0].Package)
	assert.Equal(t, 2, len(packages[0].Groups))
	assert.Equal(t, "system", packages[0].Groups[0])
	assert.Equal(t, "boxstrapper", packages[0].Groups[1])
}<commit_msg>TestSinglePackage_MultipleGroups_PostfixWhitespace<commit_after>package boxstrapper_test

import (
	"testing"
	. "boxstrapper"
  	"github.com/stretchr/testify/assert"
)

func TestSinglePackage(t *testing.T) {
	s := "i3: system"

	packages := NewPackage(s)

	assert.Equal(t, 1, len(packages))
	assert.Equal(t, "i3", packages[0].Package)
	assert.Equal(t, 1, len(packages[0].Groups))
	assert.Equal(t, "system", packages[0].Groups[0])
}

func TestSinglePackageWithLeadingWhitespace(t *testing.T) {
	s := "i3:  system"

	packages := NewPackage(s)

	assert.Equal(t, 1, len(packages))
	assert.Equal(t, "i3", packages[0].Package)
	assert.Equal(t, 1, len(packages[0].Groups))
	assert.Equal(t, "system", packages[0].Groups[0])
}

func TestSinglePackageWithTrailingWhitespace(t *testing.T) {
	s := "i3: system "

	packages := NewPackage(s)

	assert.Equal(t, 1, len(packages))
	assert.Equal(t, "i3", packages[0].Package)
	assert.Equal(t, 1, len(packages[0].Groups))
	assert.Equal(t, "system", packages[0].Groups[0])
}

func TestSinglePackage_MultipleGroups(t *testing.T) {
	s := "i3: system, boxstrapper"

	packages := NewPackage(s)

	assert.Equal(t, 1, len(packages))
	assert.Equal(t, "i3", packages[0].Package)
	assert.Equal(t, 2, len(packages[0].Groups))
	assert.Equal(t, "system", packages[0].Groups[0])
	assert.Equal(t, "boxstrapper", packages[0].Groups[1])
}

func TestSinglePackage_MultipleGroups_PrefixWhitespace(t *testing.T) {
	s := "i3: system,  boxstrapper"

	packages := NewPackage(s)

	assert.Equal(t, 1, len(packages))
	assert.Equal(t, "i3", packages[0].Package)
	assert.Equal(t, 2, len(packages[0].Groups))
	assert.Equal(t, "system", packages[0].Groups[0])
	assert.Equal(t, "boxstrapper", packages[0].Groups[1])
}

func TestSinglePackage_MultipleGroups_PostfixWhitespace(t *testing.T) {
	s := "i3: system, boxstrapper "

	packages := NewPackage(s)

	assert.Equal(t, 1, len(packages))
	assert.Equal(t, "i3", packages[0].Package)
	assert.Equal(t, 2, len(packages[0].Groups))
	assert.Equal(t, "system", packages[0].Groups[0])
	assert.Equal(t, "boxstrapper", packages[0].Groups[1])
}<|endoftext|>


<commit_before>package plugin

import (
	"errors"
	"fmt"
	"net"
	"net/rpc"
	"os"
	"time"
)

type cliConnection struct {
	cliServerPort string
}

func NewCliConnection(cliServerPort string) *cliConnection {
	return &cliConnection{
		cliServerPort: cliServerPort,
	}
}

func (cliConnection *cliConnection) sendPluginMetadataToCliServer(metadata PluginMetadata) {
	cliServerConn, err := rpc.Dial("tcp", "127.0.0.1:"+cliConnection.cliServerPort)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	defer cliServerConn.Close()

	var success bool

	err = cliServerConn.Call("CliRpcCmd.SetPluginMetadata", metadata, &success)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	if !success {
		os.Exit(1)
	}

	os.Exit(0)
}

func (cliConnection *cliConnection) CliCommandWithoutTerminalOutput(args ...string) ([]string, error) {
	return cliConnection.callCliCommand(true, args...)
}

func (cliConnection *cliConnection) CliCommand(args ...string) ([]string, error) {
	return cliConnection.callCliCommand(false, args...)
}

func (cliConnection *cliConnection) callCliCommand(silently bool, args ...string) ([]string, error) {
	client, err := rpc.Dial("tcp", "127.0.0.1:"+cliConnection.cliServerPort)
	if err != nil {
		return []string{}, err
	}
	defer client.Close()

	var success bool

	client.Call("CliRpcCmd.DisableTerminalOutput", silently, &success)
	err = client.Call("CliRpcCmd.CallCoreCommand", args, &success)

	var cmdOutput []string
	outputErr := client.Call("CliRpcCmd.GetOutputAndReset", success, &cmdOutput)

	if err != nil {
		return cmdOutput, err
	} else if !success {
		return cmdOutput, errors.New("Error executing cli core command")
	}

	if outputErr != nil {
		return cmdOutput, errors.New("something completely unexpected happened")
	}
	return cmdOutput, nil
}

func (cliConnection *cliConnection) pingCLI() {
	//call back to cf saying we have been setup
	var connErr error
	var conn net.Conn
	for i := 0; i < 5; i++ {
		conn, connErr = net.Dial("tcp", "127.0.0.1:"+cliConnection.cliServerPort)
		defer conn.Close()
		if connErr != nil {
			time.Sleep(200 * time.Millisecond)
		} else {
			break
		}
	}
	if connErr != nil {
		fmt.Println(connErr)
		os.Exit(1)
	}
}
<commit_msg>fixes error when plugin rpc server is not reachable<commit_after>package plugin

import (
	"errors"
	"fmt"
	"net"
	"net/rpc"
	"os"
	"time"
)

type cliConnection struct {
	cliServerPort string
}

func NewCliConnection(cliServerPort string) *cliConnection {
	return &cliConnection{
		cliServerPort: cliServerPort,
	}
}

func (cliConnection *cliConnection) sendPluginMetadataToCliServer(metadata PluginMetadata) {
	cliServerConn, err := rpc.Dial("tcp", "127.0.0.1:"+cliConnection.cliServerPort)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	defer cliServerConn.Close()

	var success bool

	err = cliServerConn.Call("CliRpcCmd.SetPluginMetadata", metadata, &success)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	if !success {
		os.Exit(1)
	}

	os.Exit(0)
}

func (cliConnection *cliConnection) CliCommandWithoutTerminalOutput(args ...string) ([]string, error) {
	return cliConnection.callCliCommand(true, args...)
}

func (cliConnection *cliConnection) CliCommand(args ...string) ([]string, error) {
	return cliConnection.callCliCommand(false, args...)
}

func (cliConnection *cliConnection) callCliCommand(silently bool, args ...string) ([]string, error) {
	client, err := rpc.Dial("tcp", "127.0.0.1:"+cliConnection.cliServerPort)
	if err != nil {
		return []string{}, err
	}
	defer client.Close()

	var success bool

	client.Call("CliRpcCmd.DisableTerminalOutput", silently, &success)
	err = client.Call("CliRpcCmd.CallCoreCommand", args, &success)

	var cmdOutput []string
	outputErr := client.Call("CliRpcCmd.GetOutputAndReset", success, &cmdOutput)

	if err != nil {
		return cmdOutput, err
	} else if !success {
		return cmdOutput, errors.New("Error executing cli core command")
	}

	if outputErr != nil {
		return cmdOutput, errors.New("something completely unexpected happened")
	}
	return cmdOutput, nil
}

func (cliConnection *cliConnection) pingCLI() {
	//call back to cf saying we have been setup
	var connErr error
	var conn net.Conn
	for i := 0; i < 5; i++ {
		conn, connErr = net.Dial("tcp", "127.0.0.1:"+cliConnection.cliServerPort)
		if connErr != nil {
			time.Sleep(200 * time.Millisecond)
		} else {
			conn.Close()
			break
		}
	}
	if connErr != nil {
		fmt.Println(connErr)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"io"
	"os"
	"reflect"
	"sort"
	"strings"
	"time"

	"github.com/codegangsta/cli"
)

var (
	defaultAmqpURI                = "amqp://"
	defaultBaseDir                = "."
	defaultFilePollingInterval, _ = time.ParseDuration("5s")
	defaultPoolSize               = 1
	defaultProviderName           = "docker"
	defaultQueueType              = "amqp"

	defaultHardTimeout, _            = time.ParseDuration("50m")
	defaultLogTimeout, _             = time.ParseDuration("10m")
	defaultScriptUploadTimeout, _    = time.ParseDuration("3m30s")
	defaultStartupTimeout, _         = time.ParseDuration("4m")
	defaultBuildCacheFetchTimeout, _ = time.ParseDuration("5m")
	defaultBuildCachePushTimeout, _  = time.ParseDuration("5m")

	defaultHostname, _ = os.Hostname()
	defaultLanguage    = "default"
	defaultDist        = "precise"
	defaultGroup       = "stable"
	defaultOS          = "linux"

	configType = reflect.ValueOf(Config{}).Type()

	boolFlagType     = reflect.ValueOf(cli.BoolFlag{}).Type()
	durationFlagType = reflect.ValueOf(cli.DurationFlag{}).Type()
	intFlagType      = reflect.ValueOf(cli.IntFlag{}).Type()
	stringFlagType   = reflect.ValueOf(cli.StringFlag{}).Type()

	zeroStringValue = reflect.Zero(reflect.ValueOf("").Type())

	defs = []*ConfigDef{
		NewConfigDef("ProviderName", &cli.StringFlag{
			Value: defaultProviderName,
			Usage: "The name of the provider to use. See below for provider-specific configuration",
		}),
		NewConfigDef("QueueType", &cli.StringFlag{
			Value: defaultQueueType,
			Usage: `The name of the queue type to use ("amqp" or "file")`,
		}),
		NewConfigDef("AmqpURI", &cli.StringFlag{
			Value: defaultAmqpURI,
			Usage: `The URI to the AMQP server to connect to (only valid for "amqp" queue type)`,
		}),
		NewConfigDef("BaseDir", &cli.StringFlag{
			Value: defaultBaseDir,
			Usage: `The base directory for file-based queues (only valid for "file" queue type)`,
		}),
		NewConfigDef("FilePollingInterval", &cli.DurationFlag{
			Value: defaultFilePollingInterval,
			Usage: `The interval at which file-based queues are checked (only valid for "file" queue type)`,
		}),
		NewConfigDef("PoolSize", &cli.IntFlag{
			Value: defaultPoolSize,
			Usage: "The size of the processor pool, affecting the number of jobs this worker can run in parallel",
		}),
		NewConfigDef("BuildAPIURI", &cli.StringFlag{
			Usage: "The full URL to the build API endpoint to use. Note that this also requires the path of the URL. If a username is included in the URL, this will be translated to a token passed in the Authorization header",
		}),
		NewConfigDef("QueueName", &cli.StringFlag{
			Usage: "The AMQP queue to subscribe to for jobs",
		}),
		NewConfigDef("LibratoEmail", &cli.StringFlag{
			Usage: "Librato metrics account email",
		}),
		NewConfigDef("LibratoToken", &cli.StringFlag{
			Usage: "Librato metrics account token",
		}),
		NewConfigDef("LibratoSource", &cli.StringFlag{
			Value: defaultHostname,
			Usage: "Librato metrics source name",
		}),
		NewConfigDef("SentryDSN", &cli.StringFlag{
			Usage: "The DSN to send Sentry events to",
		}),
		NewConfigDef("SentryHookErrors", &cli.BoolFlag{
			Usage: "Add logrus.ErrorLevel to logrus sentry hook",
		}),
		NewConfigDef("Hostname", &cli.StringFlag{
			Value: defaultHostname,
			Usage: "Host name used in log output to identify the source of a job",
		}),
		NewConfigDef("DefaultLanguage", &cli.StringFlag{
			Value: defaultLanguage,
			Usage: "Default \"language\" value for each job",
		}),
		NewConfigDef("DefaultDist", &cli.StringFlag{
			Value: defaultDist,
			Usage: "Default \"dist\" value for each job",
		}),
		NewConfigDef("DefaultGroup", &cli.StringFlag{
			Value: defaultGroup,
			Usage: "Default \"group\" value for each job",
		}),
		NewConfigDef("DefaultOS", &cli.StringFlag{
			Value: defaultOS,
			Usage: "Default \"os\" value for each job",
		}),
		NewConfigDef("HardTimeout", &cli.DurationFlag{
			Value: defaultHardTimeout,
			Usage: "The outermost (maximum) timeout for a given job, at which time the job is cancelled",
		}),
		NewConfigDef("LogTimeout", &cli.DurationFlag{
			Value: defaultLogTimeout,
			Usage: "The timeout for a job that's not outputting anything",
		}),
		NewConfigDef("ScriptUploadTimeout", &cli.DurationFlag{
			Value: defaultScriptUploadTimeout,
			Usage: "The timeout for the script upload step",
		}),
		NewConfigDef("StartupTimeout", &cli.DurationFlag{
			Value: defaultStartupTimeout,
			Usage: "The timeout for execution environment to be ready",
		}),

		// build script generator flags
		NewConfigDef("BuildCacheFetchTimeout", &cli.DurationFlag{
			Value: defaultBuildCacheFetchTimeout,
		}),
		NewConfigDef("BuildCachePushTimeout", &cli.DurationFlag{
			Value: defaultBuildCachePushTimeout,
		}),
		NewConfigDef("BuildAptCache", &cli.StringFlag{}),
		NewConfigDef("BuildNpmCache", &cli.StringFlag{}),
		NewConfigDef("BuildParanoid", &cli.BoolFlag{}),
		NewConfigDef("BuildFixResolvConf", &cli.BoolFlag{}),
		NewConfigDef("BuildFixEtcHosts", &cli.BoolFlag{}),
		NewConfigDef("BuildCacheType", &cli.StringFlag{}),
		NewConfigDef("BuildCacheS3Scheme", &cli.StringFlag{}),
		NewConfigDef("BuildCacheS3Region", &cli.StringFlag{}),
		NewConfigDef("BuildCacheS3Bucket", &cli.StringFlag{}),
		NewConfigDef("BuildCacheS3AccessKeyID", &cli.StringFlag{}),
		NewConfigDef("BuildCacheS3SecretAccessKey", &cli.StringFlag{}),

		// non-config and special case flags
		NewConfigDef("SkipShutdownOnLogTimeout", &cli.StringFlag{
			Usage: "Special-case mode to aid with debugging timed out jobs",
		}),
		NewConfigDef("BuildAPIInsecureSkipVerify", &cli.BoolFlag{
			Usage: "Skip build API TLS verification (useful for Enterprise and testing)",
		}),
		NewConfigDef("pprof-port", &cli.StringFlag{
			Usage: "enable pprof http endpoint at port",
		}),
		NewConfigDef("silence-metrics", &cli.BoolFlag{
			Usage: "silence metrics logging in case no Librato creds have been provided",
		}),
		NewConfigDef("echo-config", &cli.BoolFlag{
			Usage: "echo parsed config and exit",
		}),
		NewConfigDef("list-backend-providers", &cli.BoolFlag{
			Usage: "echo backend provider list and exit",
		}),
		NewConfigDef("debug", &cli.BoolFlag{
			Usage: "set log level to debug",
		}),
	}

	// Flags is the list of all CLI flags accepted by travis-worker
	Flags = defFlags(defs)
)

func twEnvVars(key string) string {
	return strings.ToUpper(strings.Join(twEnvVarsSlice(key), ","))
}

func twEnvVarsSlice(key string) []string {
	return []string{
		fmt.Sprintf("TRAVIS_WORKER_%s", key),
		key,
	}
}

func init() {
	wd, err := os.Getwd()
	if err != nil {
		return
	}

	defaultBaseDir = wd
}

func defFlags(defs []*ConfigDef) []cli.Flag {
	f := []cli.Flag{}

	for _, def := range defs {
		f = append(f, def.Flag)
	}

	return f
}

type ConfigDef struct {
	FieldName string
	Name      string
	EnvVar    string
	Flag      cli.Flag
}

func NewConfigDef(fieldName string, flag cli.Flag) *ConfigDef {
	if fieldName == "" {
		panic("empty field name")
	}

	name := ""

	if string(fieldName[0]) == strings.ToLower(string(fieldName[0])) {
		name = fieldName
	} else {
		field, _ := configType.FieldByName(fieldName)
		name = field.Tag.Get("config")
	}

	env := strings.ToUpper(strings.Replace(name, "-", "_", -1))

	def := &ConfigDef{
		FieldName: fieldName,
		Name:      name,
		EnvVar:    env,
	}

	envPrefixed := twEnvVars(env)

	if f, ok := flag.(*cli.BoolFlag); ok {
		def.Flag, f.Name, f.EnvVar = f, name, envPrefixed
		return def
	} else if f, ok := flag.(*cli.StringFlag); ok {
		def.Flag, f.Name, f.EnvVar = f, name, envPrefixed
		return def
	} else if f, ok := flag.(*cli.IntFlag); ok {
		def.Flag, f.Name, f.EnvVar = f, name, envPrefixed
		return def
	} else if f, ok := flag.(*cli.DurationFlag); ok {
		def.Flag, f.Name, f.EnvVar = f, name, envPrefixed
		return def
	} else {
		return def
	}
}

// Config contains all the configuration needed to run the worker.
type Config struct {
	ProviderName    string `config:"provider-name"`
	QueueType       string `config:"queue-type"`
	AmqpURI         string `config:"amqp-uri"`
	BaseDir         string `config:"base-dir"`
	PoolSize        int    `config:"pool-size"`
	BuildAPIURI     string `config:"build-api-uri"`
	QueueName       string `config:"queue-name"`
	LibratoEmail    string `config:"librato-email"`
	LibratoToken    string `config:"librato-token"`
	LibratoSource   string `config:"librato-source"`
	SentryDSN       string `config:"sentry-dsn"`
	Hostname        string `config:"hostname"`
	DefaultLanguage string `config:"default-language"`
	DefaultDist     string `config:"default-dist"`
	DefaultGroup    string `config:"default-group"`
	DefaultOS       string `config:"default-os"`

	FilePollingInterval time.Duration `config:"file-polling-interval"`
	HardTimeout         time.Duration `config:"hard-timeout"`
	LogTimeout          time.Duration `config:"log-timeout"`
	ScriptUploadTimeout time.Duration `config:"script-upload-timeout"`
	StartupTimeout      time.Duration `config:"startup-timeout"`

	SentryHookErrors           bool `config:"sentry-hook-errors"`
	BuildAPIInsecureSkipVerify bool `config:"build-api-insecure-skip-verify"`
	SkipShutdownOnLogTimeout   bool `config:"skip-shutdown-on-log-timeout"`

	// build script generator options
	BuildCacheFetchTimeout time.Duration `config:"build-cache-fetch-timeout"`
	BuildCachePushTimeout  time.Duration `config:"build-cache-push-timeout"`

	BuildParanoid      bool `config:"build-paranoid"`
	BuildFixResolvConf bool `config:"build-fix-resolv-conf"`
	BuildFixEtcHosts   bool `config:"build-fix-etc-hosts"`

	BuildAptCache               string `config:"build-apt-cache"`
	BuildNpmCache               string `config:"build-npm-cache"`
	BuildCacheType              string `config:"build-cache-type"`
	BuildCacheS3Scheme          string `config:"build-cache-s3-scheme"`
	BuildCacheS3Region          string `config:"build-cache-s3-region"`
	BuildCacheS3Bucket          string `config:"build-cache-s3-bucket"`
	BuildCacheS3AccessKeyID     string `config:"build-cache-s3-access-key-id"`
	BuildCacheS3SecretAccessKey string `config:"build-cache-s3-secret-access-key"`

	ProviderConfig *ProviderConfig
}

// FromCLIContext creates a Config using a cli.Context by pulling configuration
// from the flags in the context.
func FromCLIContext(c *cli.Context) *Config {
	cfg := &Config{}
	cfgVal := reflect.ValueOf(cfg)

	for _, def := range defs {
		field := cfgVal.FieldByName(def.FieldName)
		if field == zeroStringValue {
			continue
		}

		switch reflect.TypeOf(def.Flag) {
		case boolFlagType:
			field.SetBool(c.Bool(def.Name))
		case durationFlagType:
			field.Set(reflect.ValueOf(c.Duration(def.Name)))
		case intFlagType:
			field.SetInt(int64(c.Int(def.Name)))
		case stringFlagType:
			field.SetString(c.String(def.Name))
		}
	}

	cfg.ProviderConfig = ProviderConfigFromEnviron(cfg.ProviderName)

	return cfg
}

// WriteEnvConfig writes the given configuration to out. The format of the
// output is a list of environment variables settings suitable to be sourced
// by a Bourne-like shell.
func WriteEnvConfig(cfg *Config, out io.Writer) {
	cfgMap := map[string]interface{}{}

	for _, def := range defs {
		cfgMap[def.Name] = reflect.ValueOf(cfg).FieldByName(def.FieldName).Interface()
	}

	sortedCfgMapKeys := []string{}

	for key := range cfgMap {
		sortedCfgMapKeys = append(sortedCfgMapKeys, key)
	}

	sort.Strings(sortedCfgMapKeys)

	fmt.Fprintf(out, "# travis-worker env config generated %s\n", time.Now().UTC())
	for _, key := range sortedCfgMapKeys {
		envKey := fmt.Sprintf("TRAVIS_WORKER_%s", strings.ToUpper(strings.Replace(key, "-", "_", -1)))
		fmt.Fprintf(out, "export %s=%q\n", envKey, fmt.Sprintf("%v", cfgMap[key]))
	}
	fmt.Fprintf(out, "\n# travis-worker provider config:\n")
	cfg.ProviderConfig.Each(func(key, value string) {
		envKey := strings.ToUpper(fmt.Sprintf("TRAVIS_WORKER_%s_%s", cfg.ProviderName, strings.Replace(key, "-", "_", -1)))
		fmt.Fprintf(out, "export %s=%q\n", envKey, value)
	})
	fmt.Fprintf(out, "# end travis-worker env config\n")
}
<commit_msg>Fix some runtime panics<commit_after>package config

import (
	"fmt"
	"io"
	"os"
	"reflect"
	"sort"
	"strings"
	"time"

	"github.com/codegangsta/cli"
)

var (
	defaultAmqpURI                = "amqp://"
	defaultBaseDir                = "."
	defaultFilePollingInterval, _ = time.ParseDuration("5s")
	defaultPoolSize               = 1
	defaultProviderName           = "docker"
	defaultQueueType              = "amqp"

	defaultHardTimeout, _            = time.ParseDuration("50m")
	defaultLogTimeout, _             = time.ParseDuration("10m")
	defaultScriptUploadTimeout, _    = time.ParseDuration("3m30s")
	defaultStartupTimeout, _         = time.ParseDuration("4m")
	defaultBuildCacheFetchTimeout, _ = time.ParseDuration("5m")
	defaultBuildCachePushTimeout, _  = time.ParseDuration("5m")

	defaultHostname, _ = os.Hostname()
	defaultLanguage    = "default"
	defaultDist        = "precise"
	defaultGroup       = "stable"
	defaultOS          = "linux"

	configType = reflect.ValueOf(Config{}).Type()

	boolFlagType     = reflect.ValueOf(cli.BoolFlag{}).Type()
	durationFlagType = reflect.ValueOf(cli.DurationFlag{}).Type()
	intFlagType      = reflect.ValueOf(cli.IntFlag{}).Type()
	stringFlagType   = reflect.ValueOf(cli.StringFlag{}).Type()

	zeroStringValue = reflect.Zero(reflect.ValueOf("").Type())

	defs = []*ConfigDef{
		NewConfigDef("ProviderName", &cli.StringFlag{
			Value: defaultProviderName,
			Usage: "The name of the provider to use. See below for provider-specific configuration",
		}),
		NewConfigDef("QueueType", &cli.StringFlag{
			Value: defaultQueueType,
			Usage: `The name of the queue type to use ("amqp" or "file")`,
		}),
		NewConfigDef("AmqpURI", &cli.StringFlag{
			Value: defaultAmqpURI,
			Usage: `The URI to the AMQP server to connect to (only valid for "amqp" queue type)`,
		}),
		NewConfigDef("BaseDir", &cli.StringFlag{
			Value: defaultBaseDir,
			Usage: `The base directory for file-based queues (only valid for "file" queue type)`,
		}),
		NewConfigDef("FilePollingInterval", &cli.DurationFlag{
			Value: defaultFilePollingInterval,
			Usage: `The interval at which file-based queues are checked (only valid for "file" queue type)`,
		}),
		NewConfigDef("PoolSize", &cli.IntFlag{
			Value: defaultPoolSize,
			Usage: "The size of the processor pool, affecting the number of jobs this worker can run in parallel",
		}),
		NewConfigDef("BuildAPIURI", &cli.StringFlag{
			Usage: "The full URL to the build API endpoint to use. Note that this also requires the path of the URL. If a username is included in the URL, this will be translated to a token passed in the Authorization header",
		}),
		NewConfigDef("QueueName", &cli.StringFlag{
			Usage: "The AMQP queue to subscribe to for jobs",
		}),
		NewConfigDef("LibratoEmail", &cli.StringFlag{
			Usage: "Librato metrics account email",
		}),
		NewConfigDef("LibratoToken", &cli.StringFlag{
			Usage: "Librato metrics account token",
		}),
		NewConfigDef("LibratoSource", &cli.StringFlag{
			Value: defaultHostname,
			Usage: "Librato metrics source name",
		}),
		NewConfigDef("SentryDSN", &cli.StringFlag{
			Usage: "The DSN to send Sentry events to",
		}),
		NewConfigDef("SentryHookErrors", &cli.BoolFlag{
			Usage: "Add logrus.ErrorLevel to logrus sentry hook",
		}),
		NewConfigDef("Hostname", &cli.StringFlag{
			Value: defaultHostname,
			Usage: "Host name used in log output to identify the source of a job",
		}),
		NewConfigDef("DefaultLanguage", &cli.StringFlag{
			Value: defaultLanguage,
			Usage: "Default \"language\" value for each job",
		}),
		NewConfigDef("DefaultDist", &cli.StringFlag{
			Value: defaultDist,
			Usage: "Default \"dist\" value for each job",
		}),
		NewConfigDef("DefaultGroup", &cli.StringFlag{
			Value: defaultGroup,
			Usage: "Default \"group\" value for each job",
		}),
		NewConfigDef("DefaultOS", &cli.StringFlag{
			Value: defaultOS,
			Usage: "Default \"os\" value for each job",
		}),
		NewConfigDef("HardTimeout", &cli.DurationFlag{
			Value: defaultHardTimeout,
			Usage: "The outermost (maximum) timeout for a given job, at which time the job is cancelled",
		}),
		NewConfigDef("LogTimeout", &cli.DurationFlag{
			Value: defaultLogTimeout,
			Usage: "The timeout for a job that's not outputting anything",
		}),
		NewConfigDef("ScriptUploadTimeout", &cli.DurationFlag{
			Value: defaultScriptUploadTimeout,
			Usage: "The timeout for the script upload step",
		}),
		NewConfigDef("StartupTimeout", &cli.DurationFlag{
			Value: defaultStartupTimeout,
			Usage: "The timeout for execution environment to be ready",
		}),

		// build script generator flags
		NewConfigDef("BuildCacheFetchTimeout", &cli.DurationFlag{
			Value: defaultBuildCacheFetchTimeout,
		}),
		NewConfigDef("BuildCachePushTimeout", &cli.DurationFlag{
			Value: defaultBuildCachePushTimeout,
		}),
		NewConfigDef("BuildAptCache", &cli.StringFlag{}),
		NewConfigDef("BuildNpmCache", &cli.StringFlag{}),
		NewConfigDef("BuildParanoid", &cli.BoolFlag{}),
		NewConfigDef("BuildFixResolvConf", &cli.BoolFlag{}),
		NewConfigDef("BuildFixEtcHosts", &cli.BoolFlag{}),
		NewConfigDef("BuildCacheType", &cli.StringFlag{}),
		NewConfigDef("BuildCacheS3Scheme", &cli.StringFlag{}),
		NewConfigDef("BuildCacheS3Region", &cli.StringFlag{}),
		NewConfigDef("BuildCacheS3Bucket", &cli.StringFlag{}),
		NewConfigDef("BuildCacheS3AccessKeyID", &cli.StringFlag{}),
		NewConfigDef("BuildCacheS3SecretAccessKey", &cli.StringFlag{}),

		// non-config and special case flags
		NewConfigDef("SkipShutdownOnLogTimeout", &cli.StringFlag{
			Usage: "Special-case mode to aid with debugging timed out jobs",
		}),
		NewConfigDef("BuildAPIInsecureSkipVerify", &cli.BoolFlag{
			Usage: "Skip build API TLS verification (useful for Enterprise and testing)",
		}),
		NewConfigDef("pprof-port", &cli.StringFlag{
			Usage: "enable pprof http endpoint at port",
		}),
		NewConfigDef("silence-metrics", &cli.BoolFlag{
			Usage: "silence metrics logging in case no Librato creds have been provided",
		}),
		NewConfigDef("echo-config", &cli.BoolFlag{
			Usage: "echo parsed config and exit",
		}),
		NewConfigDef("list-backend-providers", &cli.BoolFlag{
			Usage: "echo backend provider list and exit",
		}),
		NewConfigDef("debug", &cli.BoolFlag{
			Usage: "set log level to debug",
		}),
	}

	// Flags is the list of all CLI flags accepted by travis-worker
	Flags = defFlags(defs)
)

func twEnvVars(key string) string {
	return strings.ToUpper(strings.Join(twEnvVarsSlice(key), ","))
}

func twEnvVarsSlice(key string) []string {
	return []string{
		fmt.Sprintf("TRAVIS_WORKER_%s", key),
		key,
	}
}

func init() {
	wd, err := os.Getwd()
	if err != nil {
		return
	}

	defaultBaseDir = wd
}

func defFlags(defs []*ConfigDef) []cli.Flag {
	f := []cli.Flag{}

	for _, def := range defs {
		f = append(f, def.Flag)
	}

	return f
}

type ConfigDef struct {
	FieldName string
	Name      string
	EnvVar    string
	Flag      cli.Flag
}

func NewConfigDef(fieldName string, flag cli.Flag) *ConfigDef {
	if fieldName == "" {
		panic("empty field name")
	}

	name := ""

	if string(fieldName[0]) == strings.ToLower(string(fieldName[0])) {
		name = fieldName
	} else {
		field, _ := configType.FieldByName(fieldName)
		name = field.Tag.Get("config")
	}

	env := strings.ToUpper(strings.Replace(name, "-", "_", -1))

	def := &ConfigDef{
		FieldName: fieldName,
		Name:      name,
		EnvVar:    env,
	}

	envPrefixed := twEnvVars(env)

	if f, ok := flag.(*cli.BoolFlag); ok {
		def.Flag, f.Name, f.EnvVar = f, name, envPrefixed
		return def
	} else if f, ok := flag.(*cli.StringFlag); ok {
		def.Flag, f.Name, f.EnvVar = f, name, envPrefixed
		return def
	} else if f, ok := flag.(*cli.IntFlag); ok {
		def.Flag, f.Name, f.EnvVar = f, name, envPrefixed
		return def
	} else if f, ok := flag.(*cli.DurationFlag); ok {
		def.Flag, f.Name, f.EnvVar = f, name, envPrefixed
		return def
	} else {
		return def
	}
}

// Config contains all the configuration needed to run the worker.
type Config struct {
	ProviderName    string `config:"provider-name"`
	QueueType       string `config:"queue-type"`
	AmqpURI         string `config:"amqp-uri"`
	BaseDir         string `config:"base-dir"`
	PoolSize        int    `config:"pool-size"`
	BuildAPIURI     string `config:"build-api-uri"`
	QueueName       string `config:"queue-name"`
	LibratoEmail    string `config:"librato-email"`
	LibratoToken    string `config:"librato-token"`
	LibratoSource   string `config:"librato-source"`
	SentryDSN       string `config:"sentry-dsn"`
	Hostname        string `config:"hostname"`
	DefaultLanguage string `config:"default-language"`
	DefaultDist     string `config:"default-dist"`
	DefaultGroup    string `config:"default-group"`
	DefaultOS       string `config:"default-os"`

	FilePollingInterval time.Duration `config:"file-polling-interval"`
	HardTimeout         time.Duration `config:"hard-timeout"`
	LogTimeout          time.Duration `config:"log-timeout"`
	ScriptUploadTimeout time.Duration `config:"script-upload-timeout"`
	StartupTimeout      time.Duration `config:"startup-timeout"`

	SentryHookErrors           bool `config:"sentry-hook-errors"`
	BuildAPIInsecureSkipVerify bool `config:"build-api-insecure-skip-verify"`
	SkipShutdownOnLogTimeout   bool `config:"skip-shutdown-on-log-timeout"`

	// build script generator options
	BuildCacheFetchTimeout time.Duration `config:"build-cache-fetch-timeout"`
	BuildCachePushTimeout  time.Duration `config:"build-cache-push-timeout"`

	BuildParanoid      bool `config:"build-paranoid"`
	BuildFixResolvConf bool `config:"build-fix-resolv-conf"`
	BuildFixEtcHosts   bool `config:"build-fix-etc-hosts"`

	BuildAptCache               string `config:"build-apt-cache"`
	BuildNpmCache               string `config:"build-npm-cache"`
	BuildCacheType              string `config:"build-cache-type"`
	BuildCacheS3Scheme          string `config:"build-cache-s3-scheme"`
	BuildCacheS3Region          string `config:"build-cache-s3-region"`
	BuildCacheS3Bucket          string `config:"build-cache-s3-bucket"`
	BuildCacheS3AccessKeyID     string `config:"build-cache-s3-access-key-id"`
	BuildCacheS3SecretAccessKey string `config:"build-cache-s3-secret-access-key"`

	ProviderConfig *ProviderConfig
}

// FromCLIContext creates a Config using a cli.Context by pulling configuration
// from the flags in the context.
func FromCLIContext(c *cli.Context) *Config {
	cfg := &Config{}
	cfgVal := reflect.ValueOf(cfg).Elem()

	for _, def := range defs {
		field := cfgVal.FieldByName(def.FieldName)
		if field == zeroStringValue {
			continue
		}

		switch reflect.TypeOf(def.Flag) {
		case boolFlagType:
			field.SetBool(c.Bool(def.Name))
		case durationFlagType:
			field.Set(reflect.ValueOf(c.Duration(def.Name)))
		case intFlagType:
			field.SetInt(int64(c.Int(def.Name)))
		case stringFlagType:
			field.SetString(c.String(def.Name))
		}
	}

	cfg.ProviderConfig = ProviderConfigFromEnviron(cfg.ProviderName)

	return cfg
}

// WriteEnvConfig writes the given configuration to out. The format of the
// output is a list of environment variables settings suitable to be sourced
// by a Bourne-like shell.
func WriteEnvConfig(cfg *Config, out io.Writer) {
	cfgMap := map[string]interface{}{}
	cfgElem := reflect.ValueOf(cfg).Elem()

	for _, def := range defs {
		if def.FieldName == "" || (string(def.FieldName[0]) == strings.ToLower(string(def.FieldName[0]))) {
			continue
		}

		field := cfgElem.FieldByName(def.FieldName)
		if field == zeroStringValue {
			return
		}

		cfgMap[def.Name] = field.Interface()
	}

	sortedCfgMapKeys := []string{}

	for key := range cfgMap {
		sortedCfgMapKeys = append(sortedCfgMapKeys, key)
	}

	sort.Strings(sortedCfgMapKeys)

	fmt.Fprintf(out, "# travis-worker env config generated %s\n", time.Now().UTC())
	for _, key := range sortedCfgMapKeys {
		envKey := fmt.Sprintf("TRAVIS_WORKER_%s", strings.ToUpper(strings.Replace(key, "-", "_", -1)))
		fmt.Fprintf(out, "export %s=%q\n", envKey, fmt.Sprintf("%v", cfgMap[key]))
	}
	fmt.Fprintf(out, "\n# travis-worker provider config:\n")
	cfg.ProviderConfig.Each(func(key, value string) {
		envKey := strings.ToUpper(fmt.Sprintf("TRAVIS_WORKER_%s_%s", cfg.ProviderName, strings.Replace(key, "-", "_", -1)))
		fmt.Fprintf(out, "export %s=%q\n", envKey, value)
	})
	fmt.Fprintf(out, "# end travis-worker env config\n")
}
<|endoftext|>


<commit_before>package config

import (
	"flag"
	"regexp"

	"github.com/gorilla/securecookie"

	"golang.org/x/oauth2"
)

const (
	PWDHostnameRegex      = "[0-9]{1,3}-[0-9]{1,3}-[0-9]{1,3}-[0-9]{1,3}"
	PortRegex             = "[0-9]{1,5}"
	AliasnameRegex        = "[0-9|a-z|A-Z|-]*"
	AliasSessionRegex     = "[0-9|a-z|A-Z]{8}"
	AliasGroupRegex       = "(" + AliasnameRegex + ")-(" + AliasSessionRegex + ")"
	PWDHostPortGroupRegex = "^.*ip(" + PWDHostnameRegex + ")(?:-?(" + PortRegex + "))?(?:\\..*)?$"
	AliasPortGroupRegex   = "^.*pwd" + AliasGroupRegex + "(?:-?(" + PortRegex + "))?\\..*$"
)

var NameFilter = regexp.MustCompile(PWDHostPortGroupRegex)
var AliasFilter = regexp.MustCompile(AliasPortGroupRegex)

var PortNumber, Key, Cert, SessionsFile, PWDContainerName, L2ContainerName, L2Subdomain, HashKey, SSHKeyPath, L2RouterIP, DindVolumeSize, CookieHashKey, CookieBlockKey, DefaultDinDImage, DefaultSessionDuration string
var UseLetsEncrypt, ExternalDindVolume, NoWindows bool
var LetsEncryptCertsDir string
var MaxLoadAvg float64
var ForceTLS bool
var SecureCookie *securecookie.SecureCookie
var AdminToken string

var PlaygroundDomain string

var SegmentId string

// TODO move this to a sync map so it can be updated on demand when the configuration for a playground changes
var Providers = map[string]map[string]*oauth2.Config{}

func ParseFlags() {
	flag.StringVar(&LetsEncryptCertsDir, "letsencrypt-certs-dir", "/certs", "Path where let's encrypt certs will be stored")
	flag.BoolVar(&UseLetsEncrypt, "letsencrypt-enable", false, "Enabled let's encrypt tls certificates")
	flag.BoolVar(&ForceTLS, "tls", false, "Use TLS to connect to docker daemons")
	flag.StringVar(&PortNumber, "port", "3000", "Port number")
	flag.StringVar(&Key, "tls-server-key", "./pwd/server-key.pem", "Server key for SSL")
	flag.StringVar(&Cert, "tls-cert", "./pwd/server.pem", "Give a SSL cert")
	flag.StringVar(&SessionsFile, "save", "./pwd/sessions", "Tell where to store sessions file")
	flag.StringVar(&PWDContainerName, "name", "pwd", "Container name used to run PWD (used to be able to connect it to the networks it creates)")
	flag.StringVar(&L2ContainerName, "l2", "l2", "Container name used to run L2 Router")
	flag.StringVar(&L2RouterIP, "l2-ip", "", "Host IP address for L2 router ping response")
	flag.StringVar(&L2Subdomain, "l2-subdomain", "direct", "Subdomain to the L2 Router")
	flag.StringVar(&HashKey, "hash_key", "salmonrosado", "Hash key to use for cookies")
	flag.StringVar(&DindVolumeSize, "dind-volume-size", "5G", "Dind volume folder size")
	flag.BoolVar(&NoWindows, "win-disable", false, "Disable windows instances")
	flag.BoolVar(&ExternalDindVolume, "dind-external-volume", false, "Use external dind volume though XFS volume driver")
	flag.Float64Var(&MaxLoadAvg, "maxload", 100, "Maximum allowed load average before failing ping requests")
	flag.StringVar(&SSHKeyPath, "ssh_key_path", "", "SSH Private Key to use")
	flag.StringVar(&CookieHashKey, "cookie-hash-key", "", "Hash key to use to validate cookies")
	flag.StringVar(&CookieBlockKey, "cookie-block-key", "", "Block key to use to encrypt cookies")
	flag.StringVar(&DefaultDinDImage, "default-dind-image", "franela/dind", "Default DinD image to use if not specified otherwise")
	flag.StringVar(&DefaultSessionDuration, "default-session-duration", "4h", "Default session duration if not specified otherwise")

	flag.StringVar(&PlaygroundDomain, "playground-domain", "localhost", "Domain to use for the playground")
	flag.StringVar(&AdminToken, "admin-token", "", "Token to validate admin user for admin endpoints")

	flag.StringVar(&SegmentId, "segment-id", "", "Segment id to post metrics")

	flag.Parse()

	SecureCookie = securecookie.New([]byte(CookieHashKey), []byte(CookieBlockKey))

}
<commit_msg>Remove unused config parameters<commit_after>package config

import (
	"flag"
	"regexp"

	"github.com/gorilla/securecookie"

	"golang.org/x/oauth2"
)

const (
	PWDHostnameRegex      = "[0-9]{1,3}-[0-9]{1,3}-[0-9]{1,3}-[0-9]{1,3}"
	PortRegex             = "[0-9]{1,5}"
	AliasnameRegex        = "[0-9|a-z|A-Z|-]*"
	AliasSessionRegex     = "[0-9|a-z|A-Z]{8}"
	AliasGroupRegex       = "(" + AliasnameRegex + ")-(" + AliasSessionRegex + ")"
	PWDHostPortGroupRegex = "^.*ip(" + PWDHostnameRegex + ")(?:-?(" + PortRegex + "))?(?:\\..*)?$"
	AliasPortGroupRegex   = "^.*pwd" + AliasGroupRegex + "(?:-?(" + PortRegex + "))?\\..*$"
)

var NameFilter = regexp.MustCompile(PWDHostPortGroupRegex)
var AliasFilter = regexp.MustCompile(AliasPortGroupRegex)

var PortNumber, SessionsFile, PWDContainerName, L2ContainerName, L2Subdomain, HashKey, SSHKeyPath, L2RouterIP, DindVolumeSize, CookieHashKey, CookieBlockKey, DefaultDinDImage, DefaultSessionDuration string
var UseLetsEncrypt, ExternalDindVolume, NoWindows bool
var LetsEncryptCertsDir string
var MaxLoadAvg float64
var ForceTLS bool
var SecureCookie *securecookie.SecureCookie
var AdminToken string

var PlaygroundDomain string

var SegmentId string

// TODO move this to a sync map so it can be updated on demand when the configuration for a playground changes
var Providers = map[string]map[string]*oauth2.Config{}

func ParseFlags() {
	flag.StringVar(&LetsEncryptCertsDir, "letsencrypt-certs-dir", "/certs", "Path where let's encrypt certs will be stored")
	flag.BoolVar(&UseLetsEncrypt, "letsencrypt-enable", false, "Enabled let's encrypt tls certificates")
	flag.BoolVar(&ForceTLS, "tls", false, "Use TLS to connect to docker daemons")
	flag.StringVar(&PortNumber, "port", "3000", "Port number")
	flag.StringVar(&SessionsFile, "save", "./pwd/sessions", "Tell where to store sessions file")
	flag.StringVar(&PWDContainerName, "name", "pwd", "Container name used to run PWD (used to be able to connect it to the networks it creates)")
	flag.StringVar(&L2ContainerName, "l2", "l2", "Container name used to run L2 Router")
	flag.StringVar(&L2RouterIP, "l2-ip", "", "Host IP address for L2 router ping response")
	flag.StringVar(&L2Subdomain, "l2-subdomain", "direct", "Subdomain to the L2 Router")
	flag.StringVar(&HashKey, "hash_key", "salmonrosado", "Hash key to use for cookies")
	flag.StringVar(&DindVolumeSize, "dind-volume-size", "5G", "Dind volume folder size")
	flag.BoolVar(&NoWindows, "win-disable", false, "Disable windows instances")
	flag.BoolVar(&ExternalDindVolume, "dind-external-volume", false, "Use external dind volume though XFS volume driver")
	flag.Float64Var(&MaxLoadAvg, "maxload", 100, "Maximum allowed load average before failing ping requests")
	flag.StringVar(&SSHKeyPath, "ssh_key_path", "", "SSH Private Key to use")
	flag.StringVar(&CookieHashKey, "cookie-hash-key", "", "Hash key to use to validate cookies")
	flag.StringVar(&CookieBlockKey, "cookie-block-key", "", "Block key to use to encrypt cookies")
	flag.StringVar(&DefaultDinDImage, "default-dind-image", "franela/dind", "Default DinD image to use if not specified otherwise")
	flag.StringVar(&DefaultSessionDuration, "default-session-duration", "4h", "Default session duration if not specified otherwise")

	flag.StringVar(&PlaygroundDomain, "playground-domain", "localhost", "Domain to use for the playground")
	flag.StringVar(&AdminToken, "admin-token", "", "Token to validate admin user for admin endpoints")

	flag.StringVar(&SegmentId, "segment-id", "", "Segment id to post metrics")

	flag.Parse()

	SecureCookie = securecookie.New([]byte(CookieHashKey), []byte(CookieBlockKey))

}
<|endoftext|>