<commit_before>package lfs

import (
	"fmt"
	"path/filepath"
	"sync"
	"sync/atomic"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/vendor/_nuts/github.com/cheggaaa/pb"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

type Transferable interface {
	Check() (*objectResource, *WrappedError)
	Transfer(CopyCallback) *WrappedError
	Object() *objectResource
	Oid() string
	Size() int64
	SetObject(*objectResource)
}

// TransferQueue provides a queue that will allow concurrent transfers.
type TransferQueue struct {
	transferc        chan Transferable
	errorc           chan *WrappedError
	watchers         []chan string
	errors           []*WrappedError
	wg               sync.WaitGroup
	workers          int
	files            int
	finished         int64
	size             int64
	authCond         *sync.Cond
	transferables    map[string]Transferable
	bar              *pb.ProgressBar
	clientAuthorized int32
	transferKind     string
}

// newTransferQueue builds a TransferQueue, allowing `workers` concurrent transfers.
func newTransferQueue(workers, files int) *TransferQueue {
	return &TransferQueue{
		transferc:     make(chan Transferable, files),
		errorc:        make(chan *WrappedError),
		watchers:      make([]chan string, 0),
		workers:       workers,
		files:         files,
		authCond:      sync.NewCond(&sync.Mutex{}),
		transferables: make(map[string]Transferable),
	}
}

// Add adds a Transferable to the transfer queue.
func (q *TransferQueue) Add(t Transferable) {
	q.transferables[t.Oid()] = t
}

// Watch returns a channel where the queue will write the OID of each transfer
// as it completes. The channel will be closed when the queue finishes processing.
func (q *TransferQueue) Watch() chan string {
	c := make(chan string, q.files)
	q.watchers = append(q.watchers, c)
	return c
}

// processIndividual processes the queue of transfers one at a time by making
// a POST call for each object, feeding the results to the transfer workers.
// If configured, the object transfers can still happen concurrently, the
// sequential nature here is only for the meta POST calls.
func (q *TransferQueue) processIndividual() {
	apic := make(chan Transferable, q.files)
	workersReady := make(chan int, q.workers)
	var wg sync.WaitGroup

	for i := 0; i < q.workers; i++ {
		go func() {
			workersReady <- 1
			for t := range apic {
				// If an API authorization has not occured, we wait until we're woken up.
				q.authCond.L.Lock()
				if atomic.LoadInt32(&q.clientAuthorized) == 0 {
					q.authCond.Wait()
				}
				q.authCond.L.Unlock()

				obj, err := t.Check()
				if err != nil {
					q.errorc <- err
					wg.Done()
					continue
				}
				if obj != nil {
					q.wg.Add(1)
					t.SetObject(obj)
					q.transferc <- t
				}
				wg.Done()
			}
		}()
	}

	q.bar.Prefix(fmt.Sprintf("(%d of %d files) ", q.finished, len(q.transferables)))
	q.bar.Start()

	for _, t := range q.transferables {
		wg.Add(1)
		apic <- t
	}

	<-workersReady
	q.authCond.Signal() // Signal the first goroutine to run
	close(apic)
	wg.Wait()

	close(q.transferc)
}

// processBatch processes the queue of transfers using the batch endpoint,
// making only one POST call for all objects. The results are then handed
// off to the transfer workers.
func (q *TransferQueue) processBatch() error {
	transfers := make([]*objectResource, 0, len(q.transferables))
	for _, t := range q.transferables {
		transfers = append(transfers, &objectResource{Oid: t.Oid(), Size: t.Size()})
	}

	objects, err := Batch(transfers)
	if err != nil {
		if isNotImplError(err) {
			tracerx.Printf("queue: batch not implemented, disabling")
			configFile := filepath.Join(LocalGitDir, "config")
			git.Config.SetLocal(configFile, "lfs.batch", "false")
		}

		return err
	}

	q.files = 0

	for _, o := range objects {
		if _, ok := o.Links[q.transferKind]; ok {
			// This object needs to be transfered
			if transfer, ok := q.transferables[o.Oid]; ok {
				q.files++
				q.wg.Add(1)
				transfer.SetObject(o)
				q.transferc <- transfer
			}
		}
	}

	close(q.transferc)
	q.bar.Prefix(fmt.Sprintf("(%d of %d files) ", q.finished, q.files))
	q.bar.Start()
	sendApiEvent(apiEventSuccess) // Wake up transfer workers
	return nil
}

// Process starts the transfer queue and displays a progress bar. Process will
// do individual or batch transfers depending on the Config.BatchTransfer() value.
// Process will transfer files sequentially or concurrently depending on the
// Concig.ConcurrentTransfers() value.
func (q *TransferQueue) Process() {
	q.bar = pb.New64(q.size)
	q.bar.SetUnits(pb.U_BYTES)
	q.bar.ShowBar = false

	// This goroutine collects errors returned from transfers
	go func() {
		for err := range q.errorc {
			q.errors = append(q.errors, err)
		}
	}()

	// This goroutine watches for apiEvents. In order to prevent multiple
	// credential requests from happening, the queue is processed sequentially
	// until an API request succeeds (meaning authenication has happened successfully).
	// Once the an API request succeeds, all worker goroutines are woken up and allowed
	// to process transfers. Once a success happens, this goroutine exits.
	go func() {
		for {
			event := <-apiEvent
			switch event {
			case apiEventSuccess:
				atomic.StoreInt32(&q.clientAuthorized, 1)
				q.authCond.Broadcast() // Wake all remaining goroutines
				return
			case apiEventFail:
				q.authCond.Signal() // Wake the next goroutine
			}
		}
	}()

	for i := 0; i < q.workers; i++ {
		// These are the worker goroutines that process transfers
		go func() {
			for transfer := range q.transferc {
				cb := func(total, read int64, current int) error {
					q.bar.Add(current)
					return nil
				}

				if err := transfer.Transfer(cb); err != nil {
					q.errorc <- err
				} else {
					oid := transfer.Oid()
					for _, c := range q.watchers {
						c <- oid
					}
				}

				f := atomic.AddInt64(&q.finished, 1)
				q.bar.Prefix(fmt.Sprintf("(%d of %d files) ", f, q.files))
				q.wg.Done()
			}
		}()
	}

	if Config.BatchTransfer() {
		if err := q.processBatch(); err != nil {
			q.processIndividual()
		}
	} else {
		q.processIndividual()
	}

	q.wg.Wait()
	close(q.errorc)
	for _, watcher := range q.watchers {
		close(watcher)
	}

	q.bar.Finish()
}

// Errors returns any errors encountered during transfer.
func (q *TransferQueue) Errors() []*WrappedError {
	return q.errors
}
<commit_msg>Resolved race condition<commit_after>package lfs

import (
	"fmt"
	"path/filepath"
	"sync"
	"sync/atomic"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/vendor/_nuts/github.com/cheggaaa/pb"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

type Transferable interface {
	Check() (*objectResource, *WrappedError)
	Transfer(CopyCallback) *WrappedError
	Object() *objectResource
	Oid() string
	Size() int64
	SetObject(*objectResource)
}

// TransferQueue provides a queue that will allow concurrent transfers.
type TransferQueue struct {
	transferc        chan Transferable
	errorc           chan *WrappedError
	watchers         []chan string
	errors           []*WrappedError
	wg               sync.WaitGroup
	workers          int
	files            int
	finished         int64
	size             int64
	authCond         *sync.Cond
	transferables    map[string]Transferable
	bar              *pb.ProgressBar
	clientAuthorized int32
	transferKind     string
}

// newTransferQueue builds a TransferQueue, allowing `workers` concurrent transfers.
func newTransferQueue(workers, files int) *TransferQueue {
	return &TransferQueue{
		transferc:     make(chan Transferable, files),
		errorc:        make(chan *WrappedError),
		watchers:      make([]chan string, 0),
		workers:       workers,
		files:         files,
		authCond:      sync.NewCond(&sync.Mutex{}),
		transferables: make(map[string]Transferable),
	}
}

// Add adds a Transferable to the transfer queue.
func (q *TransferQueue) Add(t Transferable) {
	q.transferables[t.Oid()] = t
}

// Watch returns a channel where the queue will write the OID of each transfer
// as it completes. The channel will be closed when the queue finishes processing.
func (q *TransferQueue) Watch() chan string {
	c := make(chan string, q.files)
	q.watchers = append(q.watchers, c)
	return c
}

// processIndividual processes the queue of transfers one at a time by making
// a POST call for each object, feeding the results to the transfer workers.
// If configured, the object transfers can still happen concurrently, the
// sequential nature here is only for the meta POST calls.
func (q *TransferQueue) processIndividual() {
	apic := make(chan Transferable, q.files)
	workersReady := make(chan int, q.workers)
	var wg sync.WaitGroup

	for i := 0; i < q.workers; i++ {
		go func() {
			for t := range apic {
				// If an API authorization has not occured, we wait until we're woken up.
				q.authCond.L.Lock()
				if atomic.LoadInt32(&q.clientAuthorized) == 0 {
					workersReady <- 1
					q.authCond.Wait()
				}
				q.authCond.L.Unlock()

				obj, err := t.Check()
				if err != nil {
					q.errorc <- err
					wg.Done()
					continue
				}
				if obj != nil {
					q.wg.Add(1)
					t.SetObject(obj)
					q.transferc <- t
				}
				wg.Done()
			}
		}()
	}

	q.bar.Prefix(fmt.Sprintf("(%d of %d files) ", q.finished, len(q.transferables)))
	q.bar.Start()

	for _, t := range q.transferables {
		wg.Add(1)
		apic <- t
	}

	go func() {
		wg.Wait()
		close(workersReady)
	}()

	<-workersReady
	q.authCond.L.Lock()
	q.authCond.Signal() // Signal the first goroutine to run
	q.authCond.L.Unlock()

	close(apic)
	for _ = range workersReady {
	}

	close(q.transferc)
}

// processBatch processes the queue of transfers using the batch endpoint,
// making only one POST call for all objects. The results are then handed
// off to the transfer workers.
func (q *TransferQueue) processBatch() error {
	transfers := make([]*objectResource, 0, len(q.transferables))
	for _, t := range q.transferables {
		transfers = append(transfers, &objectResource{Oid: t.Oid(), Size: t.Size()})
	}

	objects, err := Batch(transfers)
	if err != nil {
		if isNotImplError(err) {
			tracerx.Printf("queue: batch not implemented, disabling")
			configFile := filepath.Join(LocalGitDir, "config")
			git.Config.SetLocal(configFile, "lfs.batch", "false")
		}

		return err
	}

	q.files = 0

	for _, o := range objects {
		if _, ok := o.Links[q.transferKind]; ok {
			// This object needs to be transfered
			if transfer, ok := q.transferables[o.Oid]; ok {
				q.files++
				q.wg.Add(1)
				transfer.SetObject(o)
				q.transferc <- transfer
			}
		}
	}

	close(q.transferc)
	q.bar.Prefix(fmt.Sprintf("(%d of %d files) ", q.finished, q.files))
	q.bar.Start()
	sendApiEvent(apiEventSuccess) // Wake up transfer workers
	return nil
}

// Process starts the transfer queue and displays a progress bar. Process will
// do individual or batch transfers depending on the Config.BatchTransfer() value.
// Process will transfer files sequentially or concurrently depending on the
// Concig.ConcurrentTransfers() value.
func (q *TransferQueue) Process() {
	q.bar = pb.New64(q.size)
	q.bar.SetUnits(pb.U_BYTES)
	q.bar.ShowBar = false

	// This goroutine collects errors returned from transfers
	go func() {
		for err := range q.errorc {
			q.errors = append(q.errors, err)
		}
	}()

	// This goroutine watches for apiEvents. In order to prevent multiple
	// credential requests from happening, the queue is processed sequentially
	// until an API request succeeds (meaning authenication has happened successfully).
	// Once the an API request succeeds, all worker goroutines are woken up and allowed
	// to process transfers. Once a success happens, this goroutine exits.
	go func() {
		for {
			event := <-apiEvent
			switch event {
			case apiEventSuccess:
				atomic.StoreInt32(&q.clientAuthorized, 1)
				q.authCond.Broadcast() // Wake all remaining goroutines
				return
			case apiEventFail:
				q.authCond.Signal() // Wake the next goroutine
			}
		}
	}()

	for i := 0; i < q.workers; i++ {
		// These are the worker goroutines that process transfers
		go func() {
			for transfer := range q.transferc {
				cb := func(total, read int64, current int) error {
					q.bar.Add(current)
					return nil
				}

				if err := transfer.Transfer(cb); err != nil {
					q.errorc <- err
				} else {
					oid := transfer.Oid()
					for _, c := range q.watchers {
						c <- oid
					}
				}

				f := atomic.AddInt64(&q.finished, 1)
				q.bar.Prefix(fmt.Sprintf("(%d of %d files) ", f, q.files))
				q.wg.Done()
			}
		}()
	}

	if Config.BatchTransfer() {
		if err := q.processBatch(); err != nil {
			q.processIndividual()
		}
	} else {
		q.processIndividual()
	}

	q.wg.Wait()
	close(q.errorc)
	for _, watcher := range q.watchers {
		close(watcher)
	}

	q.bar.Finish()
}

// Errors returns any errors encountered during transfer.
func (q *TransferQueue) Errors() []*WrappedError {
	return q.errors
}
<|endoftext|>


<commit_before>/*
ImagePartition Action

This action creates an image file, partitions it and formats the filesystems.

Yaml syntax:
 - action: image-partition
   imagename: image_name
   imagesize: size
   partitiontype: gpt
   gpt_gap: offset
   partitions:
     <list of partitions>
   mountpoints:
     <list of mount points>

Mandatory properties:

- imagename -- the name of the image file.

- imagesize -- generated image size in human-readable form, examples: 100MB, 1GB, etc.

- partitiontype -- partition table type. Currently only 'gpt' and 'msdos'
partition tables are supported.

- gpt_gap -- shifting GPT allow to use this gap for bootloaders, for example if
U-Boot intersects with original GPT placement.
Only works if parted supports an extra argument to mklabel to specify the gpt offset.

- partitions -- list of partitions, at least one partition is needed.
Partition properties are described below.

- mountpoints -- list of mount points for partitions.
Properties for mount points are described below.

Yaml syntax for partitions:

   partitions:
     - name: label
	   name: partition name
	   fs: filesystem
	   start: offset
	   end: offset
	   flags: list of flags

Mandatory properties:

- name -- is used for referencing named partition for mount points
configuration (below) and label the filesystem located on this partition.

- fs -- filesystem type used for formatting.

'none' fs type should be used for partition without filesystem.

- start -- offset from beginning of the disk there the partition starts.

- end -- offset from beginning of the disk there the partition ends.

For 'start' and 'end' properties offset can be written in human readable
form -- '32MB', '1GB' or as disk percentage -- '100%'.

Optional properties:

- flags -- list of additional flags for partition compatible with parted(8)
'set' command.

Yaml syntax for mount points:

   mountpoints:
     - mountpoint: path
	   partition: partition label
	   options: list of options

Mandatory properties:

- partition -- partition name for mounting.

- mountpoint -- path in the target root filesystem where the named partition
should be mounted.

Optional properties:

- options -- list of options to be added to appropriate entry in fstab file.

Layout example for Raspberry PI 3:

 - action: image-partition
   imagename: "debian-rpi3.img"
   imagesize: 1GB
   partitiontype: msdos
   mountpoints:
     - mountpoint: /
       partition: root
     - mountpoint: /boot/firmware
       partition: firmware
       options: [ x-systemd.automount ]
   partitions:
     - name: firmware
       fs: vfat
       start: 0%
       end: 64MB
     - name: root
       fs: ext4
       start: 64MB
       end: 100%
       flags: [ boot ]
*/
package actions

import (
	"errors"
	"fmt"
	"github.com/docker/go-units"
	"github.com/go-debos/fakemachine"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"

	"github.com/go-debos/debos"
)

type Partition struct {
	number int
	Name   string
	Start  string
	End    string
	FS     string
	Flags  []string
	FSUUID string
}

type Mountpoint struct {
	Mountpoint string
	Partition  string
	Options    []string
	part       *Partition
}

type ImagePartitionAction struct {
	debos.BaseAction `yaml:",inline"`
	ImageName        string
	ImageSize        string
	PartitionType    string
	GptGap           string "gpt_gap"
	Partitions       []Partition
	Mountpoints      []Mountpoint
	size             int64
	usingLoop        bool
}

func (i *ImagePartitionAction) generateFSTab(context *debos.DebosContext) error {
	context.ImageFSTab.Reset()

	for _, m := range i.Mountpoints {
		options := []string{"defaults"}
		options = append(options, m.Options...)
		if m.part.FSUUID == "" {
			return fmt.Errorf("Missing fs UUID for partition %s!?!", m.part.Name)
		}
		context.ImageFSTab.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0\t0\n",
			m.part.FSUUID, m.Mountpoint, m.part.FS,
			strings.Join(options, ",")))
	}

	return nil
}

func (i *ImagePartitionAction) generateKernelRoot(context *debos.DebosContext) error {
	for _, m := range i.Mountpoints {
		if m.Mountpoint == "/" {
			if m.part.FSUUID == "" {
				return errors.New("No fs UUID for root partition !?!")
			}
			context.ImageKernelRoot = fmt.Sprintf("root=UUID=%s", m.part.FSUUID)
			break
		}
	}

	return nil
}

func (i ImagePartitionAction) getPartitionDevice(number int, context debos.DebosContext) string {
	suffix := "p"
	/* Check partition naming first: if used 'by-id'i naming convention */
	if strings.Contains(context.Image, "/disk/by-id/") {
		suffix = "-part"
	}

	/* If the iamge device has a digit as the last character, the partition
	 * suffix is p<number> else it's just <number> */
	last := context.Image[len(context.Image)-1]
	if last >= '0' && last <= '9' {
		return fmt.Sprintf("%s%s%d", context.Image, suffix, number)
	} else {
		return fmt.Sprintf("%s%d", context.Image, number)
	}
}

func (i ImagePartitionAction) PreMachine(context *debos.DebosContext, m *fakemachine.Machine,
	args *[]string) error {
	image, err := m.CreateImage(i.ImageName, i.size)
	if err != nil {
		return err
	}

	context.Image = image
	*args = append(*args, "--internal-image", image)
	return nil
}

func (i ImagePartitionAction) formatPartition(p *Partition, context debos.DebosContext) error {
	label := fmt.Sprintf("Formatting partition %d", p.number)
	path := i.getPartitionDevice(p.number, context)

	cmdline := []string{}
	switch p.FS {
	case "vfat":
		cmdline = append(cmdline, "mkfs.vfat", "-n", p.Name)
	case "btrfs":
		// Force formatting to prevent failure in case if partition was formatted already
		cmdline = append(cmdline, "mkfs.btrfs", "-L", p.Name, "-f")
	case "none":
	default:
		cmdline = append(cmdline, fmt.Sprintf("mkfs.%s", p.FS), "-L", p.Name)
	}

	if len(cmdline) != 0 {
		cmdline = append(cmdline, path)

		cmd := debos.Command{}
		if err := cmd.Run(label, cmdline...); err != nil {
			return err
		}
	}

	if p.FS != "none" {
		uuid, err := exec.Command("blkid", "-o", "value", "-s", "UUID", "-p", "-c", "none", path).Output()
		if err != nil {
			return fmt.Errorf("Failed to get uuid: %s", err)
		}
		p.FSUUID = strings.TrimSpace(string(uuid[:]))
	}

	return nil
}

func (i *ImagePartitionAction) PreNoMachine(context *debos.DebosContext) error {

	img, err := os.OpenFile(i.ImageName, os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		return fmt.Errorf("Couldn't open image file: %v", err)
	}

	err = img.Truncate(i.size)
	if err != nil {
		return fmt.Errorf("Couldn't resize image file: %v", err)
	}

	img.Close()

	loop, err := exec.Command("losetup", "-f", "--show", i.ImageName).Output()
	if err != nil {
		return fmt.Errorf("Failed to setup loop device")
	}
	context.Image = strings.TrimSpace(string(loop[:]))
	i.usingLoop = true

	return nil
}

func (i ImagePartitionAction) Run(context *debos.DebosContext) error {
	i.LogStart()

	command := []string{"parted", "-s", context.Image, "mklabel", i.PartitionType}
	if len(i.GptGap) > 0 {
		command = append(command, i.GptGap)
	}
	err := debos.Command{}.Run("parted", command...)
	if err != nil {
		return err
	}
	for idx, _ := range i.Partitions {
		p := &i.Partitions[idx]
		var name string
		if i.PartitionType == "gpt" {
			name = p.Name
		} else {
			name = "primary"
		}

		command := []string{"parted", "-a", "none", "-s", "--", context.Image, "mkpart", name}
		switch p.FS {
		case "vfat":
			command = append(command, "fat32")
		case "none":
		default:
			command = append(command, p.FS)
		}
		command = append(command, p.Start, p.End)

		err = debos.Command{}.Run("parted", command...)
		if err != nil {
			return err
		}

		if p.Flags != nil {
			for _, flag := range p.Flags {
				err = debos.Command{}.Run("parted", "parted", "-s", context.Image, "set",
					fmt.Sprintf("%d", p.number), flag, "on")
				if err != nil {
					return err
				}
			}
		}

		devicePath := i.getPartitionDevice(p.number, *context)
		// Give a chance for udevd to create proper symlinks
		err = debos.Command{}.Run("udevadm", "udevadm", "settle", "-t", "5",
			"-E", devicePath)
		if err != nil {
			return err
		}

		err = i.formatPartition(p, *context)
		if err != nil {
			return err
		}

		context.ImagePartitions = append(context.ImagePartitions,
			debos.Partition{p.Name, devicePath})
	}

	context.ImageMntDir = path.Join(context.Scratchdir, "mnt")
	os.MkdirAll(context.ImageMntDir, 0755)
	for _, m := range i.Mountpoints {
		dev := i.getPartitionDevice(m.part.number, *context)
		mntpath := path.Join(context.ImageMntDir, m.Mountpoint)
		os.MkdirAll(mntpath, 0755)
		err := syscall.Mount(dev, mntpath, m.part.FS, 0, "")
		if err != nil {
			return fmt.Errorf("%s mount failed: %v", m.part.Name, err)
		}
	}

	err = i.generateFSTab(context)
	if err != nil {
		return err
	}

	err = i.generateKernelRoot(context)
	if err != nil {
		return err
	}

	return nil
}

func (i ImagePartitionAction) Cleanup(context debos.DebosContext) error {
	for idx := len(i.Mountpoints) - 1; idx >= 0; idx-- {
		m := i.Mountpoints[idx]
		mntpath := path.Join(context.ImageMntDir, m.Mountpoint)
		syscall.Unmount(mntpath, 0)
	}

	if i.usingLoop {
		exec.Command("losetup", "-d", context.Image).Run()
	}

	return nil
}

func (i *ImagePartitionAction) Verify(context *debos.DebosContext) error {
	if len(i.GptGap) > 0 {
		log.Println("WARNING: special version of parted is needed for 'gpt_gap' option")
		if i.PartitionType != "gpt" {
			return fmt.Errorf("gpt_gap property could be used only with 'gpt' label")
		}
		// Just check if it contains correct value
		_, err := units.FromHumanSize(i.GptGap)
		if err != nil {
			return fmt.Errorf("Failed to parse GPT offset: %s", i.GptGap)
		}
	}

	num := 1
	for idx, _ := range i.Partitions {
		p := &i.Partitions[idx]
		p.number = num
		num++
		if p.Name == "" {
			return fmt.Errorf("Partition without a name")
		}
		if p.Start == "" {
			return fmt.Errorf("Partition %s missing start", p.Name)
		}
		if p.End == "" {
			return fmt.Errorf("Partition %s missing end", p.Name)
		}

		switch p.FS {
		case "fat32":
			p.FS = "vfat"
		case "":
			return fmt.Errorf("Partition %s missing fs type", p.Name)
		}
	}

	for idx, _ := range i.Mountpoints {
		m := &i.Mountpoints[idx]
		for pidx, _ := range i.Partitions {
			p := &i.Partitions[pidx]
			if m.Partition == p.Name {
				m.part = p
				break
			}
		}
		if m.part == nil {
			return fmt.Errorf("Couldn't fount partition for %s", m.Mountpoint)
		}
	}

	size, err := units.FromHumanSize(i.ImageSize)
	if err != nil {
		return fmt.Errorf("Failed to parse image size: %s", i.ImageSize)
	}

	i.size = size
	return nil
}
<commit_msg>Fix partition races while partitioning<commit_after>/*
ImagePartition Action

This action creates an image file, partitions it and formats the filesystems.

Yaml syntax:
 - action: image-partition
   imagename: image_name
   imagesize: size
   partitiontype: gpt
   gpt_gap: offset
   partitions:
     <list of partitions>
   mountpoints:
     <list of mount points>

Mandatory properties:

- imagename -- the name of the image file.

- imagesize -- generated image size in human-readable form, examples: 100MB, 1GB, etc.

- partitiontype -- partition table type. Currently only 'gpt' and 'msdos'
partition tables are supported.

- gpt_gap -- shifting GPT allow to use this gap for bootloaders, for example if
U-Boot intersects with original GPT placement.
Only works if parted supports an extra argument to mklabel to specify the gpt offset.

- partitions -- list of partitions, at least one partition is needed.
Partition properties are described below.

- mountpoints -- list of mount points for partitions.
Properties for mount points are described below.

Yaml syntax for partitions:

   partitions:
     - name: label
	   name: partition name
	   fs: filesystem
	   start: offset
	   end: offset
	   flags: list of flags

Mandatory properties:

- name -- is used for referencing named partition for mount points
configuration (below) and label the filesystem located on this partition.

- fs -- filesystem type used for formatting.

'none' fs type should be used for partition without filesystem.

- start -- offset from beginning of the disk there the partition starts.

- end -- offset from beginning of the disk there the partition ends.

For 'start' and 'end' properties offset can be written in human readable
form -- '32MB', '1GB' or as disk percentage -- '100%'.

Optional properties:

- flags -- list of additional flags for partition compatible with parted(8)
'set' command.

Yaml syntax for mount points:

   mountpoints:
     - mountpoint: path
	   partition: partition label
	   options: list of options

Mandatory properties:

- partition -- partition name for mounting.

- mountpoint -- path in the target root filesystem where the named partition
should be mounted.

Optional properties:

- options -- list of options to be added to appropriate entry in fstab file.

Layout example for Raspberry PI 3:

 - action: image-partition
   imagename: "debian-rpi3.img"
   imagesize: 1GB
   partitiontype: msdos
   mountpoints:
     - mountpoint: /
       partition: root
     - mountpoint: /boot/firmware
       partition: firmware
       options: [ x-systemd.automount ]
   partitions:
     - name: firmware
       fs: vfat
       start: 0%
       end: 64MB
     - name: root
       fs: ext4
       start: 64MB
       end: 100%
       flags: [ boot ]
*/
package actions

import (
	"errors"
	"fmt"
	"github.com/docker/go-units"
	"github.com/go-debos/fakemachine"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/go-debos/debos"
)

type Partition struct {
	number int
	Name   string
	Start  string
	End    string
	FS     string
	Flags  []string
	FSUUID string
}

type Mountpoint struct {
	Mountpoint string
	Partition  string
	Options    []string
	part       *Partition
}

type ImagePartitionAction struct {
	debos.BaseAction `yaml:",inline"`
	ImageName        string
	ImageSize        string
	PartitionType    string
	GptGap           string "gpt_gap"
	Partitions       []Partition
	Mountpoints      []Mountpoint
	size             int64
	usingLoop        bool
}

func (i *ImagePartitionAction) generateFSTab(context *debos.DebosContext) error {
	context.ImageFSTab.Reset()

	for _, m := range i.Mountpoints {
		options := []string{"defaults"}
		options = append(options, m.Options...)
		if m.part.FSUUID == "" {
			return fmt.Errorf("Missing fs UUID for partition %s!?!", m.part.Name)
		}
		context.ImageFSTab.WriteString(fmt.Sprintf("UUID=%s\t%s\t%s\t%s\t0\t0\n",
			m.part.FSUUID, m.Mountpoint, m.part.FS,
			strings.Join(options, ",")))
	}

	return nil
}

func (i *ImagePartitionAction) generateKernelRoot(context *debos.DebosContext) error {
	for _, m := range i.Mountpoints {
		if m.Mountpoint == "/" {
			if m.part.FSUUID == "" {
				return errors.New("No fs UUID for root partition !?!")
			}
			context.ImageKernelRoot = fmt.Sprintf("root=UUID=%s", m.part.FSUUID)
			break
		}
	}

	return nil
}

func (i ImagePartitionAction) getPartitionDevice(number int, context debos.DebosContext) string {
	/* Always look up canonical device as udev might not generate the by-id
	 * symlinks while there is an flock on /dev/vda */
	device, _ := filepath.EvalSymlinks(context.Image)

	suffix := "p"
	/* Check partition naming first: if used 'by-id'i naming convention */
	if strings.Contains(device, "/disk/by-id/") {
		suffix = "-part"
	}

	/* If the iamge device has a digit as the last character, the partition
	 * suffix is p<number> else it's just <number> */
	last := device[len(device)-1]
	if last >= '0' && last <= '9' {
		return fmt.Sprintf("%s%s%d", device, suffix, number)
	} else {
		return fmt.Sprintf("%s%d", device, number)
	}
}

func (i ImagePartitionAction) PreMachine(context *debos.DebosContext, m *fakemachine.Machine,
	args *[]string) error {
	image, err := m.CreateImage(i.ImageName, i.size)
	if err != nil {
		return err
	}

	context.Image = image
	*args = append(*args, "--internal-image", image)
	return nil
}

func (i ImagePartitionAction) formatPartition(p *Partition, context debos.DebosContext) error {
	label := fmt.Sprintf("Formatting partition %d", p.number)
	path := i.getPartitionDevice(p.number, context)

	cmdline := []string{}
	switch p.FS {
	case "vfat":
		cmdline = append(cmdline, "mkfs.vfat", "-n", p.Name)
	case "btrfs":
		// Force formatting to prevent failure in case if partition was formatted already
		cmdline = append(cmdline, "mkfs.btrfs", "-L", p.Name, "-f")
	case "none":
	default:
		cmdline = append(cmdline, fmt.Sprintf("mkfs.%s", p.FS), "-L", p.Name)
	}

	if len(cmdline) != 0 {
		cmdline = append(cmdline, path)

		cmd := debos.Command{}
		if err := cmd.Run(label, cmdline...); err != nil {
			return err
		}
	}

	if p.FS != "none" {
		uuid, err := exec.Command("blkid", "-o", "value", "-s", "UUID", "-p", "-c", "none", path).Output()
		if err != nil {
			return fmt.Errorf("Failed to get uuid: %s", err)
		}
		p.FSUUID = strings.TrimSpace(string(uuid[:]))
	}

	return nil
}

func (i *ImagePartitionAction) PreNoMachine(context *debos.DebosContext) error {

	img, err := os.OpenFile(i.ImageName, os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		return fmt.Errorf("Couldn't open image file: %v", err)
	}

	err = img.Truncate(i.size)
	if err != nil {
		return fmt.Errorf("Couldn't resize image file: %v", err)
	}

	img.Close()

	loop, err := exec.Command("losetup", "-f", "--show", i.ImageName).Output()
	if err != nil {
		return fmt.Errorf("Failed to setup loop device")
	}
	context.Image = strings.TrimSpace(string(loop[:]))
	i.usingLoop = true

	return nil
}

func (i ImagePartitionAction) Run(context *debos.DebosContext) error {
	i.LogStart()

	/* Exclusively Lock image device file to prevent udev from triggering
	 * partition rescans, which cause confusion as some time asynchronously the
	 * partition device might disappear and reappear due to that! */
	imageFD, err := os.Open(context.Image)
	if err != nil {
		return err
	}
	/* Defer will keep the fd open until the function returns, at which points
	 * the filesystems will have been mounted protecting from more udev funnyness
	 */
	defer imageFD.Close()

	err = syscall.Flock(int(imageFD.Fd()), syscall.LOCK_EX)
	if err != nil {
		return err
	}

	command := []string{"parted", "-s", context.Image, "mklabel", i.PartitionType}
	if len(i.GptGap) > 0 {
		command = append(command, i.GptGap)
	}
	err = debos.Command{}.Run("parted", command...)
	if err != nil {
		return err
	}
	for idx, _ := range i.Partitions {
		p := &i.Partitions[idx]
		var name string
		if i.PartitionType == "gpt" {
			name = p.Name
		} else {
			name = "primary"
		}

		command := []string{"parted", "-a", "none", "-s", "--", context.Image, "mkpart", name}
		switch p.FS {
		case "vfat":
			command = append(command, "fat32")
		case "none":
		default:
			command = append(command, p.FS)
		}
		command = append(command, p.Start, p.End)

		err = debos.Command{}.Run("parted", command...)
		if err != nil {
			return err
		}

		if p.Flags != nil {
			for _, flag := range p.Flags {
				err = debos.Command{}.Run("parted", "parted", "-s", context.Image, "set",
					fmt.Sprintf("%d", p.number), flag, "on")
				if err != nil {
					return err
				}
			}
		}

		devicePath := i.getPartitionDevice(p.number, *context)

		err = i.formatPartition(p, *context)
		if err != nil {
			return err
		}

		context.ImagePartitions = append(context.ImagePartitions,
			debos.Partition{p.Name, devicePath})
	}

	context.ImageMntDir = path.Join(context.Scratchdir, "mnt")
	os.MkdirAll(context.ImageMntDir, 0755)
	for _, m := range i.Mountpoints {
		dev := i.getPartitionDevice(m.part.number, *context)
		mntpath := path.Join(context.ImageMntDir, m.Mountpoint)
		os.MkdirAll(mntpath, 0755)
		err := syscall.Mount(dev, mntpath, m.part.FS, 0, "")
		if err != nil {
			return fmt.Errorf("%s mount failed: %v", m.part.Name, err)
		}
	}

	err = i.generateFSTab(context)
	if err != nil {
		return err
	}

	err = i.generateKernelRoot(context)
	if err != nil {
		return err
	}

	return nil
}

func (i ImagePartitionAction) Cleanup(context debos.DebosContext) error {
	for idx := len(i.Mountpoints) - 1; idx >= 0; idx-- {
		m := i.Mountpoints[idx]
		mntpath := path.Join(context.ImageMntDir, m.Mountpoint)
		syscall.Unmount(mntpath, 0)
	}

	if i.usingLoop {
		exec.Command("losetup", "-d", context.Image).Run()
	}

	return nil
}

func (i *ImagePartitionAction) Verify(context *debos.DebosContext) error {
	if len(i.GptGap) > 0 {
		log.Println("WARNING: special version of parted is needed for 'gpt_gap' option")
		if i.PartitionType != "gpt" {
			return fmt.Errorf("gpt_gap property could be used only with 'gpt' label")
		}
		// Just check if it contains correct value
		_, err := units.FromHumanSize(i.GptGap)
		if err != nil {
			return fmt.Errorf("Failed to parse GPT offset: %s", i.GptGap)
		}
	}

	num := 1
	for idx, _ := range i.Partitions {
		p := &i.Partitions[idx]
		p.number = num
		num++
		if p.Name == "" {
			return fmt.Errorf("Partition without a name")
		}
		if p.Start == "" {
			return fmt.Errorf("Partition %s missing start", p.Name)
		}
		if p.End == "" {
			return fmt.Errorf("Partition %s missing end", p.Name)
		}

		switch p.FS {
		case "fat32":
			p.FS = "vfat"
		case "":
			return fmt.Errorf("Partition %s missing fs type", p.Name)
		}
	}

	for idx, _ := range i.Mountpoints {
		m := &i.Mountpoints[idx]
		for pidx, _ := range i.Partitions {
			p := &i.Partitions[pidx]
			if m.Partition == p.Name {
				m.part = p
				break
			}
		}
		if m.part == nil {
			return fmt.Errorf("Couldn't fount partition for %s", m.Mountpoint)
		}
	}

	size, err := units.FromHumanSize(i.ImageSize)
	if err != nil {
		return fmt.Errorf("Failed to parse image size: %s", i.ImageSize)
	}

	i.size = size
	return nil
}
<|endoftext|>


<commit_before><commit_msg>auto-updated agent/uiserver/bindata_assetfs.go from commit daf897f1a<commit_after><|endoftext|>


<commit_before><commit_msg>auto-updated agent/uiserver/bindata_assetfs.go from commit ace794d21<commit_after><|endoftext|>


<commit_before>package acme

type dvsniChallenge struct{}

func (s *dvsniChallenge) CanSolve() bool {
	return false
}

func (s *dvsniChallenge) Solve(challenge challenge, domain string) {

}
<commit_msg>Implement new interface with DVSNI<commit_after>package acme

type dvsniChallenge struct{}

func (s *dvsniChallenge) CanSolve() bool {
	return false
}

func (s *dvsniChallenge) Solve(challenge challenge, domain string) error {
	return nil
}
<|endoftext|>


<commit_before>package actions

import (
	"errors"

	"code.google.com/p/cascadia"

	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"

	"golang.org/x/net/html"

	"github.com/zaibon/ircbot"
)

type TitleExtract struct {
	selector cascadia.Selector
}

func NewTitleExtract() *TitleExtract {
	return &TitleExtract{
		selector: cascadia.MustCompile("title"),
	}
}

func (u *TitleExtract) Command() []string {
	return []string{
		"PRIVMSG",
	}
}

func (u *TitleExtract) Usage() string {
	return ""
}

func (u *TitleExtract) Do(b *ircbot.IrcBot, m *ircbot.IrcMsg) {
	u.do(b, m)
}

func (u *TitleExtract) do(b *ircbot.IrcBot, m *ircbot.IrcMsg) {
	for _, word := range m.Trailing {

		if !strings.Contains(word, "http") {
			continue
		}

		URL, err := url.Parse(word)
		if err != nil {
			fmt.Println("err parse url: ", err)
			continue
		}

		go func() {
			fmt.Println("INFO : start extractTitle,", URL.String())
			title, err := extractTitle(URL.String(), u.selector)
			if err == nil {
				b.Say(m.Channel(), title)
			}
			fmt.Println("INFO : title %s,", title)
		}()
	}
}

func extractTitle(url string, selector cascadia.Selector) (string, error) {
	resp, err := http.Get(url)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	contentType := resp.Header.Get("Content-Type")

	switch {
	case strings.Contains(contentType, "text/html"):
		return cssSelectHTML(resp.Body, selector)
	default:
		return "", fmt.Errorf("mime not supported")
	}
}

func cssSelectHTML(r io.Reader, selector cascadia.Selector) (string, error) {
	doc, err := html.Parse(r)
	if err != nil {
		return "", err
	}

	var title string
	titleNode := selector.MatchFirst(doc)
	if titleNode != nil && titleNode.FirstChild != nil {
		title = titleNode.FirstChild.Data
	}
	if title == "" {
		return title, errors.New("no Title")
	}
	return strings.TrimSpace(title), nil
}
<commit_msg>use goquerry to extractTitle<commit_after>package actions

import (
	"github.com/PuerkitoBio/goquery"

	"code.google.com/p/cascadia"

	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"

	"github.com/zaibon/ircbot"
)

type TitleExtract struct {
	selector cascadia.Selector
}

func NewTitleExtract() *TitleExtract {
	return &TitleExtract{
		selector: cascadia.MustCompile("title"),
	}
}

func (u *TitleExtract) Command() []string {
	return []string{
		"PRIVMSG",
	}
}

func (u *TitleExtract) Usage() string {
	return ""
}

func (u *TitleExtract) Do(b *ircbot.IrcBot, m *ircbot.IrcMsg) {
	u.do(b, m)
}

func (u *TitleExtract) do(b *ircbot.IrcBot, m *ircbot.IrcMsg) {
	for _, word := range m.Trailing {

		if !strings.Contains(word, "http") {
			continue
		}

		URL, err := url.Parse(word)
		if err != nil {
			fmt.Println("err parse url: ", err)
			continue
		}

		go func() {
			fmt.Println("INFO : start extractTitle,", URL.String())
			title, err := extractTitle(URL.String(), u.selector)
			if err == nil {
				b.Say(m.Channel(), title)
			}
			fmt.Println("INFO : title %s,", title)
		}()
	}
}

func extractTitle(url string, selector cascadia.Selector) (string, error) {
	resp, err := http.Get(url)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	contentType := resp.Header.Get("Content-Type")

	switch {
	case strings.Contains(contentType, "text/html"):
		return cssSelectHTML(resp.Body, selector)
	default:
		return "", fmt.Errorf("mime not supported")
	}

}

func cssSelectHTML(r io.Reader, selector cascadia.Selector) (string, error) {
	doc, err := goquery.NewDocumentFromReader(r)
	if err != nil {
		return "", err
	}
	title := ""
	doc.Find("title").Each(func(i int, s *goquery.Selection) {
		title = title + s.Text()
	})
	return title, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Brian J. Downs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package openweathermap

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

// ForecastSys area population
type ForecastSys struct {
	Population int `json:"population"`
}

// Temperature holds returned termperate sure stats
type Temperature struct {
	Day   float64 `json:"day"`
	Min   float64 `json:"min"`
	Max   float64 `json:"max"`
	Night float64 `json:"night"`
	Eve   float64 `json:"eve"`
	Morn  float64 `json:"morn"`
}

// City data for given location
type City struct {
	ID         int         `json:"id"`
	Name       string      `json:"name"`
	Coord      Coordinates `json:"coord"`
	Country    string      `json:"country"`
	Population int         `json:"population"`
	Sys        ForecastSys `json:"sys"`
}

// ForecastWeatherList holds specific query data
type ForecastWeatherList struct {
	Dt       int         `json:"dt"`
	Temp     Temperature `json:"temp"`
	Pressure float64     `json:"pressure"`
	Humidity int         `json:"humidity"`
	Weather  []Weather   `json:"weather"`
	Speed    float64     `json:"speed"`
	Deg      int         `json:"deg"`
	Clouds   int         `json:"clouds"`
	Snow     float64     `json:"snow"`
	Rain     float64     `json:"rain"`
}

// ForecastWeatherData will hold returned data from queries
type ForecastWeatherData struct {
	COD     string                `json:"cod"`
	Message float64               `json:"message"`
	City    City                  `json:"city"`
	Cnt     int                   `json:"cnt"`
	List    []ForecastWeatherList `json:"list"`
	Unit    string
	Lang    string
	Key     string
}

// NewForecast returns a new HistoricalWeatherData pointer with
// the supplied arguments.
func NewForecast(unit, lang string) (*ForecastWeatherData, error) {
	unitChoice := strings.ToUpper(unit)
	langChoice := strings.ToUpper(lang)

	f := &ForecastWeatherData{}

	if ValidDataUnit(unitChoice) {
		f.Unit = DataUnits[unitChoice]
	} else {
		return nil, errUnitUnavailable
	}

	if ValidLangCode(langChoice) {
		f.Lang = langChoice
	} else {
		return nil, errLangUnavailable
	}

	f.Key = getKey()

	return f, nil
}

// DailyByName will provide a forecast for the location given for the
// number of days given.
func (f *ForecastWeatherData) DailyByName(location string, days int) error {
	var err error
	var response *http.Response

	response, err = http.Get(fmt.Sprintf(forecastBase, f.Key, fmt.Sprintf("%s=%s","q", url.QueryEscape(location)), f.Unit, f.Lang, days))
	if err != nil {
		return err
	}
	defer response.Body.Close()

	if err = json.NewDecoder(response.Body).Decode(&f); err != nil {
		return err
	}

	return nil
}

// DailyByCoordinates will provide a forecast for the coordinates ID give
// for the number of days given.
func (f *ForecastWeatherData) DailyByCoordinates(location *Coordinates, days int) error {
 	println(
		string(fmt.Sprintf(forecastBase, f.Key, fmt.Sprintf("lat=%f&lon=%f", location.Latitude, location.Longitude), f.Unit, f.Lang, days)),
	)

	response, err := http.Get(fmt.Sprintf(forecastBase, f.Key, fmt.Sprintf("lat=%f&lon=%f", location.Latitude, location.Longitude), f.Unit, f.Lang, days))
	if err != nil {
		return err
	}
	defer response.Body.Close()

	if err = json.NewDecoder(response.Body).Decode(&f); err != nil {
		return err
	}

	return nil
}

// DailyByID will provide a forecast for the location ID give for the
// number of days given.
func (f *ForecastWeatherData) DailyByID(id, days int) error {
	response, err := http.Get(fmt.Sprintf(forecastBase, f.Key, fmt.Sprintf("%s=%s","id", strconv.Itoa(id)), f.Unit, f.Lang, days))
	if err != nil {
		return err
	}
	defer response.Body.Close()

	if err = json.NewDecoder(response.Body).Decode(&f); err != nil {
		return err
	}

	return nil
}
<commit_msg>remove debugging logs<commit_after>// Copyright 2015 Brian J. Downs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package openweathermap

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

// ForecastSys area population
type ForecastSys struct {
	Population int `json:"population"`
}

// Temperature holds returned termperate sure stats
type Temperature struct {
	Day   float64 `json:"day"`
	Min   float64 `json:"min"`
	Max   float64 `json:"max"`
	Night float64 `json:"night"`
	Eve   float64 `json:"eve"`
	Morn  float64 `json:"morn"`
}

// City data for given location
type City struct {
	ID         int         `json:"id"`
	Name       string      `json:"name"`
	Coord      Coordinates `json:"coord"`
	Country    string      `json:"country"`
	Population int         `json:"population"`
	Sys        ForecastSys `json:"sys"`
}

// ForecastWeatherList holds specific query data
type ForecastWeatherList struct {
	Dt       int         `json:"dt"`
	Temp     Temperature `json:"temp"`
	Pressure float64     `json:"pressure"`
	Humidity int         `json:"humidity"`
	Weather  []Weather   `json:"weather"`
	Speed    float64     `json:"speed"`
	Deg      int         `json:"deg"`
	Clouds   int         `json:"clouds"`
	Snow     float64     `json:"snow"`
	Rain     float64     `json:"rain"`
}

// ForecastWeatherData will hold returned data from queries
type ForecastWeatherData struct {
	COD     string                `json:"cod"`
	Message float64               `json:"message"`
	City    City                  `json:"city"`
	Cnt     int                   `json:"cnt"`
	List    []ForecastWeatherList `json:"list"`
	Unit    string
	Lang    string
	Key     string
}

// NewForecast returns a new HistoricalWeatherData pointer with
// the supplied arguments.
func NewForecast(unit, lang string) (*ForecastWeatherData, error) {
	unitChoice := strings.ToUpper(unit)
	langChoice := strings.ToUpper(lang)

	f := &ForecastWeatherData{}

	if ValidDataUnit(unitChoice) {
		f.Unit = DataUnits[unitChoice]
	} else {
		return nil, errUnitUnavailable
	}

	if ValidLangCode(langChoice) {
		f.Lang = langChoice
	} else {
		return nil, errLangUnavailable
	}

	f.Key = getKey()

	return f, nil
}

// DailyByName will provide a forecast for the location given for the
// number of days given.
func (f *ForecastWeatherData) DailyByName(location string, days int) error {
	var err error
	var response *http.Response

	response, err = http.Get(fmt.Sprintf(forecastBase, f.Key, fmt.Sprintf("%s=%s","q", url.QueryEscape(location)), f.Unit, f.Lang, days))
	if err != nil {
		return err
	}
	defer response.Body.Close()

	if err = json.NewDecoder(response.Body).Decode(&f); err != nil {
		return err
	}

	return nil
}

// DailyByCoordinates will provide a forecast for the coordinates ID give
// for the number of days given.
func (f *ForecastWeatherData) DailyByCoordinates(location *Coordinates, days int) error {
	response, err := http.Get(fmt.Sprintf(forecastBase, f.Key, fmt.Sprintf("lat=%f&lon=%f", location.Latitude, location.Longitude), f.Unit, f.Lang, days))
	if err != nil {
		return err
	}
	defer response.Body.Close()

	if err = json.NewDecoder(response.Body).Decode(&f); err != nil {
		return err
	}

	return nil
}

// DailyByID will provide a forecast for the location ID give for the
// number of days given.
func (f *ForecastWeatherData) DailyByID(id, days int) error {
	response, err := http.Get(fmt.Sprintf(forecastBase, f.Key, fmt.Sprintf("%s=%s","id", strconv.Itoa(id)), f.Unit, f.Lang, days))
	if err != nil {
		return err
	}
	defer response.Body.Close()

	if err = json.NewDecoder(response.Body).Decode(&f); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package configurator

import (
	"fmt"
	"os"
	"reflect"
	"testing"

	// jww "github.com/spf13/jwalterweatherman"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/assert"
)

func TestConfigLoadFailureScenarios(t *testing.T) {
	type testConfig struct {
		Config
		Value string
	}

	// Pass a struct directly instead of pointer reference
	config := testConfig{}
	err := config.Load(config)

	assert.NotNil(t, err)
	assert.Equal(t, err, ErrNotStructPointer)

	// Pass a non-struct by reference
	badValue := []int{1, 2, 3}
	c := Config{}
	err = c.Load(&badValue)

	assert.NotNil(t, err)
	assert.Equal(t, err, ErrNotStruct)
}

func TestLoadFromDefaultsSuccess(t *testing.T) {
	type testEnvConfigWithDefaults struct {
		Config

		// Annotation your properties with where config data can come from
		Environment string  `env:"APP_ENV" default:"development"`
		Host        string  `env:"APP_HOST" default:"localhost"`
		Port        int     `env:"APP_PORT" default:"3306"`
		TestFloat   float32 `env:"APP_FLOAT" default:"1.5"`
		TestBool    bool    `env:"APP_BOOL" default:"true"`
	}

	config := testEnvConfigWithDefaults{}
	os.Clearenv()

	err := config.Load(&config)
	assert.Nil(t, err)

	assert.Equal(t, "development", config.Environment)
	assert.Equal(t, "localhost", config.Host)
	assert.Equal(t, 3306, config.Port)
	assert.Equal(t, float32(1.5), config.TestFloat)
	assert.True(t, config.TestBool)
}

func TestLoadFromDefaultsWithoutOverridingAlreadySetValues(t *testing.T) {
	type testEnvConfigWithDefaults struct {
		Config

		// Annotation your properties with where config data can come from
		Environment string  `env:"APP_ENV" default:"development"`
		Host        string  `env:"APP_HOST" default:"localhost"`
		Port        int     `env:"APP_PORT" default:"3306"`
		TestFloat   float32 `env:"APP_FLOAT" default:"1.5"`
		TestBool    bool    `env:"APP_BOOL" default:"true"`
		Counter     uint32  `env:"APP_COUNTER" default:"1"`
	}

	config := testEnvConfigWithDefaults{}
	config.Host = "testhost"
	config.Port = 1000
	os.Clearenv()

	err := config.Load(&config)
	assert.Nil(t, err)

	assert.Equal(t, "development", config.Environment)
	assert.Equal(t, "testhost", config.Host)
	assert.Equal(t, 1000, config.Port)
	assert.Equal(t, uint32(1), config.Counter)
	assert.Equal(t, float32(1.5), config.TestFloat)
	assert.True(t, config.TestBool)
}

func TestLoadFromDefaultsFailureBadFloatDefault(t *testing.T) {
	type badDefaults struct {
		Config
		TestFloat float32 `env:"APP_FLOAT" default:"badvalue"`
	}

	config := badDefaults{}
	os.Clearenv()

	err := config.Load(&config)
	assert.NotNil(t, err)
}

func TestLoadFromDefaultsFailureBadBoolDefault(t *testing.T) {
	type badDefaults struct {
		Config
		TestBool bool `env:"APP_BOOL" default:"badvalue"`
	}

	config := badDefaults{}
	os.Clearenv()

	err := config.Load(&config)
	assert.NotNil(t, err)
}

func TestLoadFromDefaultsFailureBadIntDefault(t *testing.T) {
	type badDefaults struct {
		Config
		TestPort int `env:"APP_PORT" default:"badvalue"`
	}

	config := badDefaults{}
	os.Clearenv()

	err := config.Load(&config)
	assert.NotNil(t, err)
}

func TestLoadFromEnvVariablesSuccess(t *testing.T) {
	type testEnvConfigWithDefaults struct {
		Config

		// Annotation your properties with where config data can come from
		Environment string  `env:"APP_ENV" default:"development"`
		Host        string  `env:"APP_HOST" default:"localhost"`
		Port        int     `env:"APP_PORT" default:"3306"`
		TestFloat   float32 `env:"APP_FLOAT" default:"1.5"`
		TestBool    bool    `env:"APP_BOOL" default:"true"`
	}

	config := testEnvConfigWithDefaults{}
	os.Clearenv()
	os.Setenv("APP_ENV", "dev")
	os.Setenv("APP_HOST", "127.0.0.1")
	os.Setenv("APP_PORT", "4000")
	os.Setenv("APP_FLOAT", "3.14159")
	os.Setenv("APP_BOOL", "false")

	err := config.Load(&config)
	assert.Nil(t, err)

	assert.Equal(t, "dev", config.Environment)
	assert.Equal(t, "127.0.0.1", config.Host)
	assert.Equal(t, 4000, config.Port)
	assert.Equal(t, float32(3.14159), config.TestFloat)
	assert.False(t, config.TestBool)
}

func TestLoadConfigFromFlagsSuccess(t *testing.T) {
	type testFlagConfig struct {
		Config

		// Annotation your properties with where config data can come from
		Environment string  `flag:"env" default:"development"`
		Host        string  `flag:"host" default:"localhost"`
		Port        int     `flag:"port" default:"3306"`
		Version     float32 `flag:"version" default:"1.5"`
		TestBool    bool    `flag:"testbool" default:"false"`
	}

	config := testFlagConfig{}
	config.viper = viper.New()
	os.Clearenv()

	ptrRef := reflect.ValueOf(&config)
	structRef := ptrRef.Elem()
	flagValues := parseFlagValues(structRef)
	flagSet := config.bindFlagValues(flagValues)

	fmt.Println("flag vals", flagValues)

	var expectedFlagValues = map[string]string{
		"env":      "dev",
		"host":     "127.0.0.1",
		"port":     "4000",
		"version":  "3.14159",
		"testbool": "true",
	}

	// Update the flag values with actual
	flagSet.VisitAll(func(flag *pflag.Flag) {
		flag.Value.Set(expectedFlagValues[flag.Name])
		flag.Changed = true
	})

	defaultValues := parseDefaultValues(structRef)
	config.populateDefaults(defaultValues)
	config.populateConfigStruct(structRef)

	assert.Equal(t, "dev", config.Environment)
	assert.Equal(t, "127.0.0.1", config.Host)
	assert.Equal(t, 4000, config.Port)
	assert.Equal(t, float32(3.14159), config.Version)
	assert.True(t, config.TestBool)
}

func TestLoadConfigFromFlagsFailureUseDefaults(t *testing.T) {
	type testFlagConfig struct {
		Config

		// Annotation your properties with where config data can come from
		Environment string  `flag:"env1" default:"development"`
		Host        string  `flag:"host1" default:"localhost"`
		Port        int     `flag:"port1" default:"3306"`
		Version     float32 `flag:"version1" default:"1.5"`
		TestBool    bool    `flag:"testbool1" default:"false"`
	}

	config := testFlagConfig{}
	config.Load(&config)

	// Config should be loaded with defaults
	assert.Equal(t, "development", config.Environment)
	assert.Equal(t, "localhost", config.Host)
	assert.Equal(t, 3306, config.Port)
	assert.Equal(t, float32(1.5), config.Version)
	assert.False(t, config.TestBool)
}

func TestLoadConfigFromFlagsFailureBadValues(t *testing.T) {
	type testFlagConfig struct {
		Config

		// Annotation your properties with where config data can come from
		Environment string  `flag:"env2" default:"development"`
		Host        string  `flag:"host2" default:"localhost"`
		Port        int     `flag:"port2" default:"3306"`
		Version     float32 `flag:"version2" default:"1.5"`
		TestBool    bool    `flag:"testbool2" default:"false"`
	}

	config := testFlagConfig{}
	config.viper = viper.New()
	os.Clearenv()

	ptrRef := reflect.ValueOf(&config)
	structRef := ptrRef.Elem()
	flagValues := parseFlagValues(structRef)
	flagSet := config.bindFlagValues(flagValues)

	var expectedFlagValues = map[string]string{
		"env2":      "dev",
		"host2":     "127.0.0.1",
		"port2":     "asb",
		"version2":  "3.14159",
		"testbool2": "true",
	}

	// Update the flag values with actual
	flagSet.VisitAll(func(flag *pflag.Flag) {
		flag.Value.Set(expectedFlagValues[flag.Name])
		flag.Changed = true
	})

	defaultValues := parseDefaultValues(structRef)
	config.populateDefaults(defaultValues)
	err := config.populateConfigStruct(structRef)

	assert.NotNil(t, err)
}

func TestLoadConfigFromFileSuccess(t *testing.T) {
	type testFileConfig struct {
		Config

		Environment string  `file:"env" default:"development"`
		Host        string  `file:"host" default:"localhost"`
		Port        int     `file:"port" default:"3306"`
		Version     float32 `file:"version" default:"1.5"`
		TestBool    bool    `file:"enabled" default:"false"`
	}

	config := testFileConfig{}
	config.FileName = "good_config"
	config.FilePaths = []string{
		"./_test/",
	}

	err := config.Load(&config)
	assert.Nil(t, err)

	assert.Equal(t, "development", config.Environment)
	assert.Equal(t, "127.0.0.1", config.Host)
	assert.Equal(t, 3000, config.Port)
	assert.Equal(t, float32(2), config.Version)
	assert.True(t, config.TestBool)
}

func TestLoadConfigFromFileFailureBadValue(t *testing.T) {
	type testFileConfig struct {
		Config

		Environment string  `file:"env" default:"development"`
		Host        uint32  `file:"host" default:"3000"` // changed host to uint32, while it's a string in JSON file
		Port        int     `file:"port" default:"3306"`
		Version     float32 `file:"version" default:"1.5"`
		TestBool    bool    `file:"enabled" default:"false"`
	}

	config := testFileConfig{}
	config.FileName = "good_config"
	config.FilePaths = []string{
		"./_test/",
	}

	err := config.Load(&config)
	assert.NotNil(t, err)

}
<commit_msg>Removing config file test folder, now creating test files on-demand<commit_after>package configurator

import (
	"fmt"
	"os"
	"reflect"
	"testing"

	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/assert"
)

func TestConfigLoadFailureScenarios(t *testing.T) {
	type testConfig struct {
		Config
		Value string
	}

	// Pass a struct directly instead of pointer reference
	config := testConfig{}
	err := config.Load(config)

	assert.NotNil(t, err)
	assert.Equal(t, err, ErrNotStructPointer)

	// Pass a non-struct by reference
	badValue := []int{1, 2, 3}
	c := Config{}
	err = c.Load(&badValue)

	assert.NotNil(t, err)
	assert.Equal(t, err, ErrNotStruct)
}

func TestLoadFromDefaultsSuccess(t *testing.T) {
	type testEnvConfigWithDefaults struct {
		Config

		// Annotation your properties with where config data can come from
		Environment string  `env:"APP_ENV" default:"development"`
		Host        string  `env:"APP_HOST" default:"localhost"`
		Port        int     `env:"APP_PORT" default:"3306"`
		TestFloat   float32 `env:"APP_FLOAT" default:"1.5"`
		TestBool    bool    `env:"APP_BOOL" default:"true"`
	}

	config := testEnvConfigWithDefaults{}
	os.Clearenv()

	err := config.Load(&config)
	assert.Nil(t, err)

	assert.Equal(t, "development", config.Environment)
	assert.Equal(t, "localhost", config.Host)
	assert.Equal(t, 3306, config.Port)
	assert.Equal(t, float32(1.5), config.TestFloat)
	assert.True(t, config.TestBool)
}

func TestLoadFromDefaultsWithoutOverridingAlreadySetValues(t *testing.T) {
	type testEnvConfigWithDefaults struct {
		Config

		// Annotation your properties with where config data can come from
		Environment string  `env:"APP_ENV" default:"development"`
		Host        string  `env:"APP_HOST" default:"localhost"`
		Port        int     `env:"APP_PORT" default:"3306"`
		TestFloat   float32 `env:"APP_FLOAT" default:"1.5"`
		TestBool    bool    `env:"APP_BOOL" default:"true"`
		Counter     uint32  `env:"APP_COUNTER" default:"1"`
	}

	config := testEnvConfigWithDefaults{}
	config.Host = "testhost"
	config.Port = 1000
	os.Clearenv()

	err := config.Load(&config)
	assert.Nil(t, err)

	assert.Equal(t, "development", config.Environment)
	assert.Equal(t, "testhost", config.Host)
	assert.Equal(t, 1000, config.Port)
	assert.Equal(t, uint32(1), config.Counter)
	assert.Equal(t, float32(1.5), config.TestFloat)
	assert.True(t, config.TestBool)
}

func TestLoadFromDefaultsFailureBadFloatDefault(t *testing.T) {
	type badDefaults struct {
		Config
		TestFloat float32 `env:"APP_FLOAT" default:"badvalue"`
	}

	config := badDefaults{}
	os.Clearenv()

	err := config.Load(&config)
	assert.NotNil(t, err)
}

func TestLoadFromDefaultsFailureBadBoolDefault(t *testing.T) {
	type badDefaults struct {
		Config
		TestBool bool `env:"APP_BOOL" default:"badvalue"`
	}

	config := badDefaults{}
	os.Clearenv()

	err := config.Load(&config)
	assert.NotNil(t, err)
}

func TestLoadFromDefaultsFailureBadIntDefault(t *testing.T) {
	type badDefaults struct {
		Config
		TestPort int `env:"APP_PORT" default:"badvalue"`
	}

	config := badDefaults{}
	os.Clearenv()

	err := config.Load(&config)
	assert.NotNil(t, err)
}

func TestLoadFromEnvVariablesSuccess(t *testing.T) {
	type testEnvConfigWithDefaults struct {
		Config

		// Annotation your properties with where config data can come from
		Environment string  `env:"APP_ENV" default:"development"`
		Host        string  `env:"APP_HOST" default:"localhost"`
		Port        int     `env:"APP_PORT" default:"3306"`
		TestFloat   float32 `env:"APP_FLOAT" default:"1.5"`
		TestBool    bool    `env:"APP_BOOL" default:"true"`
	}

	config := testEnvConfigWithDefaults{}
	os.Clearenv()
	os.Setenv("APP_ENV", "dev")
	os.Setenv("APP_HOST", "127.0.0.1")
	os.Setenv("APP_PORT", "4000")
	os.Setenv("APP_FLOAT", "3.14159")
	os.Setenv("APP_BOOL", "false")

	err := config.Load(&config)
	assert.Nil(t, err)

	assert.Equal(t, "dev", config.Environment)
	assert.Equal(t, "127.0.0.1", config.Host)
	assert.Equal(t, 4000, config.Port)
	assert.Equal(t, float32(3.14159), config.TestFloat)
	assert.False(t, config.TestBool)
}

func TestLoadConfigFromFlagsSuccess(t *testing.T) {
	type testFlagConfig struct {
		Config

		// Annotation your properties with where config data can come from
		Environment string  `flag:"env" default:"development"`
		Host        string  `flag:"host" default:"localhost"`
		Port        int     `flag:"port" default:"3306"`
		Version     float32 `flag:"version" default:"1.5"`
		TestBool    bool    `flag:"testbool" default:"false"`
	}

	config := testFlagConfig{}
	config.viper = viper.New()
	os.Clearenv()

	ptrRef := reflect.ValueOf(&config)
	structRef := ptrRef.Elem()
	flagValues := parseFlagValues(structRef)
	flagSet := config.bindFlagValues(flagValues)

	fmt.Println("flag vals", flagValues)

	var expectedFlagValues = map[string]string{
		"env":      "dev",
		"host":     "127.0.0.1",
		"port":     "4000",
		"version":  "3.14159",
		"testbool": "true",
	}

	// Update the flag values with actual
	flagSet.VisitAll(func(flag *pflag.Flag) {
		flag.Value.Set(expectedFlagValues[flag.Name])
		flag.Changed = true
	})

	defaultValues := parseDefaultValues(structRef)
	config.populateDefaults(defaultValues)
	config.populateConfigStruct(structRef)

	assert.Equal(t, "dev", config.Environment)
	assert.Equal(t, "127.0.0.1", config.Host)
	assert.Equal(t, 4000, config.Port)
	assert.Equal(t, float32(3.14159), config.Version)
	assert.True(t, config.TestBool)
}

func TestLoadConfigFromFlagsFailureUseDefaults(t *testing.T) {
	type testFlagConfig struct {
		Config

		// Annotation your properties with where config data can come from
		Environment string  `flag:"env1" default:"development"`
		Host        string  `flag:"host1" default:"localhost"`
		Port        int     `flag:"port1" default:"3306"`
		Version     float32 `flag:"version1" default:"1.5"`
		TestBool    bool    `flag:"testbool1" default:"false"`
	}

	config := testFlagConfig{}
	config.Load(&config)

	// Config should be loaded with defaults
	assert.Equal(t, "development", config.Environment)
	assert.Equal(t, "localhost", config.Host)
	assert.Equal(t, 3306, config.Port)
	assert.Equal(t, float32(1.5), config.Version)
	assert.False(t, config.TestBool)
}

func TestLoadConfigFromFlagsFailureBadValues(t *testing.T) {
	type testFlagConfig struct {
		Config

		// Annotation your properties with where config data can come from
		Environment string  `flag:"env2" default:"development"`
		Host        string  `flag:"host2" default:"localhost"`
		Port        int     `flag:"port2" default:"3306"`
		Version     float32 `flag:"version2" default:"1.5"`
		TestBool    bool    `flag:"testbool2" default:"false"`
	}

	config := testFlagConfig{}
	config.viper = viper.New()
	os.Clearenv()

	ptrRef := reflect.ValueOf(&config)
	structRef := ptrRef.Elem()
	flagValues := parseFlagValues(structRef)
	flagSet := config.bindFlagValues(flagValues)

	var expectedFlagValues = map[string]string{
		"env2":      "dev",
		"host2":     "127.0.0.1",
		"port2":     "asb",
		"version2":  "3.14159",
		"testbool2": "true",
	}

	// Update the flag values with actual
	flagSet.VisitAll(func(flag *pflag.Flag) {
		flag.Value.Set(expectedFlagValues[flag.Name])
		flag.Changed = true
	})

	defaultValues := parseDefaultValues(structRef)
	config.populateDefaults(defaultValues)
	err := config.populateConfigStruct(structRef)

	assert.NotNil(t, err)
}

func TestLoadConfigFromFileSuccess(t *testing.T) {
	type testFileConfig struct {
		Config

		Environment string  `file:"env" default:"development"`
		Host        string  `file:"host" default:"localhost"`
		Port        int     `file:"port" default:"3306"`
		Version     float32 `file:"version" default:"1.5"`
		TestBool    bool    `file:"enabled" default:"false"`
	}

	configData := []byte(`{
    "env": "development",
    "host": "127.0.0.1",
    "port": 3000,
    "version": 2,
    "enabled": true
	}`)

	filePath := "./config.json"
	testConfig, err := os.Create(filePath)

	defer func() {
		testConfig.Close()
		os.Remove(filePath)
	}()

	assert.Nil(t, err)

	_, err = testConfig.Write(configData)
	assert.Nil(t, err, "File write error should be nil!")
	testConfig.Sync()

	config := testFileConfig{}
	config.FileName = "config"
	config.FilePaths = []string{
		".",
	}

	err = config.Load(&config)
	assert.Nil(t, err)

	assert.Equal(t, "development", config.Environment)
	assert.Equal(t, "127.0.0.1", config.Host)
	assert.Equal(t, 3000, config.Port)
	assert.Equal(t, float32(2), config.Version)
	assert.True(t, config.TestBool)
}

func TestLoadConfigFromFileFailureBadValue(t *testing.T) {
	type testFileConfig struct {
		Config

		Environment string  `file:"env" default:"development"`
		Host        uint32  `file:"host" default:"3000"` // changed host to uint32, while it's a string in JSON file
		Port        int     `file:"port" default:"3306"`
		Version     float32 `file:"version" default:"1.5"`
		TestBool    bool    `file:"enabled" default:"false"`
	}

	configData := []byte(`{
    "env": "development",
    "host": "127.0.0.1",
    "port": 3000,
    "version": 2,
    "enabled": true
	}`)

	filePath := "./config.json"
	testConfig, err := os.Create(filePath)

	defer func() {
		testConfig.Close()
		os.Remove(filePath)
	}()

	assert.Nil(t, err)

	_, err = testConfig.Write(configData)
	assert.Nil(t, err, "File write error should be nil!")
	testConfig.Sync()

	config := testFileConfig{}
	config.FileName = "config"
	config.FilePaths = []string{
		".",
	}

	err = config.Load(&config)
	assert.NotNil(t, err)
}

func TestCompoundSourcesScenario(t *testing.T) {
	type testFileConfig struct {
		Config

		Environment string  `file:"env" env:"APP_ENV" flag:"env5" default:"development"`
		Host        string  `file:"host" env:"APP_HOST" flag:"host5" default:"3000"` // changed host to uint32, while it's a string in JSON file
		Port        uint    `file:"port" env:"APP_PORT" flag:"port5" default:"3306"`
		Version     float32 `file:"version" env:"APP_VERSION" flag:"version5" default:"1.5"`
		Restricted  bool    `file:"enabled" env:"APP_RESTRICTED" flag:"restricted" default:"false"`
	}
}
<|endoftext|>


<commit_before>package shippers

import "fmt"
import "net"
import "net/url"
import "strings"
import "time"
import "github.com/josegonzalez/metricsd/structs"
import "github.com/Sirupsen/logrus"
import ini "github.com/vaughan0/go-ini"

type GraphiteShipper struct {
	debug   bool
	enabled bool
	host    string
	prefix  string
	port    string
}

func (this *GraphiteShipper) Enabled() bool {
	return this.enabled
}

func (this *GraphiteShipper) State(state bool) {
	this.enabled = state
}

func (this *GraphiteShipper) Setup(conf ini.File) {
	this.State(true)

	useDebug, ok := conf.Get("GraphiteShipper", "debug")
	if ok && useDebug == "true" {
		this.debug = true
	} else {
		this.debug = false
	}

	this.host = "127.0.0.1"
	this.port = "2003"
	useGraphiteUrl, ok := conf.Get("GraphiteShipper", "url")
	if ok {
		graphiteUrl, err := url.Parse(useGraphiteUrl)
		if err == nil {
			splitted := strings.Split(graphiteUrl.Host, ":")
			this.host, this.port = splitted[0], "2003"
			switch {
			case len(splitted) > 2:
				logrus.Warning("error parsing graphite url")
				logrus.Warning("using default 127.0.0.1:2003 for graphite url")
			case len(splitted) > 1:
				this.host, this.port = splitted[0], splitted[1]
			default:
				this.host, this.port = splitted[0], "2003"
			}
		} else {
			logrus.Warning("error parsing graphite url: %s", err)
			logrus.Warning("using default 127.0.0.1:2003 for graphite url")
		}
	}

	usePrefix, ok := conf.Get("GraphiteShipper", "prefix")
	if ok {
		this.prefix = fmt.Sprintf("%s.", usePrefix)
	}
}

func (this *GraphiteShipper) Ship(logs structs.MetricSlice) error {
	con, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", this.host, this.port), 1*time.Second)
	if err != nil {
		logrus.Warning("connecting to graphite failed with err: ", err)
		return err
	}
	defer con.Close()

	for _, item := range logs {
		serialized := item.ToGraphite(this.prefix)
		if this.debug {
			fmt.Printf("%s\n", serialized)
		}
		fmt.Fprintln(con, serialized)
	}

	return nil
}
<commit_msg>Follow variable naming conventions for URL<commit_after>package shippers

import "fmt"
import "net"
import "net/url"
import "strings"
import "time"
import "github.com/josegonzalez/metricsd/structs"
import "github.com/Sirupsen/logrus"
import ini "github.com/vaughan0/go-ini"

type GraphiteShipper struct {
	debug   bool
	enabled bool
	host    string
	prefix  string
	port    string
}

func (this *GraphiteShipper) Enabled() bool {
	return this.enabled
}

func (this *GraphiteShipper) State(state bool) {
	this.enabled = state
}

func (this *GraphiteShipper) Setup(conf ini.File) {
	this.State(true)

	useDebug, ok := conf.Get("GraphiteShipper", "debug")
	if ok && useDebug == "true" {
		this.debug = true
	} else {
		this.debug = false
	}

	this.host = "127.0.0.1"
	this.port = "2003"
	useGraphiteURL, ok := conf.Get("GraphiteShipper", "url")
	if ok {
		graphiteURL, err := url.Parse(useGraphiteURL)
		if err == nil {
			splitted := strings.Split(graphiteURL.Host, ":")
			this.host, this.port = splitted[0], "2003"
			switch {
			case len(splitted) > 2:
				logrus.Warning("error parsing graphite url")
				logrus.Warning("using default 127.0.0.1:2003 for graphite url")
			case len(splitted) > 1:
				this.host, this.port = splitted[0], splitted[1]
			default:
				this.host, this.port = splitted[0], "2003"
			}
		} else {
			logrus.Warning("error parsing graphite url: %s", err)
			logrus.Warning("using default 127.0.0.1:2003 for graphite url")
		}
	}

	usePrefix, ok := conf.Get("GraphiteShipper", "prefix")
	if ok {
		this.prefix = fmt.Sprintf("%s.", usePrefix)
	}
}

func (this *GraphiteShipper) Ship(logs structs.MetricSlice) error {
	con, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", this.host, this.port), 1*time.Second)
	if err != nil {
		logrus.Warning("connecting to graphite failed with err: ", err)
		return err
	}
	defer con.Close()

	for _, item := range logs {
		serialized := item.ToGraphite(this.prefix)
		if this.debug {
			fmt.Printf("%s\n", serialized)
		}
		fmt.Fprintln(con, serialized)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"code.google.com/p/go.crypto/ssh"

	"code.google.com/p/gopass"
	"github.com/codegangsta/cli"
)

func decryptPEM(pemblock *pem.Block, filename string) ([]byte, error) {
	var err error
	if _, err = fmt.Fprintf(os.Stderr, "Enter passphrase for %s: ", filename); err != nil {
		return []byte(""), err
	}

	// we already emit the prompt to stderr; GetPass only emits to stdout
	var passwd string
	if passwd, err = gopass.GetPass(""); err != nil {
		return []byte(""), err
	}

	var decryptedBytes []byte
	if decryptedBytes, err = x509.DecryptPEMBlock(pemblock, []byte(passwd)); err != nil {
		return []byte(""), err
	}

	pemBytes := pem.Block{
		Type:  "RSA PRIVATE KEY",
		Bytes: decryptedBytes,
	}
	decryptedPEM := pem.EncodeToMemory(&pemBytes)
	return decryptedPEM, nil
}

func getPrivateKey(c *cli.Context) (filename string) {
	if c.String("key") == "" {
		filename = filepath.Join(os.Getenv("HOME"), "/.ssh/id_rsa")
	} else {
		filename = c.String("key")
	}
	return filename
}

func getAccountAndUserName(c *cli.Context) (string, string) {
	if c.String("credentials") != "" {
		result := strings.Split(c.String("credentials"), "@")
		return result[1], result[0]
	} else {
		return c.String("account"), c.String("username")
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "credulous"
	app.Usage = "Use it!"
	app.Version = "0.1.3"

	app.Commands = []cli.Command{
		{
			Name:  "save",
			Usage: "Save AWS credentials for a file.",
			Flags: []cli.Flag{
				cli.StringFlag{"key, k", "", "SSH public key"},
			},
			Action: func(c *cli.Context) {
				var pubkeyFile string
				if c.String("key") == "" {
					pubkeyFile = filepath.Join(os.Getenv("HOME"), "/.ssh/id_rsa.pub")
				} else {
					pubkeyFile = c.String("key")
				}

				AWSAccessKeyId := os.Getenv("AWS_ACCESS_KEY_ID")
				AWSSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
				if AWSAccessKeyId == "" || AWSSecretAccessKey == "" {
					fmt.Println("Can't save, no credentials in the environment")
					os.Exit(1)
				}
				pubkeyString, err := ioutil.ReadFile(pubkeyFile)
				panic_the_err(err)
				pubkey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(pubkeyString))
				SaveCredentials(AWSAccessKeyId, AWSSecretAccessKey, pubkey)
			},
		},
		{
			Name:  "source",
			Usage: "Source AWS credentials from a file.",
			Flags: []cli.Flag{
				cli.StringFlag{"account, a", "", "AWS Account alias or id"},
				cli.StringFlag{"key, k", "", "SSH private key"},
				cli.StringFlag{"username, u", "", "IAM User"},
				cli.StringFlag{"credentials, c", "", "Credentials, for example username@account"},
			},
			Action: func(c *cli.Context) {
				keyfile := getPrivateKey(c)
				account, username := getAccountAndUserName(c)
				cred, err := RetrieveCredentials(account, username, keyfile)
				if err != nil {
					panic_the_err(err)
				}
				err = ValidateCredentials(account, username, cred)
				if err != nil {
					panic_the_err(err)
				}
				cred.Display(os.Stdout)
			},
		},
		{
			Name:  "display",
			Usage: "Display loaded AWS credentials",
			Action: func(c *cli.Context) {
				AWSAccessKeyId := os.Getenv("AWS_ACCESS_KEY_ID")
				AWSSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
				fmt.Printf("AWS_ACCESS_KEY_ID: %s\n", AWSAccessKeyId)
				fmt.Printf("AWS_SECRET_ACCESS_KEY: %s\n", AWSSecretAccessKey)
			},
		},
		{
			Name:  "list",
			Usage: "List the sets of available credentials",
			Action: func(c *cli.Context) {
				rootDir, err := os.Open(getRootPath())
				if err != nil {
					panic_the_err(err)
				}
				set, err := listAvailableCredentials(rootDir)
				if err != nil {
					panic_the_err(err)
				}
				for _, cred := range set {
					fmt.Println(cred)
				}
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>credulous source username@account_alias works<commit_after>package main

import (
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"code.google.com/p/go.crypto/ssh"

	"code.google.com/p/gopass"
	"github.com/codegangsta/cli"
)

func decryptPEM(pemblock *pem.Block, filename string) ([]byte, error) {
	var err error
	if _, err = fmt.Fprintf(os.Stderr, "Enter passphrase for %s: ", filename); err != nil {
		return []byte(""), err
	}

	// we already emit the prompt to stderr; GetPass only emits to stdout
	var passwd string
	if passwd, err = gopass.GetPass(""); err != nil {
		return []byte(""), err
	}

	var decryptedBytes []byte
	if decryptedBytes, err = x509.DecryptPEMBlock(pemblock, []byte(passwd)); err != nil {
		return []byte(""), err
	}

	pemBytes := pem.Block{
		Type:  "RSA PRIVATE KEY",
		Bytes: decryptedBytes,
	}
	decryptedPEM := pem.EncodeToMemory(&pemBytes)
	return decryptedPEM, nil
}

func getPrivateKey(c *cli.Context) (filename string) {
	if c.String("key") == "" {
		filename = filepath.Join(os.Getenv("HOME"), "/.ssh/id_rsa")
	} else {
		filename = c.String("key")
	}
	return filename
}

func getAccountAndUserName(c *cli.Context) (string, string) {
  // XXX Check input for errors
  if len(c.Args()) > 0 {
		result := strings.Split(c.Args()[0], "@")
		return result[1], result[0]
  } else if c.String("credentials") != "" {
		result := strings.Split(c.String("credentials"), "@")
		return result[1], result[0]
	} else {
		return c.String("account"), c.String("username")
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "credulous"
	app.Usage = "Use it!"
	app.Version = "0.1.3"

	app.Commands = []cli.Command{
		{
			Name:  "save",
			Usage: "Save AWS credentials for a file.",
			Flags: []cli.Flag{
				cli.StringFlag{"key, k", "", "SSH public key"},
			},
			Action: func(c *cli.Context) {
				var pubkeyFile string
				if c.String("key") == "" {
					pubkeyFile = filepath.Join(os.Getenv("HOME"), "/.ssh/id_rsa.pub")
				} else {
					pubkeyFile = c.String("key")
				}

				AWSAccessKeyId := os.Getenv("AWS_ACCESS_KEY_ID")
				AWSSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
				if AWSAccessKeyId == "" || AWSSecretAccessKey == "" {
					fmt.Println("Can't save, no credentials in the environment")
					os.Exit(1)
				}
				pubkeyString, err := ioutil.ReadFile(pubkeyFile)
				panic_the_err(err)
				pubkey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(pubkeyString))
				SaveCredentials(AWSAccessKeyId, AWSSecretAccessKey, pubkey)
			},
		},
		{
			Name:  "source",
			Usage: "Source AWS credentials from a file.",
			Flags: []cli.Flag{
				cli.StringFlag{"account, a", "", "AWS Account alias or id"},
				cli.StringFlag{"key, k", "", "SSH private key"},
				cli.StringFlag{"username, u", "", "IAM User"},
				cli.StringFlag{"credentials, c", "", "Credentials, for example username@account"},
			},
			Action: func(c *cli.Context) {
				keyfile := getPrivateKey(c)
				account, username := getAccountAndUserName(c)
				cred, err := RetrieveCredentials(account, username, keyfile)
				if err != nil {
					panic_the_err(err)
				}
				err = ValidateCredentials(account, username, cred)
				if err != nil {
					panic_the_err(err)
				}
				cred.Display(os.Stdout)
			},
		},
		{
			Name:  "display",
			Usage: "Display loaded AWS credentials",
			Action: func(c *cli.Context) {
				AWSAccessKeyId := os.Getenv("AWS_ACCESS_KEY_ID")
				AWSSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
				fmt.Printf("AWS_ACCESS_KEY_ID: %s\n", AWSAccessKeyId)
				fmt.Printf("AWS_SECRET_ACCESS_KEY: %s\n", AWSSecretAccessKey)
			},
		},
		{
			Name:  "list",
			Usage: "List the sets of available credentials",
			Action: func(c *cli.Context) {
				rootDir, err := os.Open(getRootPath())
				if err != nil {
					panic_the_err(err)
				}
				set, err := listAvailableCredentials(rootDir)
				if err != nil {
					panic_the_err(err)
				}
				for _, cred := range set {
					fmt.Println(cred)
				}
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package gapi

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
)

type AlertNotification struct {
	Id                    int64       `json:"id,omitempty"`
	Name                  string      `json:"name"`
	Type                  string      `json:"type"`
	IsDefault             bool        `json:"isDefault"`
	DisableResolveMessage bool        `json:"disableResolveMessage"`
	SendReminder          bool        `json:"sendReminder"`
	Frequency             string      `json:"frequency"`
	Settings              interface{} `json:"settings"`
}

func (c *Client) AlertNotifications() ([]AlertNotification, error) {
	alertnotifications := make([]AlertNotification, 0)

	req, err := c.newRequest("GET", "/api/alert-notifications/", nil, nil)
	if err != nil {
		return nil, err
	}

	resp, err := c.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		return nil, errors.New(resp.Status)
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(data, &alertnotifications)
	return alertnotifications, err
}

func (c *Client) AlertNotification(id int64) (*AlertNotification, error) {
	path := fmt.Sprintf("/api/alert-notifications/%d", id)
	req, err := c.newRequest("GET", path, nil, nil)
	if err != nil {
		return nil, err
	}

	resp, err := c.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		return nil, errors.New(resp.Status)
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	result := &AlertNotification{}
	err = json.Unmarshal(data, &result)
	return result, err
}

func (c *Client) NewAlertNotification(a *AlertNotification) (int64, error) {
	data, err := json.Marshal(a)
	if err != nil {
		return 0, err
	}
	req, err := c.newRequest("POST", "/api/alert-notifications", nil, bytes.NewBuffer(data))
	if err != nil {
		return 0, err
	}

	resp, err := c.Do(req)
	if err != nil {
		return 0, err
	}
	if resp.StatusCode != 200 {
		return 0, errors.New(resp.Status)
	}

	data, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return 0, err
	}

	result := struct {
		Id int64 `json:"id"`
	}{}
	err = json.Unmarshal(data, &result)
	return result.Id, err
}

func (c *Client) UpdateAlertNotification(a *AlertNotification) error {
	path := fmt.Sprintf("/api/alert-notifications/%d", a.Id)
	data, err := json.Marshal(a)
	if err != nil {
		return err
	}
	req, err := c.newRequest("PUT", path, nil, bytes.NewBuffer(data))
	if err != nil {
		return err
	}

	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return errors.New(resp.Status)
	}

	return nil
}

func (c *Client) DeleteAlertNotification(id int64) error {
	path := fmt.Sprintf("/api/alert-notifications/%d", id)
	req, err := c.newRequest("DELETE", path, nil, nil)
	if err != nil {
		return err
	}

	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return errors.New(resp.Status)
	}

	return nil
}
<commit_msg>add AlertNotification Uid field (#34)<commit_after>package gapi

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
)

type AlertNotification struct {
	Id                    int64       `json:"id,omitempty"`
	Uid                   string      `json:"uid"`
	Name                  string      `json:"name"`
	Type                  string      `json:"type"`
	IsDefault             bool        `json:"isDefault"`
	DisableResolveMessage bool        `json:"disableResolveMessage"`
	SendReminder          bool        `json:"sendReminder"`
	Frequency             string      `json:"frequency"`
	Settings              interface{} `json:"settings"`
}

func (c *Client) AlertNotifications() ([]AlertNotification, error) {
	alertnotifications := make([]AlertNotification, 0)

	req, err := c.newRequest("GET", "/api/alert-notifications/", nil, nil)
	if err != nil {
		return nil, err
	}

	resp, err := c.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		return nil, errors.New(resp.Status)
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(data, &alertnotifications)
	return alertnotifications, err
}

func (c *Client) AlertNotification(id int64) (*AlertNotification, error) {
	path := fmt.Sprintf("/api/alert-notifications/%d", id)
	req, err := c.newRequest("GET", path, nil, nil)
	if err != nil {
		return nil, err
	}

	resp, err := c.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		return nil, errors.New(resp.Status)
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	result := &AlertNotification{}
	err = json.Unmarshal(data, &result)
	return result, err
}

func (c *Client) NewAlertNotification(a *AlertNotification) (int64, error) {
	data, err := json.Marshal(a)
	if err != nil {
		return 0, err
	}
	req, err := c.newRequest("POST", "/api/alert-notifications", nil, bytes.NewBuffer(data))
	if err != nil {
		return 0, err
	}

	resp, err := c.Do(req)
	if err != nil {
		return 0, err
	}
	if resp.StatusCode != 200 {
		return 0, errors.New(resp.Status)
	}

	data, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return 0, err
	}

	result := struct {
		Id int64 `json:"id"`
	}{}
	err = json.Unmarshal(data, &result)
	return result.Id, err
}

func (c *Client) UpdateAlertNotification(a *AlertNotification) error {
	path := fmt.Sprintf("/api/alert-notifications/%d", a.Id)
	data, err := json.Marshal(a)
	if err != nil {
		return err
	}
	req, err := c.newRequest("PUT", path, nil, bytes.NewBuffer(data))
	if err != nil {
		return err
	}

	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return errors.New(resp.Status)
	}

	return nil
}

func (c *Client) DeleteAlertNotification(id int64) error {
	path := fmt.Sprintf("/api/alert-notifications/%d", id)
	req, err := c.newRequest("DELETE", path, nil, nil)
	if err != nil {
		return err
	}

	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return errors.New(resp.Status)
	}

	return nil
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2014 Christian Muehlhaeuser
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *	Authors:
 *		Christian Muehlhaeuser <muesli@gmail.com>
 *		Michael Wendland <michael@michiwend.com>
 */

package smartcrop

import (
	"errors"
	"fmt"
	"image"
	_ "image/jpeg"
	_ "image/png"
	"io/ioutil"
	"os"
	"strings"
	"testing"
)

var (
	testFile = "./samples/gopher.jpg"
)

type SubImager interface {
	SubImage(r image.Rectangle) image.Image
}

func TestCrop(t *testing.T) {
	fi, _ := os.Open(testFile)
	defer fi.Close()

	img, _, err := image.Decode(fi)
	if err != nil {
		t.Error(err)
	}

	topCrop, err := SmartCrop(img, 250, 250)
	if err != nil {
		t.Error(err)
	}
	want := image.Rect(196, 42, 580, 427)
	if topCrop != want {
		t.Fatalf("want %v, got %v", want, topCrop)
	}

	sub, ok := img.(SubImager)
	if ok {
		cropImage := sub.SubImage(topCrop)
		//		cropImage := sub.SubImage(image.Rect(topCrop.X, topCrop.Y, topCrop.Width+topCrop.X, topCrop.Height+topCrop.Y))
		writeImage("jpeg", cropImage, "./smartcrop.jpg")

	} else {
		t.Error(errors.New("No SubImage support"))
	}

}

func BenchmarkEdge(b *testing.B) {
	fname := "24391757.jpg"
	fi, _ := os.Open("./samples/" + fname)
	defer fi.Close()
	img, _, err := image.Decode(fi)
	if err != nil {
		b.Error(err)
	}

	rgbaImg := toRGBA(img)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		o := image.NewRGBA(img.Bounds())
		edgeDetect(rgbaImg, o)
	}
}

func BenchmarkImageDir(b *testing.B) {
	b.StopTimer()

	files, err := ioutil.ReadDir("./samples/hatchet")
	if err != nil {
		b.Error(err)
	}

	b.StartTimer()
	for _, file := range files {
		if strings.Contains(file.Name(), ".jpg") {

			fi, _ := os.Open("./samples/hatchet/" + file.Name())
			defer fi.Close()

			img, _, err := image.Decode(fi)
			if err != nil {
				b.Error(err)
			}

			topCrop, err := SmartCrop(img, 900, 500)
			if err != nil {
				b.Error(err)
			}
			fmt.Printf("Top crop: %+v\n", topCrop)

			sub, ok := img.(SubImager)
			if ok {
				cropImage := sub.SubImage(topCrop)
				//				cropImage := sub.SubImage(image.Rect(topCrop.X, topCrop.Y, topCrop.Width+topCrop.X, topCrop.Height+topCrop.Y))
				writeImage("jpeg", cropImage, "/tmp/smartcrop/smartcrop-"+file.Name())
			} else {
				b.Error(errors.New("No SubImage support"))
			}
		}
	}
	//fmt.Println("average time/image:", b.t
}
<commit_msg>Added BenchmarkCrop()<commit_after>/*
 * Copyright (c) 2014 Christian Muehlhaeuser
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *	Authors:
 *		Christian Muehlhaeuser <muesli@gmail.com>
 *		Michael Wendland <michael@michiwend.com>
 */

package smartcrop

import (
	"errors"
	"fmt"
	"image"
	_ "image/jpeg"
	_ "image/png"
	"io/ioutil"
	"os"
	"strings"
	"testing"
)

var (
	testFile = "./samples/gopher.jpg"
)

type SubImager interface {
	SubImage(r image.Rectangle) image.Image
}

func TestCrop(t *testing.T) {
	fi, _ := os.Open(testFile)
	defer fi.Close()

	img, _, err := image.Decode(fi)
	if err != nil {
		t.Error(err)
	}

	topCrop, err := SmartCrop(img, 250, 250)
	if err != nil {
		t.Error(err)
	}
	want := image.Rect(196, 42, 580, 427)
	if topCrop != want {
		t.Fatalf("want %v, got %v", want, topCrop)
	}

	sub, ok := img.(SubImager)
	if ok {
		cropImage := sub.SubImage(topCrop)
		//		cropImage := sub.SubImage(image.Rect(topCrop.X, topCrop.Y, topCrop.Width+topCrop.X, topCrop.Height+topCrop.Y))
		writeImage("jpeg", cropImage, "./smartcrop.jpg")

	} else {
		t.Error(errors.New("No SubImage support"))
	}
}

func BenchmarkCrop(b *testing.B) {
	fi, err := os.Open(testFile)
	if err != nil {
		b.Fatal(err)
	}
	defer fi.Close()
	img, _, err := image.Decode(fi)
	if err != nil {
		b.Fatal(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if _, err := SmartCrop(img, 250, 250); err != nil {
			b.Error(err)
		}
	}
}

func BenchmarkEdge(b *testing.B) {
	fname := "24391757.jpg"
	fi, _ := os.Open("./samples/" + fname)
	defer fi.Close()
	img, _, err := image.Decode(fi)
	if err != nil {
		b.Error(err)
	}

	rgbaImg := toRGBA(img)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		o := image.NewRGBA(img.Bounds())
		edgeDetect(rgbaImg, o)
	}
}

func BenchmarkImageDir(b *testing.B) {
	b.StopTimer()

	files, err := ioutil.ReadDir("./samples/hatchet")
	if err != nil {
		b.Error(err)
	}

	b.StartTimer()
	for _, file := range files {
		if strings.Contains(file.Name(), ".jpg") {

			fi, _ := os.Open("./samples/hatchet/" + file.Name())
			defer fi.Close()

			img, _, err := image.Decode(fi)
			if err != nil {
				b.Error(err)
			}

			topCrop, err := SmartCrop(img, 900, 500)
			if err != nil {
				b.Error(err)
			}
			fmt.Printf("Top crop: %+v\n", topCrop)

			sub, ok := img.(SubImager)
			if ok {
				cropImage := sub.SubImage(topCrop)
				//				cropImage := sub.SubImage(image.Rect(topCrop.X, topCrop.Y, topCrop.Width+topCrop.X, topCrop.Height+topCrop.Y))
				writeImage("jpeg", cropImage, "/tmp/smartcrop/smartcrop-"+file.Name())
			} else {
				b.Error(errors.New("No SubImage support"))
			}
		}
	}
	//fmt.Println("average time/image:", b.t
}
<|endoftext|>