<commit_before>package lfs

import (
	"sync"
	"sync/atomic"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/config"
	"github.com/github/git-lfs/errutil"
	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/progress"
	"github.com/github/git-lfs/transfer"
	"github.com/rubyist/tracerx"
)

const (
	batchSize = 100
)

type Transferable interface {
	Oid() string
	Size() int64
	Name() string
	Path() string
	Object() *api.ObjectResource
	SetObject(*api.ObjectResource)
	// Legacy API check - TODO remove this and only support batch
	LegacyCheck() (*api.ObjectResource, error)
}

// TransferQueue organises the wider process of uploading and downloading,
// including calling the API, passing the actual transfer request to transfer
// adapters, and dealing with progress, errors and retries
type TransferQueue struct {
	direction         transfer.Direction
	adapter           transfer.TransferAdapter
	adapterInProgress bool
	adapterResultChan chan transfer.TransferResult
	adapterInitMutex  sync.Mutex
	dryRun            bool
	retrying          uint32
	meter             *progress.ProgressMeter
	errors            []error
	transferables     map[string]Transferable
	retries           []Transferable
	batcher           *Batcher
	apic              chan Transferable // Channel for processing individual API requests
	retriesc          chan Transferable // Channel for processing retries
	errorc            chan error        // Channel for processing errors
	watchers          []chan string
	trMutex           *sync.Mutex
	errorwait         sync.WaitGroup
	retrywait         sync.WaitGroup
	wait              sync.WaitGroup // Incremented on Add(), decremented on transfer complete or skip
	oldApiWorkers     int            // Number of non-batch API workers to spawn (deprecated)
}

// newTransferQueue builds a TransferQueue, direction and underlying mechanism determined by adapter
func newTransferQueue(files int, size int64, dryRun bool, dir transfer.Direction) *TransferQueue {
	q := &TransferQueue{
		direction:     dir,
		dryRun:        dryRun,
		meter:         progress.NewProgressMeter(files, size, dryRun, config.Config.Getenv("GIT_LFS_PROGRESS")),
		apic:          make(chan Transferable, batchSize),
		retriesc:      make(chan Transferable, batchSize),
		errorc:        make(chan error),
		oldApiWorkers: config.Config.ConcurrentTransfers(),
		transferables: make(map[string]Transferable),
		trMutex:       &sync.Mutex{},
	}

	q.errorwait.Add(1)
	q.retrywait.Add(1)

	q.run()

	return q
}

// Add adds a Transferable to the transfer queue.
func (q *TransferQueue) Add(t Transferable) {
	q.wait.Add(1)
	q.trMutex.Lock()
	q.transferables[t.Oid()] = t
	q.trMutex.Unlock()

	if q.batcher != nil {
		q.batcher.Add(t)
		return
	}

	q.apic <- t
}

func (q *TransferQueue) useAdapter(name string) {
	q.adapterInitMutex.Lock()
	defer q.adapterInitMutex.Unlock()

	if q.adapter != nil {
		if q.adapter.Name() == name {
			// re-use, this is the normal path
			return
		}
		// If the adapter we're using isn't the same as the one we've been
		// told to use now, must wait for the current one to finish then switch
		// This will probably never happen but is just in case server starts
		// changing adapter support in between batches
		q.finishAdapter()
	}
	q.adapter = transfer.NewAdapterOrDefault(name, q.direction)
}

func (q *TransferQueue) finishAdapter() {
	if q.adapterInProgress {
		q.adapter.End()
		q.adapterInProgress = false
		q.adapter = nil
	}
}

func (q *TransferQueue) addToAdapter(t Transferable) {

	tr := transfer.NewTransfer(t.Name(), t.Object(), t.Path())

	if q.dryRun {
		// Don't actually transfer
		res := transfer.TransferResult{tr, nil}
		q.handleTransferResult(res)
		return
	}
	q.ensureAdapterBegun()
	q.adapter.Add(tr)
}

func (q *TransferQueue) Skip(size int64) {
	q.meter.Skip(size)
}

func (q *TransferQueue) transferKind() string {
	if q.direction == transfer.Download {
		return "download"
	} else {
		return "upload"
	}
}

func (q *TransferQueue) ensureAdapterBegun() {
	q.adapterInitMutex.Lock()
	defer q.adapterInitMutex.Unlock()

	if q.adapterInProgress {
		return
	}

	adapterResultChan := make(chan transfer.TransferResult, 20)

	// Progress callback - receives byte updates
	cb := func(name string, total, read int64, current int) error {
		q.meter.TransferBytes(q.transferKind(), name, read, total, current)
		return nil
	}

	tracerx.Printf("tq: starting transfer adapter %q", q.adapter.Name())
	q.adapter.Begin(config.Config.ConcurrentTransfers(), cb, adapterResultChan)
	q.adapterInProgress = true

	// Collector for completed transfers
	// q.wait.Done() in handleTransferResult is enough to know when this is complete for all transfers
	go func() {
		for res := range adapterResultChan {
			q.handleTransferResult(res)
		}
	}()

}

func (q *TransferQueue) handleTransferResult(res transfer.TransferResult) {
	if res.Error != nil {
		if q.canRetry(res.Error) {
			tracerx.Printf("tq: retrying object %s", res.Transfer.Object.Oid)
			q.trMutex.Lock()
			t, ok := q.transferables[res.Transfer.Object.Oid]
			q.trMutex.Unlock()
			if ok {
				q.retry(t)
			} else {
				q.errorc <- res.Error
			}
		} else {
			q.errorc <- res.Error
		}
	} else {
		oid := res.Transfer.Object.Oid
		for _, c := range q.watchers {
			c <- oid
		}
	}
	q.meter.FinishTransfer(res.Transfer.Name)
	q.wait.Done()

}

// Wait waits for the queue to finish processing all transfers. Once Wait is
// called, Add will no longer add transferables to the queue. Any failed
// transfers will be automatically retried once.
func (q *TransferQueue) Wait() {
	if q.batcher != nil {
		q.batcher.Exit()
	}

	q.wait.Wait()

	// Handle any retries
	close(q.retriesc)
	q.retrywait.Wait()
	atomic.StoreUint32(&q.retrying, 1)

	if len(q.retries) > 0 {
		tracerx.Printf("tq: retrying %d failed transfers", len(q.retries))
		for _, t := range q.retries {
			q.Add(t)
		}
		if q.batcher != nil {
			q.batcher.Exit()
		}
		q.wait.Wait()
	}

	atomic.StoreUint32(&q.retrying, 0)

	close(q.apic)
	q.finishAdapter()
	close(q.errorc)

	for _, watcher := range q.watchers {
		close(watcher)
	}

	q.meter.Finish()
	q.errorwait.Wait()
}

// Watch returns a channel where the queue will write the OID of each transfer
// as it completes. The channel will be closed when the queue finishes processing.
func (q *TransferQueue) Watch() chan string {
	c := make(chan string, batchSize)
	q.watchers = append(q.watchers, c)
	return c
}

// individualApiRoutine processes the queue of transfers one at a time by making
// a POST call for each object, feeding the results to the transfer workers.
// If configured, the object transfers can still happen concurrently, the
// sequential nature here is only for the meta POST calls.
// TODO LEGACY API: remove when legacy API removed
func (q *TransferQueue) individualApiRoutine(apiWaiter chan interface{}) {
	for t := range q.apic {
		obj, err := t.LegacyCheck()
		if err != nil {
			if q.canRetry(err) {
				q.retry(t)
			} else {
				q.errorc <- err
			}
			q.wait.Done()
			continue
		}

		if apiWaiter != nil { // Signal to launch more individual api workers
			q.meter.Start()
			select {
			case apiWaiter <- 1:
			default:
			}
		}

		// Legacy API has no support for anything but basic transfer adapter
		q.useAdapter(transfer.BasicAdapterName)
		if obj != nil {
			t.SetObject(obj)
			q.meter.Add(t.Name())
			q.addToAdapter(t)
		} else {
			q.Skip(t.Size())
			q.wait.Done()
		}
	}
}

// legacyFallback is used when a batch request is made to a server that does
// not support the batch endpoint. When this happens, the Transferables are
// fed from the batcher into apic to be processed individually.
// TODO LEGACY API: remove when legacy API removed
func (q *TransferQueue) legacyFallback(failedBatch []interface{}) {
	tracerx.Printf("tq: batch api not implemented, falling back to individual")

	q.launchIndividualApiRoutines()

	for _, t := range failedBatch {
		q.apic <- t.(Transferable)
	}

	for {
		batch := q.batcher.Next()
		if batch == nil {
			break
		}

		for _, t := range batch {
			q.apic <- t.(Transferable)
		}
	}
}

// batchApiRoutine processes the queue of transfers using the batch endpoint,
// making only one POST call for all objects. The results are then handed
// off to the transfer workers.
func (q *TransferQueue) batchApiRoutine() {
	var startProgress sync.Once

	transferAdapterNames := transfer.GetAdapterNames(q.direction)

	for {
		batch := q.batcher.Next()
		if batch == nil {
			break
		}

		tracerx.Printf("tq: sending batch of size %d", len(batch))

		transfers := make([]*api.ObjectResource, 0, len(batch))
		for _, i := range batch {
			t := i.(Transferable)
			transfers = append(transfers, &api.ObjectResource{Oid: t.Oid(), Size: t.Size()})
		}

		if len(transfers) == 0 {
			continue
		}

		objs, adapterName, err := api.Batch(transfers, q.transferKind(), transferAdapterNames)
		if err != nil {
			if errutil.IsNotImplementedError(err) {
				git.Config.SetLocal("", "lfs.batch", "false")

				go q.legacyFallback(batch)
				return
			}

			if q.canRetry(err) {
				for _, t := range batch {
					q.retry(t.(Transferable))
				}
			} else {
				q.errorc <- err
			}

			q.wait.Add(-len(transfers))
			continue
		}

		q.useAdapter(adapterName)
		startProgress.Do(q.meter.Start)

		for _, o := range objs {
			if o.Error != nil {
				q.errorc <- errutil.Errorf(o.Error, "[%v] %v", o.Oid, o.Error.Message)
				q.Skip(o.Size)
				q.wait.Done()
				continue
			}

			if _, ok := o.Rel(q.transferKind()); ok {
				// This object needs to be transferred
				q.trMutex.Lock()
				transfer, ok := q.transferables[o.Oid]
				q.trMutex.Unlock()

				if ok {
					transfer.SetObject(o)
					q.meter.Add(transfer.Name())
					q.addToAdapter(transfer)
				} else {
					q.Skip(transfer.Size())
					q.wait.Done()
				}
			} else {
				q.Skip(o.Size)
				q.wait.Done()
			}
		}
	}
}

// This goroutine collects errors returned from transfers
func (q *TransferQueue) errorCollector() {
	for err := range q.errorc {
		q.errors = append(q.errors, err)
	}
	q.errorwait.Done()
}

func (q *TransferQueue) retryCollector() {
	for t := range q.retriesc {
		q.retries = append(q.retries, t)
	}
	q.retrywait.Done()
}

// launchIndividualApiRoutines first launches a single api worker. When it
// receives the first successful api request it launches workers - 1 more
// workers. This prevents being prompted for credentials multiple times at once
// when they're needed.
func (q *TransferQueue) launchIndividualApiRoutines() {
	go func() {
		apiWaiter := make(chan interface{})
		go q.individualApiRoutine(apiWaiter)

		<-apiWaiter

		for i := 0; i < q.oldApiWorkers-1; i++ {
			go q.individualApiRoutine(nil)
		}
	}()
}

// run starts the transfer queue, doing individual or batch transfers depending
// on the Config.BatchTransfer() value. run will transfer files sequentially or
// concurrently depending on the Config.ConcurrentTransfers() value.
func (q *TransferQueue) run() {
	go q.errorCollector()
	go q.retryCollector()

	if config.Config.BatchTransfer() {
		tracerx.Printf("tq: running as batched queue, batch size of %d", batchSize)
		q.batcher = NewBatcher(batchSize)
		go q.batchApiRoutine()
	} else {
		tracerx.Printf("tq: running as individual queue")
		q.launchIndividualApiRoutines()
	}
}

func (q *TransferQueue) retry(t Transferable) {
	q.retriesc <- t
}

func (q *TransferQueue) canRetry(err error) bool {
	if !errutil.IsRetriableError(err) || atomic.LoadUint32(&q.retrying) == 1 {
		return false
	}

	return true
}

// Errors returns any errors encountered during transfer.
func (q *TransferQueue) Errors() []error {
	return q.errors
}
<commit_msg>transfer/queue: only mark successful transfers as having finished<commit_after>package lfs

import (
	"sync"
	"sync/atomic"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/config"
	"github.com/github/git-lfs/errutil"
	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/progress"
	"github.com/github/git-lfs/transfer"
	"github.com/rubyist/tracerx"
)

const (
	batchSize = 100
)

type Transferable interface {
	Oid() string
	Size() int64
	Name() string
	Path() string
	Object() *api.ObjectResource
	SetObject(*api.ObjectResource)
	// Legacy API check - TODO remove this and only support batch
	LegacyCheck() (*api.ObjectResource, error)
}

// TransferQueue organises the wider process of uploading and downloading,
// including calling the API, passing the actual transfer request to transfer
// adapters, and dealing with progress, errors and retries
type TransferQueue struct {
	direction         transfer.Direction
	adapter           transfer.TransferAdapter
	adapterInProgress bool
	adapterResultChan chan transfer.TransferResult
	adapterInitMutex  sync.Mutex
	dryRun            bool
	retrying          uint32
	meter             *progress.ProgressMeter
	errors            []error
	transferables     map[string]Transferable
	retries           []Transferable
	batcher           *Batcher
	apic              chan Transferable // Channel for processing individual API requests
	retriesc          chan Transferable // Channel for processing retries
	errorc            chan error        // Channel for processing errors
	watchers          []chan string
	trMutex           *sync.Mutex
	errorwait         sync.WaitGroup
	retrywait         sync.WaitGroup
	wait              sync.WaitGroup // Incremented on Add(), decremented on transfer complete or skip
	oldApiWorkers     int            // Number of non-batch API workers to spawn (deprecated)
}

// newTransferQueue builds a TransferQueue, direction and underlying mechanism determined by adapter
func newTransferQueue(files int, size int64, dryRun bool, dir transfer.Direction) *TransferQueue {
	q := &TransferQueue{
		direction:     dir,
		dryRun:        dryRun,
		meter:         progress.NewProgressMeter(files, size, dryRun, config.Config.Getenv("GIT_LFS_PROGRESS")),
		apic:          make(chan Transferable, batchSize),
		retriesc:      make(chan Transferable, batchSize),
		errorc:        make(chan error),
		oldApiWorkers: config.Config.ConcurrentTransfers(),
		transferables: make(map[string]Transferable),
		trMutex:       &sync.Mutex{},
	}

	q.errorwait.Add(1)
	q.retrywait.Add(1)

	q.run()

	return q
}

// Add adds a Transferable to the transfer queue.
func (q *TransferQueue) Add(t Transferable) {
	q.wait.Add(1)
	q.trMutex.Lock()
	q.transferables[t.Oid()] = t
	q.trMutex.Unlock()

	if q.batcher != nil {
		q.batcher.Add(t)
		return
	}

	q.apic <- t
}

func (q *TransferQueue) useAdapter(name string) {
	q.adapterInitMutex.Lock()
	defer q.adapterInitMutex.Unlock()

	if q.adapter != nil {
		if q.adapter.Name() == name {
			// re-use, this is the normal path
			return
		}
		// If the adapter we're using isn't the same as the one we've been
		// told to use now, must wait for the current one to finish then switch
		// This will probably never happen but is just in case server starts
		// changing adapter support in between batches
		q.finishAdapter()
	}
	q.adapter = transfer.NewAdapterOrDefault(name, q.direction)
}

func (q *TransferQueue) finishAdapter() {
	if q.adapterInProgress {
		q.adapter.End()
		q.adapterInProgress = false
		q.adapter = nil
	}
}

func (q *TransferQueue) addToAdapter(t Transferable) {

	tr := transfer.NewTransfer(t.Name(), t.Object(), t.Path())

	if q.dryRun {
		// Don't actually transfer
		res := transfer.TransferResult{tr, nil}
		q.handleTransferResult(res)
		return
	}
	q.ensureAdapterBegun()
	q.adapter.Add(tr)
}

func (q *TransferQueue) Skip(size int64) {
	q.meter.Skip(size)
}

func (q *TransferQueue) transferKind() string {
	if q.direction == transfer.Download {
		return "download"
	} else {
		return "upload"
	}
}

func (q *TransferQueue) ensureAdapterBegun() {
	q.adapterInitMutex.Lock()
	defer q.adapterInitMutex.Unlock()

	if q.adapterInProgress {
		return
	}

	adapterResultChan := make(chan transfer.TransferResult, 20)

	// Progress callback - receives byte updates
	cb := func(name string, total, read int64, current int) error {
		q.meter.TransferBytes(q.transferKind(), name, read, total, current)
		return nil
	}

	tracerx.Printf("tq: starting transfer adapter %q", q.adapter.Name())
	q.adapter.Begin(config.Config.ConcurrentTransfers(), cb, adapterResultChan)
	q.adapterInProgress = true

	// Collector for completed transfers
	// q.wait.Done() in handleTransferResult is enough to know when this is complete for all transfers
	go func() {
		for res := range adapterResultChan {
			q.handleTransferResult(res)
		}
	}()

}

func (q *TransferQueue) handleTransferResult(res transfer.TransferResult) {
	if res.Error != nil {
		if q.canRetry(res.Error) {
			tracerx.Printf("tq: retrying object %s", res.Transfer.Object.Oid)
			q.trMutex.Lock()
			t, ok := q.transferables[res.Transfer.Object.Oid]
			q.trMutex.Unlock()
			if ok {
				q.retry(t)
			} else {
				q.errorc <- res.Error
			}
		} else {
			q.errorc <- res.Error
		}
	} else {
		oid := res.Transfer.Object.Oid
		for _, c := range q.watchers {
			c <- oid
		}

		q.meter.FinishTransfer(res.Transfer.Name)
	}

	q.wait.Done()

}

// Wait waits for the queue to finish processing all transfers. Once Wait is
// called, Add will no longer add transferables to the queue. Any failed
// transfers will be automatically retried once.
func (q *TransferQueue) Wait() {
	if q.batcher != nil {
		q.batcher.Exit()
	}

	q.wait.Wait()

	// Handle any retries
	close(q.retriesc)
	q.retrywait.Wait()
	atomic.StoreUint32(&q.retrying, 1)

	if len(q.retries) > 0 {
		tracerx.Printf("tq: retrying %d failed transfers", len(q.retries))
		for _, t := range q.retries {
			q.Add(t)
		}
		if q.batcher != nil {
			q.batcher.Exit()
		}
		q.wait.Wait()
	}

	atomic.StoreUint32(&q.retrying, 0)

	close(q.apic)
	q.finishAdapter()
	close(q.errorc)

	for _, watcher := range q.watchers {
		close(watcher)
	}

	q.meter.Finish()
	q.errorwait.Wait()
}

// Watch returns a channel where the queue will write the OID of each transfer
// as it completes. The channel will be closed when the queue finishes processing.
func (q *TransferQueue) Watch() chan string {
	c := make(chan string, batchSize)
	q.watchers = append(q.watchers, c)
	return c
}

// individualApiRoutine processes the queue of transfers one at a time by making
// a POST call for each object, feeding the results to the transfer workers.
// If configured, the object transfers can still happen concurrently, the
// sequential nature here is only for the meta POST calls.
// TODO LEGACY API: remove when legacy API removed
func (q *TransferQueue) individualApiRoutine(apiWaiter chan interface{}) {
	for t := range q.apic {
		obj, err := t.LegacyCheck()
		if err != nil {
			if q.canRetry(err) {
				q.retry(t)
			} else {
				q.errorc <- err
			}
			q.wait.Done()
			continue
		}

		if apiWaiter != nil { // Signal to launch more individual api workers
			q.meter.Start()
			select {
			case apiWaiter <- 1:
			default:
			}
		}

		// Legacy API has no support for anything but basic transfer adapter
		q.useAdapter(transfer.BasicAdapterName)
		if obj != nil {
			t.SetObject(obj)
			q.meter.Add(t.Name())
			q.addToAdapter(t)
		} else {
			q.Skip(t.Size())
			q.wait.Done()
		}
	}
}

// legacyFallback is used when a batch request is made to a server that does
// not support the batch endpoint. When this happens, the Transferables are
// fed from the batcher into apic to be processed individually.
// TODO LEGACY API: remove when legacy API removed
func (q *TransferQueue) legacyFallback(failedBatch []interface{}) {
	tracerx.Printf("tq: batch api not implemented, falling back to individual")

	q.launchIndividualApiRoutines()

	for _, t := range failedBatch {
		q.apic <- t.(Transferable)
	}

	for {
		batch := q.batcher.Next()
		if batch == nil {
			break
		}

		for _, t := range batch {
			q.apic <- t.(Transferable)
		}
	}
}

// batchApiRoutine processes the queue of transfers using the batch endpoint,
// making only one POST call for all objects. The results are then handed
// off to the transfer workers.
func (q *TransferQueue) batchApiRoutine() {
	var startProgress sync.Once

	transferAdapterNames := transfer.GetAdapterNames(q.direction)

	for {
		batch := q.batcher.Next()
		if batch == nil {
			break
		}

		tracerx.Printf("tq: sending batch of size %d", len(batch))

		transfers := make([]*api.ObjectResource, 0, len(batch))
		for _, i := range batch {
			t := i.(Transferable)
			transfers = append(transfers, &api.ObjectResource{Oid: t.Oid(), Size: t.Size()})
		}

		if len(transfers) == 0 {
			continue
		}

		objs, adapterName, err := api.Batch(transfers, q.transferKind(), transferAdapterNames)
		if err != nil {
			if errutil.IsNotImplementedError(err) {
				git.Config.SetLocal("", "lfs.batch", "false")

				go q.legacyFallback(batch)
				return
			}

			if q.canRetry(err) {
				for _, t := range batch {
					q.retry(t.(Transferable))
				}
			} else {
				q.errorc <- err
			}

			q.wait.Add(-len(transfers))
			continue
		}

		q.useAdapter(adapterName)
		startProgress.Do(q.meter.Start)

		for _, o := range objs {
			if o.Error != nil {
				q.errorc <- errutil.Errorf(o.Error, "[%v] %v", o.Oid, o.Error.Message)
				q.Skip(o.Size)
				q.wait.Done()
				continue
			}

			if _, ok := o.Rel(q.transferKind()); ok {
				// This object needs to be transferred
				q.trMutex.Lock()
				transfer, ok := q.transferables[o.Oid]
				q.trMutex.Unlock()

				if ok {
					transfer.SetObject(o)
					q.meter.Add(transfer.Name())
					q.addToAdapter(transfer)
				} else {
					q.Skip(transfer.Size())
					q.wait.Done()
				}
			} else {
				q.Skip(o.Size)
				q.wait.Done()
			}
		}
	}
}

// This goroutine collects errors returned from transfers
func (q *TransferQueue) errorCollector() {
	for err := range q.errorc {
		q.errors = append(q.errors, err)
	}
	q.errorwait.Done()
}

func (q *TransferQueue) retryCollector() {
	for t := range q.retriesc {
		q.retries = append(q.retries, t)
	}
	q.retrywait.Done()
}

// launchIndividualApiRoutines first launches a single api worker. When it
// receives the first successful api request it launches workers - 1 more
// workers. This prevents being prompted for credentials multiple times at once
// when they're needed.
func (q *TransferQueue) launchIndividualApiRoutines() {
	go func() {
		apiWaiter := make(chan interface{})
		go q.individualApiRoutine(apiWaiter)

		<-apiWaiter

		for i := 0; i < q.oldApiWorkers-1; i++ {
			go q.individualApiRoutine(nil)
		}
	}()
}

// run starts the transfer queue, doing individual or batch transfers depending
// on the Config.BatchTransfer() value. run will transfer files sequentially or
// concurrently depending on the Config.ConcurrentTransfers() value.
func (q *TransferQueue) run() {
	go q.errorCollector()
	go q.retryCollector()

	if config.Config.BatchTransfer() {
		tracerx.Printf("tq: running as batched queue, batch size of %d", batchSize)
		q.batcher = NewBatcher(batchSize)
		go q.batchApiRoutine()
	} else {
		tracerx.Printf("tq: running as individual queue")
		q.launchIndividualApiRoutines()
	}
}

func (q *TransferQueue) retry(t Transferable) {
	q.retriesc <- t
}

func (q *TransferQueue) canRetry(err error) bool {
	if !errutil.IsRetriableError(err) || atomic.LoadUint32(&q.retrying) == 1 {
		return false
	}

	return true
}

// Errors returns any errors encountered during transfer.
func (q *TransferQueue) Errors() []error {
	return q.errors
}
<|endoftext|>


<commit_before>// Copyright 2017-2020 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package shlex

import (
	"fmt"
	"reflect"
	"testing"
)

func TestArgv(t *testing.T) {
	for i, tt := range []struct {
		desc string
		in   string
		want []string
	}{
		{
			desc: "nothing to do",
			in:   "stuff",
			want: []string{"stuff"},
		},
		{
			desc: "split",
			in:   "stuff more stuff",
			want: []string{"stuff", "more", "stuff"},
		},
		{
			desc: "escape",
			in:   "stuff\\ more stuff",
			want: []string{"stuff more", "stuff"},
		},
		{
			desc: "quote",
			in:   "stuff var='more stuff'",
			want: []string{"stuff", "var=more stuff"},
		},
		{
			desc: "double quote",
			in:   "stuff var=\"more stuff\"",
			want: []string{"stuff", "var=more stuff"},
		},
		{
			desc: "quote specials",
			in:   "stuff var='more stuff $ \\ \\$ \" '",
			want: []string{"stuff", "var=more stuff $ \\ \\$ \" "},
		},
		{
			desc: "double quote",
			in:   "stuff var=\"more stuff $ \\ \\$ \\\" \"",
			want: []string{"stuff", "var=more stuff $ \\ $ \" "},
		},
	} {
		t.Run(fmt.Sprintf("Test [%02d] %s", i, tt.desc), func(t *testing.T) {
			got := Argv(tt.in)
			if !reflect.DeepEqual(got, tt.want) {
				t.Errorf("Argv = %#v, want %#v", got, tt.want)
			}

		})
	}
}
<commit_msg>shlex test more<commit_after>// Copyright 2017-2020 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package shlex

import (
	"fmt"
	"reflect"
	"testing"
)

func TestArgv(t *testing.T) {
	for i, tt := range []struct {
		desc string
		in   string
		want []string
	}{
		{
			desc: "nothing to do",
			in:   "stuff",
			want: []string{"stuff"},
		},
		{
			desc: "split",
			in:   "stuff more stuff",
			want: []string{"stuff", "more", "stuff"},
		},
		{
			desc: "escape",
			in:   "stuff\\ more stuff",
			want: []string{"stuff more", "stuff"},
		},
		{
			desc: "quote",
			in:   "stuff var='more stuff'",
			want: []string{"stuff", "var=more stuff"},
		},
		{
			desc: "double quote",
			in:   "stuff var=\"more stuff\"",
			want: []string{"stuff", "var=more stuff"},
		},
		{
			desc: "quote specials",
			in:   "stuff var='more stuff $ \\ \\$ \" '",
			want: []string{"stuff", "var=more stuff $ \\ \\$ \" "},
		},
		{
			desc: "double quote",
			in:   "stuff var=\"more stuff $ \\ \\$ \\\" \"",
			want: []string{"stuff", "var=more stuff $ \\ $ \" "},
		},
		{
			desc: "quote forgot close",
			in:   "stuff var='more stuff",
			want: []string{"stuff", "var=more stuff"},
		},
		{
			desc: "empty",
			in:   "",
			want: nil,
		},
	} {
		t.Run(fmt.Sprintf("Test [%02d] %s", i, tt.desc), func(t *testing.T) {
			got := Argv(tt.in)
			if !reflect.DeepEqual(got, tt.want) {
				t.Errorf("Argv = %#v, want %#v", got, tt.want)
			}

		})
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2020 the Octant contributors. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0
 */

package tsgen

import (
	"bytes"
	"embed"
	"encoding/gob"
	"fmt"
	"go/ast"
	"go/build"
	"go/doc"
	"go/format"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"text/template"

	"github.com/iancoleman/strcase"
)

//go:embed ts-support
var tsSupportFiles embed.FS

var tsSupportDir string = "ts-support"

// TSGen generates typescript for components.
type TSGen struct {
	disableFormat bool
}

// NewTSGen creates an instance of TSGen.
func NewTSGen(options ...Option) (*TSGen, error) {
	opts := makeDefaultOptions(options...)

	t := &TSGen{
		disableFormat: opts.disableFormat,
	}
	return t, nil
}

// Names returns all the component names ina
func (tg *TSGen) Names(source string) ([]string, error) {
	fileSet := token.NewFileSet()

	// TODO: this may not be needed as we can glean this from source.
	dirs := []string{
		filepath.Join("pkg", "view", "component"),
	}

	var files []*ast.File
	for _, dir := range dirs {
		wd := filepath.Join(".", dir)

		pkg, err := build.ImportDir(wd, build.ImportComment)
		if err != nil {
			return nil, fmt.Errorf("import dir %s: %w", wd, err)
		}

		for _, goFile := range pkg.GoFiles {
			fileName := filepath.Join(pkg.Dir, goFile)
			file, err := parser.ParseFile(fileSet, fileName, nil, parser.ParseComments|parser.AllErrors)
			if err != nil {
				return nil, fmt.Errorf("parsing %s: %w", fileName, err)
			}
			files = append(files, file)
		}
	}

	names, err := componentsInPackage(fileSet, files, source)
	if err != nil {
		return nil, fmt.Errorf("find components in package: %w", err)
	}

	return names, nil
}

// Reflect reflects on list of components. It generates a temporary binary and runs that to
// get type information for components.
func (tg *TSGen) Reflect(names []string) (*Model, error) {
	dir, err := ioutil.TempDir(".", "reflect")
	if err != nil {
		return nil, err
	}

	defer func() {
		if cErr := os.RemoveAll(dir); cErr != nil {
			log.Fatalf("unable to remove temporary directory")
		}
	}()

	b, err := tg.ReflectTemplate(names)
	if err != nil {
		return nil, fmt.Errorf("create reflect program")
	}

	reflectProgram := filepath.Join(dir, "reflect.go")

	if err := ioutil.WriteFile(reflectProgram, b, 0666); err != nil {
		return nil, fmt.Errorf("create reflect program: %w", err)
	}

	cmd := exec.Command("go", "run", reflectProgram)

	var stdout bytes.Buffer
	cmd.Stdout = &stdout

	var stderr bytes.Buffer
	cmd.Stderr = &stderr

	if err := cmd.Run(); err != nil {
		log.Printf(stderr.String())
		return nil, fmt.Errorf("run reflect program: %w", err)
	}

	m := &Model{}
	if err := gob.NewDecoder(&stdout).Decode(m); err != nil {
		return nil, fmt.Errorf("decode reflet model: %w", err)
	}

	return m, nil
}

// ReflectTemplate generates a reflect template.
func (tg *TSGen) ReflectTemplate(names []string) ([]byte, error) {
	b, err := tsSupportFiles.ReadFile(filepath.Join(tsSupportDir, "reflect.go.tmpl"))
	if err != nil {
		return nil, fmt.Errorf("load reflect template: %w", err)
	}

	t, err := template.New("reflect").Parse(string(b))
	if err != nil {
		return nil, fmt.Errorf("parse reflect template: %w", err)
	}

	opts := reflectOptions{
		Names: names,
	}

	var buf bytes.Buffer
	if err := t.Execute(&buf, opts); err != nil {
		return nil, fmt.Errorf("execute reflect template: %w", err)
	}

	return format.Source(buf.Bytes())
}

// Stage generates typescript for all components.
func (tg *TSGen) Stage(dest string, model *Model) error {
	if model == nil {
		return fmt.Errorf("model is nil")
	}

	info, err := os.Stat(dest)
	if err == nil {
		if !info.IsDir() {
			return fmt.Errorf("%q is not a directory", dest)
		}
	} else {
		if !os.IsNotExist(err) {
			return err
		}

		if err := os.MkdirAll(dest, 0777); err != nil {
			return fmt.Errorf("create destination: %w", err)
		}
	}

	sc := stageControl{root: dest}

	if err := sc.writeComponentInterface(tg.Component); err != nil {
		return fmt.Errorf("write component interface: %w", err)
	}
	if err := sc.writeFactoryInterface(tg.ComponentFactory); err != nil {
		return fmt.Errorf("write component factory interface: %w", err)
	}

	for _, c := range model.Components {
		if err := sc.writeComponent(c, func() ([]byte, error) {
			return tg.ComponentConfig(c)
		}); err != nil {
			return fmt.Errorf("write component %s: %w", c.Name, err)
		}
	}

	return nil

}

// Component generates the typescript component interface.
func (tg *TSGen) Component() ([]byte, error) {
	b, err := tsSupportFiles.ReadFile(filepath.Join(tsSupportDir, "component.ts"))
	if err != nil {
		return nil, err
	}

	return tg.formatTypescript(b)
}

// ComponentFactory generates the typescript component factory interface.
func (tg *TSGen) ComponentFactory() ([]byte, error) {
	b, err := tsSupportFiles.ReadFile(filepath.Join(tsSupportDir, "component-factory.ts"))
	if err != nil {
		return nil, err
	}

	return tg.formatTypescript(b)
}

// ComponentConfig generates a config interface and factory for a model component.
func (tg *TSGen) ComponentConfig(c Component) ([]byte, error) {
	b, err := tsSupportFiles.ReadFile(filepath.Join(tsSupportDir, "type-config.ts.tmpl"))
	if err != nil {
		return nil, fmt.Errorf("load type-config template: %w", err)
	}

	t, err := template.New("component").Parse(string(b))
	if err != nil {
		return nil, fmt.Errorf("parse type-config template: %w", err)
	}

	opts := configOptions{
		Name:   c.Name,
		Type:   c.TSName,
		Fields: c.Fields,
		Refs:   c.Referenced(),
	}

	var buf bytes.Buffer
	if err := t.Execute(&buf, opts); err != nil {
		return nil, fmt.Errorf("execute type-config template: %w", err)
	}

	return tg.formatTypescript(buf.Bytes())
}

func (tg *TSGen) formatTypescript(in []byte) ([]byte, error) {
	if tg.disableFormat {
		return in, nil
	}

	tf := NewTSFormatter()
	out, err := tf.Format(in)
	if err != nil {
		return nil, fmt.Errorf("unable to format typescript: %w", err)
	}

	return out, nil
}

type reflectOptions struct {
	Names []string
}

type configOptions struct {
	Name   string
	Type   string
	Fields []Field
	Refs   []ImportReference
}

func (co configOptions) RequiredFields() []Field {
	var list []Field
	for _, f := range co.Fields {
		if !f.Optional {
			list = append(list, f)
		}
	}

	return list
}

func (co configOptions) HasOptionalFields() bool {
	return len(co.OptionalFields()) > 0
}

func (co configOptions) OptionalFields() []Field {
	var list []Field
	for _, f := range co.Fields {
		if f.Optional {
			list = append(list, f)
		}
	}

	return list
}

func componentsInPackage(fileSet *token.FileSet, files []*ast.File, importPath string) ([]string, error) {
	p, err := doc.NewFromFiles(fileSet, files, importPath)
	if err != nil {
		return nil, fmt.Errorf("load doc from files: %w", err)
	}

	var list []string

	for _, t := range p.Types {
		if doesCommentContainComponent(t.Doc) {
			list = append(list, t.Name)
		}
	}

	return list, nil
}

var reComponentComment = regexp.MustCompile(`(?m)^\+octant:component$`)

func doesCommentContainComponent(comment string) bool {
	return reComponentComment.MatchString(comment)
}

type stageControl struct {
	root string
}

type stageGenFunc func() ([]byte, error)

func (sc stageControl) writeComponentInterface(fn stageGenFunc) error {
	p := filepath.Join(sc.root, "component.ts")
	return sc.write(fn, p)
}

func (sc stageControl) writeFactoryInterface(fn stageGenFunc) error {
	p := filepath.Join(sc.root, "component-factory.ts")
	return sc.write(fn, p)
}

func (sc stageControl) writeComponent(c Component, fn stageGenFunc) error {
	componentName := strcase.ToKebab(c.TSName)

	p := filepath.Join(sc.root, componentName+".ts")
	return sc.write(fn, p)
}

func (sc stageControl) write(fn stageGenFunc, dest string) error {
	b, err := fn()
	if err != nil {
		return err
	}

	return ioutil.WriteFile(dest, b, 0644)
}
<commit_msg>Use path.Join to find embedded files<commit_after>/*
 * Copyright (c) 2020 the Octant contributors. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0
 */

package tsgen

import (
	"bytes"
	"embed"
	"encoding/gob"
	"fmt"
	"go/ast"
	"go/build"
	"go/doc"
	"go/format"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"text/template"

	"github.com/iancoleman/strcase"
)

//go:embed ts-support
var tsSupportFiles embed.FS

var tsSupportDir string = "ts-support"

// TSGen generates typescript for components.
type TSGen struct {
	disableFormat bool
}

// NewTSGen creates an instance of TSGen.
func NewTSGen(options ...Option) (*TSGen, error) {
	opts := makeDefaultOptions(options...)

	t := &TSGen{
		disableFormat: opts.disableFormat,
	}
	return t, nil
}

// Names returns all the component names ina
func (tg *TSGen) Names(source string) ([]string, error) {
	fileSet := token.NewFileSet()

	// TODO: this may not be needed as we can glean this from source.
	dirs := []string{
		filepath.Join("pkg", "view", "component"),
	}

	var files []*ast.File
	for _, dir := range dirs {
		wd := filepath.Join(".", dir)

		pkg, err := build.ImportDir(wd, build.ImportComment)
		if err != nil {
			return nil, fmt.Errorf("import dir %s: %w", wd, err)
		}

		for _, goFile := range pkg.GoFiles {
			fileName := filepath.Join(pkg.Dir, goFile)
			file, err := parser.ParseFile(fileSet, fileName, nil, parser.ParseComments|parser.AllErrors)
			if err != nil {
				return nil, fmt.Errorf("parsing %s: %w", fileName, err)
			}
			files = append(files, file)
		}
	}

	names, err := componentsInPackage(fileSet, files, source)
	if err != nil {
		return nil, fmt.Errorf("find components in package: %w", err)
	}

	return names, nil
}

// Reflect reflects on list of components. It generates a temporary binary and runs that to
// get type information for components.
func (tg *TSGen) Reflect(names []string) (*Model, error) {
	dir, err := ioutil.TempDir(".", "reflect")
	if err != nil {
		return nil, err
	}

	defer func() {
		if cErr := os.RemoveAll(dir); cErr != nil {
			log.Fatalf("unable to remove temporary directory")
		}
	}()

	b, err := tg.ReflectTemplate(names)
	if err != nil {
		return nil, fmt.Errorf("create reflect program")
	}

	reflectProgram := filepath.Join(dir, "reflect.go")

	if err := ioutil.WriteFile(reflectProgram, b, 0666); err != nil {
		return nil, fmt.Errorf("create reflect program: %w", err)
	}

	cmd := exec.Command("go", "run", reflectProgram)

	var stdout bytes.Buffer
	cmd.Stdout = &stdout

	var stderr bytes.Buffer
	cmd.Stderr = &stderr

	if err := cmd.Run(); err != nil {
		log.Printf(stderr.String())
		return nil, fmt.Errorf("run reflect program: %w", err)
	}

	m := &Model{}
	if err := gob.NewDecoder(&stdout).Decode(m); err != nil {
		return nil, fmt.Errorf("decode reflet model: %w", err)
	}

	return m, nil
}

// ReflectTemplate generates a reflect template.
func (tg *TSGen) ReflectTemplate(names []string) ([]byte, error) {
	b, err := tsSupportFiles.ReadFile(path.Join(tsSupportDir, "reflect.go.tmpl"))
	if err != nil {
		return nil, fmt.Errorf("load reflect template: %w", err)
	}

	t, err := template.New("reflect").Parse(string(b))
	if err != nil {
		return nil, fmt.Errorf("parse reflect template: %w", err)
	}

	opts := reflectOptions{
		Names: names,
	}

	var buf bytes.Buffer
	if err := t.Execute(&buf, opts); err != nil {
		return nil, fmt.Errorf("execute reflect template: %w", err)
	}

	return format.Source(buf.Bytes())
}

// Stage generates typescript for all components.
func (tg *TSGen) Stage(dest string, model *Model) error {
	if model == nil {
		return fmt.Errorf("model is nil")
	}

	info, err := os.Stat(dest)
	if err == nil {
		if !info.IsDir() {
			return fmt.Errorf("%q is not a directory", dest)
		}
	} else {
		if !os.IsNotExist(err) {
			return err
		}

		if err := os.MkdirAll(dest, 0777); err != nil {
			return fmt.Errorf("create destination: %w", err)
		}
	}

	sc := stageControl{root: dest}

	if err := sc.writeComponentInterface(tg.Component); err != nil {
		return fmt.Errorf("write component interface: %w", err)
	}
	if err := sc.writeFactoryInterface(tg.ComponentFactory); err != nil {
		return fmt.Errorf("write component factory interface: %w", err)
	}

	for _, c := range model.Components {
		if err := sc.writeComponent(c, func() ([]byte, error) {
			return tg.ComponentConfig(c)
		}); err != nil {
			return fmt.Errorf("write component %s: %w", c.Name, err)
		}
	}

	return nil

}

// Component generates the typescript component interface.
func (tg *TSGen) Component() ([]byte, error) {
	b, err := tsSupportFiles.ReadFile(path.Join(tsSupportDir, "component.ts"))
	if err != nil {
		return nil, err
	}

	return tg.formatTypescript(b)
}

// ComponentFactory generates the typescript component factory interface.
func (tg *TSGen) ComponentFactory() ([]byte, error) {
	b, err := tsSupportFiles.ReadFile(path.Join(tsSupportDir, "component-factory.ts"))
	if err != nil {
		return nil, err
	}

	return tg.formatTypescript(b)
}

// ComponentConfig generates a config interface and factory for a model component.
func (tg *TSGen) ComponentConfig(c Component) ([]byte, error) {
	b, err := tsSupportFiles.ReadFile(path.Join(tsSupportDir, "type-config.ts.tmpl"))
	if err != nil {
		return nil, fmt.Errorf("load type-config template: %w", err)
	}

	t, err := template.New("component").Parse(string(b))
	if err != nil {
		return nil, fmt.Errorf("parse type-config template: %w", err)
	}

	opts := configOptions{
		Name:   c.Name,
		Type:   c.TSName,
		Fields: c.Fields,
		Refs:   c.Referenced(),
	}

	var buf bytes.Buffer
	if err := t.Execute(&buf, opts); err != nil {
		return nil, fmt.Errorf("execute type-config template: %w", err)
	}

	return tg.formatTypescript(buf.Bytes())
}

func (tg *TSGen) formatTypescript(in []byte) ([]byte, error) {
	if tg.disableFormat {
		return in, nil
	}

	tf := NewTSFormatter()
	out, err := tf.Format(in)
	if err != nil {
		return nil, fmt.Errorf("unable to format typescript: %w", err)
	}

	return out, nil
}

type reflectOptions struct {
	Names []string
}

type configOptions struct {
	Name   string
	Type   string
	Fields []Field
	Refs   []ImportReference
}

func (co configOptions) RequiredFields() []Field {
	var list []Field
	for _, f := range co.Fields {
		if !f.Optional {
			list = append(list, f)
		}
	}

	return list
}

func (co configOptions) HasOptionalFields() bool {
	return len(co.OptionalFields()) > 0
}

func (co configOptions) OptionalFields() []Field {
	var list []Field
	for _, f := range co.Fields {
		if f.Optional {
			list = append(list, f)
		}
	}

	return list
}

func componentsInPackage(fileSet *token.FileSet, files []*ast.File, importPath string) ([]string, error) {
	p, err := doc.NewFromFiles(fileSet, files, importPath)
	if err != nil {
		return nil, fmt.Errorf("load doc from files: %w", err)
	}

	var list []string

	for _, t := range p.Types {
		if doesCommentContainComponent(t.Doc) {
			list = append(list, t.Name)
		}
	}

	return list, nil
}

var reComponentComment = regexp.MustCompile(`(?m)^\+octant:component$`)

func doesCommentContainComponent(comment string) bool {
	return reComponentComment.MatchString(comment)
}

type stageControl struct {
	root string
}

type stageGenFunc func() ([]byte, error)

func (sc stageControl) writeComponentInterface(fn stageGenFunc) error {
	p := filepath.Join(sc.root, "component.ts")
	return sc.write(fn, p)
}

func (sc stageControl) writeFactoryInterface(fn stageGenFunc) error {
	p := filepath.Join(sc.root, "component-factory.ts")
	return sc.write(fn, p)
}

func (sc stageControl) writeComponent(c Component, fn stageGenFunc) error {
	componentName := strcase.ToKebab(c.TSName)

	p := filepath.Join(sc.root, componentName+".ts")
	return sc.write(fn, p)
}

func (sc stageControl) write(fn stageGenFunc, dest string) error {
	b, err := fn()
	if err != nil {
		return err
	}

	return ioutil.WriteFile(dest, b, 0644)
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ast_test

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/token"
)

// This example demonstrates how to inspect the AST of a Go program.
func ExampleInspect() {
	// src is the input for which we want to inspect the AST.
	src := `
package p
const c = 1.0
var X = f(3.14)*2 + c
`

	// Create the AST by parsing src.
	fset := token.NewFileSet() // positions are relative to fset
	f, err := parser.ParseFile(fset, "src.go", src, 0)
	if err != nil {
		panic(err)
	}

	// Inspect the AST and print all identifiers and literals.
	ast.Inspect(f, func(n ast.Node) bool {
		var s string
		switch x := n.(type) {
		case *ast.BasicLit:
			s = x.Value
		case *ast.Ident:
			s = x.Name
		}
		if s != "" {
			fmt.Printf("%s:\t%s\n", fset.Position(n.Pos()), s)
		}
		return true
	})

	// output:
	// src.go:2:9:	p
	// src.go:3:7:	c
	// src.go:3:11:	1.0
	// src.go:4:5:	X
	// src.go:4:9:	f
	// src.go:4:11:	3.14
	// src.go:4:17:	2
	// src.go:4:21:	c
}

// This example shows what an AST looks like when printed for debugging.
func ExamplePrint() {
	// src is the input for which we want to print the AST.
	src := `
package main
func main() {
	println("Hello, World!")
}
`

	// Create the AST by parsing src.
	fset := token.NewFileSet() // positions are relative to fset
	f, err := parser.ParseFile(fset, "", src, 0)
	if err != nil {
		panic(err)
	}

	// Print the AST.
	ast.Print(fset, f)

	// output:
	//      0  *ast.File {
	//      1  .  Package: 2:1
	//      2  .  Name: *ast.Ident {
	//      3  .  .  NamePos: 2:9
	//      4  .  .  Name: "main"
	//      5  .  }
	//      6  .  Decls: []ast.Decl (len = 1) {
	//      7  .  .  0: *ast.FuncDecl {
	//      8  .  .  .  Name: *ast.Ident {
	//      9  .  .  .  .  NamePos: 3:6
	//     10  .  .  .  .  Name: "main"
	//     11  .  .  .  .  Obj: *ast.Object {
	//     12  .  .  .  .  .  Kind: func
	//     13  .  .  .  .  .  Name: "main"
	//     14  .  .  .  .  .  Decl: *(obj @ 7)
	//     15  .  .  .  .  }
	//     16  .  .  .  }
	//     17  .  .  .  Type: *ast.FuncType {
	//     18  .  .  .  .  Func: 3:1
	//     19  .  .  .  .  Params: *ast.FieldList {
	//     20  .  .  .  .  .  Opening: 3:10
	//     21  .  .  .  .  .  Closing: 3:11
	//     22  .  .  .  .  }
	//     23  .  .  .  }
	//     24  .  .  .  Body: *ast.BlockStmt {
	//     25  .  .  .  .  Lbrace: 3:13
	//     26  .  .  .  .  List: []ast.Stmt (len = 1) {
	//     27  .  .  .  .  .  0: *ast.ExprStmt {
	//     28  .  .  .  .  .  .  X: *ast.CallExpr {
	//     29  .  .  .  .  .  .  .  Fun: *ast.Ident {
	//     30  .  .  .  .  .  .  .  .  NamePos: 4:2
	//     31  .  .  .  .  .  .  .  .  Name: "println"
	//     32  .  .  .  .  .  .  .  }
	//     33  .  .  .  .  .  .  .  Lparen: 4:9
	//     34  .  .  .  .  .  .  .  Args: []ast.Expr (len = 1) {
	//     35  .  .  .  .  .  .  .  .  0: *ast.BasicLit {
	//     36  .  .  .  .  .  .  .  .  .  ValuePos: 4:10
	//     37  .  .  .  .  .  .  .  .  .  Kind: STRING
	//     38  .  .  .  .  .  .  .  .  .  Value: "\"Hello, World!\""
	//     39  .  .  .  .  .  .  .  .  }
	//     40  .  .  .  .  .  .  .  }
	//     41  .  .  .  .  .  .  .  Ellipsis: -
	//     42  .  .  .  .  .  .  .  Rparen: 4:25
	//     43  .  .  .  .  .  .  }
	//     44  .  .  .  .  .  }
	//     45  .  .  .  .  }
	//     46  .  .  .  .  Rbrace: 5:1
	//     47  .  .  .  }
	//     48  .  .  }
	//     49  .  }
	//     50  .  Scope: *ast.Scope {
	//     51  .  .  Objects: map[string]*ast.Object (len = 1) {
	//     52  .  .  .  "main": *(obj @ 11)
	//     53  .  .  }
	//     54  .  }
	//     55  .  Unresolved: []*ast.Ident (len = 1) {
	//     56  .  .  0: *(obj @ 29)
	//     57  .  }
	//     58  }
}

// This example illustrates how to remove a variable declaration
// in a Go program while maintaining correct comment association
// using an ast.CommentMap.
func ExampleCommentMap() {
	// src is the input for which we create the AST that we
	// are going to manipulate.
	src := `
// This is the package comment.
package main

// This comment is associated with the hello constant.
const hello = "Hello, World!" // line comment 1

// This comment is associated with the foo variable.
var foo = hello // line comment 2

// This comment is associated with the main function.
func main() {
	fmt.Println(hello) // line comment 3
}
`

	// Create the AST by parsing src.
	fset := token.NewFileSet() // positions are relative to fset
	f, err := parser.ParseFile(fset, "src.go", src, parser.ParseComments)
	if err != nil {
		panic(err)
	}

	// Create an ast.CommentMap from the ast.File's comments.
	// This helps keeping the association between comments
	// and AST nodes.
	cmap := ast.NewCommentMap(fset, f, f.Comments)

	// Remove the first variable declaration from the list of declarations.
	f.Decls = removeFirstVarDecl(f.Decls)

	// Use the comment map to filter comments that don't belong anymore
	// (the comments associated with the variable declaration), and create
	// the new comments list.
	f.Comments = cmap.Filter(f).Comments()

	// Print the modified AST.
	var buf bytes.Buffer
	if err := format.Node(&buf, fset, f); err != nil {
		panic(err)
	}
	fmt.Printf("%s", buf.Bytes())

	// output:
	// // This is the package comment.
	// package main
	//
	// // This comment is associated with the hello constant.
	// const hello = "Hello, World!" // line comment 1
	//
	// // This comment is associated with the main function.
	// func main() {
	// 	fmt.Println(hello) // line comment 3
	// }
}

func removeFirstVarDecl(list []ast.Decl) []ast.Decl {
	for i, decl := range list {
		if gen, ok := decl.(*ast.GenDecl); ok && gen.Tok == token.VAR {
			copy(list[i:], list[i+1:])
			return list[:len(list)-1]
		}
	}
	panic("variable declaration not found")
}
<commit_msg>ast: make ExampleCommentMap a runnable example<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ast_test

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/token"
)

// This example demonstrates how to inspect the AST of a Go program.
func ExampleInspect() {
	// src is the input for which we want to inspect the AST.
	src := `
package p
const c = 1.0
var X = f(3.14)*2 + c
`

	// Create the AST by parsing src.
	fset := token.NewFileSet() // positions are relative to fset
	f, err := parser.ParseFile(fset, "src.go", src, 0)
	if err != nil {
		panic(err)
	}

	// Inspect the AST and print all identifiers and literals.
	ast.Inspect(f, func(n ast.Node) bool {
		var s string
		switch x := n.(type) {
		case *ast.BasicLit:
			s = x.Value
		case *ast.Ident:
			s = x.Name
		}
		if s != "" {
			fmt.Printf("%s:\t%s\n", fset.Position(n.Pos()), s)
		}
		return true
	})

	// Output:
	// src.go:2:9:	p
	// src.go:3:7:	c
	// src.go:3:11:	1.0
	// src.go:4:5:	X
	// src.go:4:9:	f
	// src.go:4:11:	3.14
	// src.go:4:17:	2
	// src.go:4:21:	c
}

// This example shows what an AST looks like when printed for debugging.
func ExamplePrint() {
	// src is the input for which we want to print the AST.
	src := `
package main
func main() {
	println("Hello, World!")
}
`

	// Create the AST by parsing src.
	fset := token.NewFileSet() // positions are relative to fset
	f, err := parser.ParseFile(fset, "", src, 0)
	if err != nil {
		panic(err)
	}

	// Print the AST.
	ast.Print(fset, f)

	// Output:
	//      0  *ast.File {
	//      1  .  Package: 2:1
	//      2  .  Name: *ast.Ident {
	//      3  .  .  NamePos: 2:9
	//      4  .  .  Name: "main"
	//      5  .  }
	//      6  .  Decls: []ast.Decl (len = 1) {
	//      7  .  .  0: *ast.FuncDecl {
	//      8  .  .  .  Name: *ast.Ident {
	//      9  .  .  .  .  NamePos: 3:6
	//     10  .  .  .  .  Name: "main"
	//     11  .  .  .  .  Obj: *ast.Object {
	//     12  .  .  .  .  .  Kind: func
	//     13  .  .  .  .  .  Name: "main"
	//     14  .  .  .  .  .  Decl: *(obj @ 7)
	//     15  .  .  .  .  }
	//     16  .  .  .  }
	//     17  .  .  .  Type: *ast.FuncType {
	//     18  .  .  .  .  Func: 3:1
	//     19  .  .  .  .  Params: *ast.FieldList {
	//     20  .  .  .  .  .  Opening: 3:10
	//     21  .  .  .  .  .  Closing: 3:11
	//     22  .  .  .  .  }
	//     23  .  .  .  }
	//     24  .  .  .  Body: *ast.BlockStmt {
	//     25  .  .  .  .  Lbrace: 3:13
	//     26  .  .  .  .  List: []ast.Stmt (len = 1) {
	//     27  .  .  .  .  .  0: *ast.ExprStmt {
	//     28  .  .  .  .  .  .  X: *ast.CallExpr {
	//     29  .  .  .  .  .  .  .  Fun: *ast.Ident {
	//     30  .  .  .  .  .  .  .  .  NamePos: 4:2
	//     31  .  .  .  .  .  .  .  .  Name: "println"
	//     32  .  .  .  .  .  .  .  }
	//     33  .  .  .  .  .  .  .  Lparen: 4:9
	//     34  .  .  .  .  .  .  .  Args: []ast.Expr (len = 1) {
	//     35  .  .  .  .  .  .  .  .  0: *ast.BasicLit {
	//     36  .  .  .  .  .  .  .  .  .  ValuePos: 4:10
	//     37  .  .  .  .  .  .  .  .  .  Kind: STRING
	//     38  .  .  .  .  .  .  .  .  .  Value: "\"Hello, World!\""
	//     39  .  .  .  .  .  .  .  .  }
	//     40  .  .  .  .  .  .  .  }
	//     41  .  .  .  .  .  .  .  Ellipsis: -
	//     42  .  .  .  .  .  .  .  Rparen: 4:25
	//     43  .  .  .  .  .  .  }
	//     44  .  .  .  .  .  }
	//     45  .  .  .  .  }
	//     46  .  .  .  .  Rbrace: 5:1
	//     47  .  .  .  }
	//     48  .  .  }
	//     49  .  }
	//     50  .  Scope: *ast.Scope {
	//     51  .  .  Objects: map[string]*ast.Object (len = 1) {
	//     52  .  .  .  "main": *(obj @ 11)
	//     53  .  .  }
	//     54  .  }
	//     55  .  Unresolved: []*ast.Ident (len = 1) {
	//     56  .  .  0: *(obj @ 29)
	//     57  .  }
	//     58  }
}

// This example illustrates how to remove a variable declaration
// in a Go program while maintaining correct comment association
// using an ast.CommentMap.
func ExampleCommentMap() {
	// src is the input for which we create the AST that we
	// are going to manipulate.
	src := `
// This is the package comment.
package main

// This comment is associated with the hello constant.
const hello = "Hello, World!" // line comment 1

// This comment is associated with the foo variable.
var foo = hello // line comment 2

// This comment is associated with the main function.
func main() {
	fmt.Println(hello) // line comment 3
}
`

	// Create the AST by parsing src.
	fset := token.NewFileSet() // positions are relative to fset
	f, err := parser.ParseFile(fset, "src.go", src, parser.ParseComments)
	if err != nil {
		panic(err)
	}

	// Create an ast.CommentMap from the ast.File's comments.
	// This helps keeping the association between comments
	// and AST nodes.
	cmap := ast.NewCommentMap(fset, f, f.Comments)

	// Remove the first variable declaration from the list of declarations.
	for i, decl := range f.Decls {
		if gen, ok := decl.(*ast.GenDecl); ok && gen.Tok == token.VAR {
			copy(f.Decls[i:], f.Decls[i+1:])
			f.Decls = f.Decls[:len(f.Decls)-1]
		}
	}

	// Use the comment map to filter comments that don't belong anymore
	// (the comments associated with the variable declaration), and create
	// the new comments list.
	f.Comments = cmap.Filter(f).Comments()

	// Print the modified AST.
	var buf bytes.Buffer
	if err := format.Node(&buf, fset, f); err != nil {
		panic(err)
	}
	fmt.Printf("%s", buf.Bytes())

	// Output:
	// // This is the package comment.
	// package main
	//
	// // This comment is associated with the hello constant.
	// const hello = "Hello, World!" // line comment 1
	//
	// // This comment is associated with the main function.
	// func main() {
	// 	fmt.Println(hello) // line comment 3
	// }
}
<|endoftext|>


<commit_before>// Copyright © 2009 Esko Luontola <www.orfjackal.net>
// This software is released under the Apache License 2.0.
// The license text is at http://www.apache.org/licenses/LICENSE-2.0

package gospec

import (
	"fmt"
	"io"
)

type PrintFormat interface {
	PrintPassing(nestingLevel int, name string)
	PrintFailing(nestingLevel int, name string, errors []*Error)
	PrintSummary(passCount int, failCount int)
}

// PrintFormat for production use.
func DefaultPrintFormat(out io.Writer) PrintFormat {
	return &defaultPrintFormat{out}
}

type defaultPrintFormat struct {
	out io.Writer
}

func (this *defaultPrintFormat) PrintPassing(nestingLevel int, name string) {
	if nestingLevel == 0 {
		fmt.Fprintf(this.out, "\n%v\n", name)
	} else {
		indent := indent(nestingLevel)
		fmt.Fprintf(this.out, "%v- %v\n", indent, name)
	}
}

func (this *defaultPrintFormat) PrintFailing(nestingLevel int, name string, errors []*Error) {
	indent := indent(nestingLevel)
	// TODO: use colors (red)
	fmt.Fprintf(this.out, "%v- %v [FAIL]\n", indent, name)
	for _, error := range errors {
		fmt.Fprintf(this.out, "[ERROR] %v\n\t%v\n", error.Location, error.Message)
	}
}

func (this *defaultPrintFormat) PrintSummary(passCount int, failCount int) {
	totalCount := passCount + failCount
	// TODO: use colors (red if failures, else green)
	fmt.Fprintf(this.out, "\n%v specs, %v failures\n", totalCount, failCount)
}

// PrintFormat for use in only tests. Does not print line numbers, colors or
// other fancy stuff. Makes comparing as a string easier.
func SimplePrintFormat(out io.Writer) PrintFormat {
	return &simplePrintFormat{out}
}

type simplePrintFormat struct {
	out io.Writer
}

func (this *simplePrintFormat) PrintPassing(nestingLevel int, name string) {
	indent := indent(nestingLevel)
	fmt.Fprintf(this.out, "%v- %v\n", indent, name)
}

func (this *simplePrintFormat) PrintFailing(nestingLevel int, name string, errors []*Error) {
	indent := indent(nestingLevel)
	fmt.Fprintf(this.out, "%v- %v [FAIL]\n", indent, name)
	for _, error := range errors {
		fmt.Fprintf(this.out, "%v    %v\n", indent, error.Message)
	}
}

func (this *simplePrintFormat) PrintSummary(passCount int, failCount int) {
	totalCount := passCount + failCount
	fmt.Fprintf(this.out, "\n%v specs, %v failures\n", totalCount, failCount)
}

func indent(level int) string {
	s := ""
	for i := 0; i < level; i++ {
		s += "  "
	}
	return s
}

<commit_msg>More empty space after error messages, to keep them separate from the next spec's name<commit_after>// Copyright © 2009 Esko Luontola <www.orfjackal.net>
// This software is released under the Apache License 2.0.
// The license text is at http://www.apache.org/licenses/LICENSE-2.0

package gospec

import (
	"fmt"
	"io"
)

type PrintFormat interface {
	PrintPassing(nestingLevel int, name string)
	PrintFailing(nestingLevel int, name string, errors []*Error)
	PrintSummary(passCount int, failCount int)
}

// PrintFormat for production use.
func DefaultPrintFormat(out io.Writer) PrintFormat {
	return &defaultPrintFormat{out}
}

type defaultPrintFormat struct {
	out io.Writer
}

func (this *defaultPrintFormat) PrintPassing(nestingLevel int, name string) {
	if nestingLevel == 0 {
		fmt.Fprintf(this.out, "\n%v\n", name)
	} else {
		indent := indent(nestingLevel)
		fmt.Fprintf(this.out, "%v- %v\n", indent, name)
	}
}

func (this *defaultPrintFormat) PrintFailing(nestingLevel int, name string, errors []*Error) {
	indent := indent(nestingLevel)
	// TODO: use colors (red)
	fmt.Fprintf(this.out, "%v- %v [FAIL]\n", indent, name)
	for _, error := range errors {
		fmt.Fprintf(this.out, "[ERROR] %v\n\t%v\n", error.Location, error.Message)
	}
	fmt.Fprint(this.out, "\n")
}

func (this *defaultPrintFormat) PrintSummary(passCount int, failCount int) {
	totalCount := passCount + failCount
	// TODO: use colors (red if failures, else green)
	fmt.Fprintf(this.out, "\n%v specs, %v failures\n", totalCount, failCount)
}

// PrintFormat for use in only tests. Does not print line numbers, colors or
// other fancy stuff. Makes comparing as a string easier.
func SimplePrintFormat(out io.Writer) PrintFormat {
	return &simplePrintFormat{out}
}

type simplePrintFormat struct {
	out io.Writer
}

func (this *simplePrintFormat) PrintPassing(nestingLevel int, name string) {
	indent := indent(nestingLevel)
	fmt.Fprintf(this.out, "%v- %v\n", indent, name)
}

func (this *simplePrintFormat) PrintFailing(nestingLevel int, name string, errors []*Error) {
	indent := indent(nestingLevel)
	fmt.Fprintf(this.out, "%v- %v [FAIL]\n", indent, name)
	for _, error := range errors {
		fmt.Fprintf(this.out, "%v    %v\n", indent, error.Message)
	}
}

func (this *simplePrintFormat) PrintSummary(passCount int, failCount int) {
	totalCount := passCount + failCount
	fmt.Fprintf(this.out, "\n%v specs, %v failures\n", totalCount, failCount)
}

func indent(level int) string {
	s := ""
	for i := 0; i < level; i++ {
		s += "  "
	}
	return s
}

<|endoftext|>


<commit_before>package permissions

import "testing"

func TestPerm(t *testing.T) {
	userstate := NewUserStateSimple()

	userstate.AddUser("bob", "hunter1", "bob@zombo.com")

	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}

	if userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should not be confirmed right now.")
	}

	userstate.MarkConfirmed("bob")

	if !userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should be marked as confirmed right now.")
	}

	if userstate.IsAdmin("bob") {
		t.Error("Error, user bob should not have admin rights")
	}

	userstate.SetAdminStatus("bob")

	if !userstate.IsAdmin("bob") {
		t.Error("Error, user bob should have admin rights")
	}

	userstate.RemoveUser("bob")

	if userstate.HasUser("bob") {
		t.Error("Error, user bob should not exist")
	}
}

func TestPasswordBasic(t *testing.T) {
	userstate := NewUserStateSimple()

	// assert default password aglo is sha256
	if userstate.PasswordAlgo() != "bcrypt" {
		t.Error("Error, bcrypt should be default password algorithm")
	}

	// set password algo
	userstate.SetPasswordAlgo("sha256")

	// assert change should be bcrypt
	if userstate.PasswordAlgo() != "sha256" {
		t.Error("Error, setting password algorithm failed")
	}

}

func TestPasswordAlgoMatching(t *testing.T) {
	userstate := NewUserStateSimple()
	// generate two different password using the same credentials but different algos
	userstate.SetPasswordAlgo("sha256")
	sha256_hash := userstate.HashPassword("testuser@example.com", "textpassword")
	userstate.SetPasswordAlgo("bcrypt")
	bcrypt_hash := userstate.HashPassword("testuser@example.com", "textpassword")

	// they shouldn't match
	if sha256_hash == bcrypt_hash {
		t.Error("Error, different algorithms should not have a password match")
	}
}

func TestUserStateKeeper(t *testing.T) {
	userstate := NewUserStateSimple()
	// Check that the userstate qualifies for the UserStateKeeper interface
	var _ UserStateKeeper = userstate
}
<commit_msg>Fixing the test<commit_after>package permissions

import "testing"

func TestPerm(t *testing.T) {
	userstate := NewUserStateSimple()

	userstate.AddUser("bob", "hunter1", "bob@zombo.com")

	if !userstate.HasUser("bob") {
		t.Error("Error, user bob should exist")
	}

	if userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should not be confirmed right now.")
	}

	userstate.MarkConfirmed("bob")

	if !userstate.IsConfirmed("bob") {
		t.Error("Error, user bob should be marked as confirmed right now.")
	}

	if userstate.IsAdmin("bob") {
		t.Error("Error, user bob should not have admin rights")
	}

	userstate.SetAdminStatus("bob")

	if !userstate.IsAdmin("bob") {
		t.Error("Error, user bob should have admin rights")
	}

	userstate.RemoveUser("bob")

	if userstate.HasUser("bob") {
		t.Error("Error, user bob should not exist")
	}
}

func TestPasswordBasic(t *testing.T) {
	userstate := NewUserStateSimple()

	// assert that the default password algorithm is bcrypt
	if userstate.PasswordAlgo() != "bcrypt" {
		t.Error("Error, bcrypt should be the default password algorithm")
	}

	// set password algo
	userstate.SetPasswordAlgo("sha256")

	// assert change should be sha256
	if userstate.PasswordAlgo() != "sha256" {
		t.Error("Error, setting password algorithm failed")
	}

}

func TestPasswordAlgoMatching(t *testing.T) {
	userstate := NewUserStateSimple()
	// generate two different password using the same credentials but different algos
	userstate.SetPasswordAlgo("sha256")
	sha256_hash := userstate.HashPassword("testuser@example.com", "textpassword")
	userstate.SetPasswordAlgo("bcrypt")
	bcrypt_hash := userstate.HashPassword("testuser@example.com", "textpassword")

	// they shouldn't match
	if sha256_hash == bcrypt_hash {
		t.Error("Error, different algorithms should not have a password match")
	}
}

func TestUserStateKeeper(t *testing.T) {
	userstate := NewUserStateSimple()
	// Check that the userstate qualifies for the UserStateKeeper interface
	var _ UserStateKeeper = userstate
}
<|endoftext|>


<commit_before>// Package odin ...
package odin

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"

	"github.com/jcelliott/lumber"

	"github.com/nanobox-io/nanobox/models"
	"github.com/nanobox-io/nanobox/util"
)

const (
	NANOBOX  = "https://api.nanobox.io/v1/"
	BONESALT = "https://api.bonesalt.com/v1/"
	DEV      = "http://api.nanobox.dev:8080/v1/"
	SIM      = "http://api.nanobox.sim/v1/"
)

var (
	// set the default endpoint to nanobox
	endpoint = "nanobox"
)

type (
	evar struct {
		ID    string `json:"id"`
		Key   string `json:"title"`
		Value string `json:"value"`
	}
)

// sets the odin endpoint
func SetEndpoint(stage string) {
	endpoint = stage
}

// Auth ...
func Auth(username, password string) (string, error) {

	//
	params := url.Values{}
	params.Set("password", password)

	//
	resBody := map[string]string{}

	//
	if err := doRequest("GET", fmt.Sprintf("users/%s/auth_token", username), params, nil, &resBody); err != nil {
		return "", err
	}

	return resBody["authentication_token"], nil
}

// App ...
func App(slug string) (models.App, error) {
	app := models.App{}

	return app, doRequest("GET", "apps/"+slug, nil, nil, &app)
}

// Deploy ...
func Deploy(appID, id, boxfile, message string) error {

	//
	body := map[string]map[string]string{
		"deploy": {
			"boxfile_content": boxfile,
			"build_id":        id,
			"commit_message":  message,
		},
	}

	return doRequest("POST", fmt.Sprintf("apps/%s/deploys", appID), nil, body, nil)
}

func ListEvars(appID string) ([]evar, error) {
	evars := []evar{}
	return evars, doRequest("GET", fmt.Sprintf("apps/%s/evars", appID), nil, nil, &evars)
}

func AddEvar(appID, key, val string) error {
	body := map[string]map[string]string{
		"evar": {
			"title": key,
			"value": val,
		},
	}

	return doRequest("POST", fmt.Sprintf("apps/%s/evars", appID), nil, body, nil)
}

func RemoveEvar(appId, id string) error {
	return doRequest("DELETE", fmt.Sprintf("apps/%s/evars/%s", appId, id), nil, nil, nil)
}

// EstablishTunnel ...
func EstablishTunnel(appID, id string) (string, string, int, error) {
	r := struct {
		Token string `json:"token"`
		Url   string `json:"url"`
		Port  int    `json:"port"`
	}{}

	err := doRequest("GET", fmt.Sprintf("apps/%s/tunnels/%s", appID, id), nil, nil, &r)

	return r.Token, r.Url, r.Port, err
}

// EstablishConsole ...
// protocol ssh/docker
func EstablishConsole(appID, id string) (string, string, string, error) {
	r := map[string]string{}
	err := doRequest("GET", fmt.Sprintf("apps/%s/consoles/%s", appID, id), nil, nil, &r)

	return r["token"], r["url"], r["protocol"], err
}

// GetWarehouse ...
func GetWarehouse(appID string) (string, string, error) {
	r := map[string]string{}
	err := doRequest("GET", fmt.Sprintf("apps/%s/services/warehouse", appID), nil, nil, &r)

	return r["token"], r["url"], err
}

func GetPreviousBuild(appID string) (string, error) {
	r := []map[string]string{}
	err := doRequest("GET", fmt.Sprintf("apps/%s/deploys", appID), nil, nil, &r)
	if err != nil {
		return "", err
	}

	if len(r) > 0 {
		return r[0]["build_id"], nil
	}

	return "", nil
}

// doRequest ...
func doRequest(method, path string, params url.Values, requestBody, responseBody interface{}) error {

	var rbodyReader io.Reader

	//
	if requestBody != nil {
		jsonBytes, err := json.Marshal(requestBody)
		if err != nil {
			return err
		}
		rbodyReader = bytes.NewBuffer(jsonBytes)
	}

	auth, _ := models.LoadAuthByEndpoint(endpoint)

	// if they have not logged in but the user name and password are both set
	// use attempt to authenticate
	if auth.Key == "" &&
		os.Getenv("NANOBOX_PASSWORD") != "" &&
		os.Getenv("NANOBOX_USERNAME") != "" &&
		path != fmt.Sprintf("users/%s/auth_token", os.Getenv("NANOBOX_USERNAME")) {

		auth.Key, _ = Auth(os.Getenv("NANOBOX_USERNAME"), os.Getenv("NANOBOX_PASSWORD"))
	}

	if params == nil {
		params = url.Values{}
	}
	params.Set("auth_token", auth.Key)

	// fetch the correct url from the endpoint
	url := odinURL()

	//
	lumber.Debug("%s%s?%s\n", url, path, params.Encode())
	req, err := http.NewRequest(method, fmt.Sprintf("%s%s?%s", url, path, params.Encode()), rbodyReader)
	if err != nil {
		return err
	}

	req.Header.Set("Content-Type", "application/json")
	lumber.Trace("REQ: %s %s %s", req.Method, req.URL, req.Proto)

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	lumber.Debug("RES: %d %s %s %s (%s)", res.StatusCode, req.Method, req.URL, req.Proto, res.Header.Get("Content-Length"))

	// print the body even if status is not 2XX
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}

	if res.StatusCode == 401 {
		return util.ErrorfQuiet("Unauthorized (%s)", b)
	}

	if res.StatusCode == 404 {
		return util.ErrorfQuiet("Not Found (%s)", b)
	}

	if res.StatusCode == 500 {
		return util.ErrorfQuiet("Internal Server Error (%s)", b)
	}

	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return util.ErrorfQuiet("bad exit response(%d %s %s %s (%s) %s)", res.StatusCode, req.Method, req.URL, req.Proto, res.Header.Get("Content-Length"), b)
	}

	if responseBody != nil {
		lumber.Debug("response body: '%s'\n", b)
		err = json.Unmarshal(b, responseBody)
		if err != nil {
			return err
		}
	}

	return nil
}

func odinURL() string {
	switch endpoint {
	case "bonesalt":
		return BONESALT
	case "dev":
		return DEV
	case "sim":
		return SIM
	default:
		return NANOBOX
	}
}
<commit_msg>allow production users to be set with a -u flag<commit_after>// Package odin ...
package odin

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"

	"github.com/jcelliott/lumber"

	"github.com/nanobox-io/nanobox/models"
	"github.com/nanobox-io/nanobox/util"
)

const (
	NANOBOX  = "https://api.nanobox.io/v1/"
	BONESALT = "https://api.bonesalt.com/v1/"
	DEV      = "http://api.nanobox.dev:8080/v1/"
	SIM      = "http://api.nanobox.sim/v1/"
)

var (
	// set the default endpoint to nanobox
	endpoint = "nanobox"
)

type (
	evar struct {
		ID    string `json:"id"`
		Key   string `json:"title"`
		Value string `json:"value"`
	}
)

// sets the odin endpoint
func SetEndpoint(stage string) {
	endpoint = stage
}

// Auth ...
func Auth(username, password string) (string, error) {

	//
	params := url.Values{}
	params.Set("password", password)

	//
	resBody := map[string]string{}

	//
	if err := doRequest("GET", fmt.Sprintf("users/%s/auth_token", username), params, nil, &resBody); err != nil {
		return "", err
	}

	return resBody["authentication_token"], nil
}

// App ...
func App(slug string) (models.App, error) {
	app := models.App{}

	return app, doRequest("GET", "apps/"+slug, nil, nil, &app)
}

// Deploy ...
func Deploy(appID, id, boxfile, message string) error {

	//
	body := map[string]map[string]string{
		"deploy": {
			"boxfile_content": boxfile,
			"build_id":        id,
			"commit_message":  message,
		},
	}

	return doRequest("POST", fmt.Sprintf("apps/%s/deploys", appID), nil, body, nil)
}

func ListEvars(appID string) ([]evar, error) {
	evars := []evar{}
	return evars, doRequest("GET", fmt.Sprintf("apps/%s/evars", appID), nil, nil, &evars)
}

func AddEvar(appID, key, val string) error {
	body := map[string]map[string]string{
		"evar": {
			"title": key,
			"value": val,
		},
	}

	return doRequest("POST", fmt.Sprintf("apps/%s/evars", appID), nil, body, nil)
}

func RemoveEvar(appId, id string) error {
	return doRequest("DELETE", fmt.Sprintf("apps/%s/evars/%s", appId, id), nil, nil, nil)
}

// EstablishTunnel ...
func EstablishTunnel(appID, id string) (string, string, int, error) {
	r := struct {
		Token string `json:"token"`
		Url   string `json:"url"`
		Port  int    `json:"port"`
	}{}

	err := doRequest("GET", fmt.Sprintf("apps/%s/tunnels/%s", appID, id), nil, nil, &r)

	return r.Token, r.Url, r.Port, err
}

// EstablishConsole ...
// protocol ssh/docker
func EstablishConsole(appID, id string) (string, string, string, error) {
	// use a default user
	params = url.Values{}
		params.Set("user", "gonano")
	if registry.GetString("console_user") != "" {
		params.Set("user", registry.GetString("console_user"))
	}

	r := map[string]string{}
	err := doRequest("GET", fmt.Sprintf("apps/%s/consoles/%s", appID, id), params, nil, &r)

	return r["token"], r["url"], r["protocol"], err
}

// GetWarehouse ...
func GetWarehouse(appID string) (string, string, error) {
	r := map[string]string{}
	err := doRequest("GET", fmt.Sprintf("apps/%s/services/warehouse", appID), nil, nil, &r)

	return r["token"], r["url"], err
}

func GetPreviousBuild(appID string) (string, error) {
	r := []map[string]string{}
	err := doRequest("GET", fmt.Sprintf("apps/%s/deploys", appID), nil, nil, &r)
	if err != nil {
		return "", err
	}

	if len(r) > 0 {
		return r[0]["build_id"], nil
	}

	return "", nil
}

// doRequest ...
func doRequest(method, path string, params url.Values, requestBody, responseBody interface{}) error {

	var rbodyReader io.Reader

	//
	if requestBody != nil {
		jsonBytes, err := json.Marshal(requestBody)
		if err != nil {
			return err
		}
		rbodyReader = bytes.NewBuffer(jsonBytes)
	}

	auth, _ := models.LoadAuthByEndpoint(endpoint)

	// if they have not logged in but the user name and password are both set
	// use attempt to authenticate
	if auth.Key == "" &&
		os.Getenv("NANOBOX_PASSWORD") != "" &&
		os.Getenv("NANOBOX_USERNAME") != "" &&
		path != fmt.Sprintf("users/%s/auth_token", os.Getenv("NANOBOX_USERNAME")) {

		auth.Key, _ = Auth(os.Getenv("NANOBOX_USERNAME"), os.Getenv("NANOBOX_PASSWORD"))
	}

	if params == nil {
		params = url.Values{}
	}
	params.Set("auth_token", auth.Key)

	// fetch the correct url from the endpoint
	url := odinURL()

	//
	lumber.Debug("%s%s?%s\n", url, path, params.Encode())
	req, err := http.NewRequest(method, fmt.Sprintf("%s%s?%s", url, path, params.Encode()), rbodyReader)
	if err != nil {
		return err
	}

	req.Header.Set("Content-Type", "application/json")
	lumber.Trace("REQ: %s %s %s", req.Method, req.URL, req.Proto)

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	lumber.Debug("RES: %d %s %s %s (%s)", res.StatusCode, req.Method, req.URL, req.Proto, res.Header.Get("Content-Length"))

	// print the body even if status is not 2XX
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}

	if res.StatusCode == 401 {
		return util.ErrorfQuiet("Unauthorized (%s)", b)
	}

	if res.StatusCode == 404 {
		return util.ErrorfQuiet("Not Found (%s)", b)
	}

	if res.StatusCode == 500 {
		return util.ErrorfQuiet("Internal Server Error (%s)", b)
	}

	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return util.ErrorfQuiet("bad exit response(%d %s %s %s (%s) %s)", res.StatusCode, req.Method, req.URL, req.Proto, res.Header.Get("Content-Length"), b)
	}

	if responseBody != nil {
		lumber.Debug("response body: '%s'\n", b)
		err = json.Unmarshal(b, responseBody)
		if err != nil {
			return err
		}
	}

	return nil
}

func odinURL() string {
	switch endpoint {
	case "bonesalt":
		return BONESALT
	case "dev":
		return DEV
	case "sim":
		return SIM
	default:
		return NANOBOX
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aws

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/client"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/coreos/pkg/capnslog"

	"github.com/coreos/mantle/platform"
)

var plog = capnslog.NewPackageLogger("github.com/coreos/mantle", "platform/api/aws")

type Options struct {
	*platform.Options
	// The AWS region regional api calls should use
	Region string
	// The profile to use when resolving credentials, if applicable
	Profile string

	// AccessKeyID is the optional access key to use. It will override all other sources
	AccessKeyID string
	// SecretKey is the optional secret key to use. It will override all other sources
	SecretKey string

	AMI           string
	InstanceType  string
	SecurityGroup string
}

type API struct {
	session client.ConfigProvider
	ec2     *ec2.EC2
	s3      *s3.S3
	opts    *Options
}

// New creates a new AWS API wrapper. It uses credentials from any of the
// standard credentials sources, including the environment and the profile
// configured in ~/.aws.
// No validation is done that credentials exist and before using the API a
// preflight check is recommended via api.PreflightCheck
func New(opts *Options) (*API, error) {
	sess, err := session.NewSessionWithOptions(session.Options{
		Profile: opts.Profile,
		Config:  aws.Config{Region: aws.String(opts.Region)},
	})
	if err != nil {
		return nil, err
	}

	if opts.AccessKeyID != "" {
		sess.Config.WithCredentials(credentials.NewStaticCredentials(opts.AccessKeyID, opts.SecretKey, ""))
	}

	api := &API{
		session: sess,
		ec2:     ec2.New(sess),
		s3:      s3.New(sess),
		opts:    opts,
	}

	return api, nil
}

// PreflightCheck validates that the aws configuration provided has valid
// credentials
func (a *API) PreflightCheck() error {
	iamClient := iam.New(a.session)
	_, err := iamClient.GetUser(&iam.GetUserInput{})

	return err
}
<commit_msg>platform/aws: fix pre-flight check<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aws

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/client"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/sts"
	"github.com/coreos/pkg/capnslog"

	"github.com/coreos/mantle/platform"
)

var plog = capnslog.NewPackageLogger("github.com/coreos/mantle", "platform/api/aws")

type Options struct {
	*platform.Options
	// The AWS region regional api calls should use
	Region string
	// The profile to use when resolving credentials, if applicable
	Profile string

	// AccessKeyID is the optional access key to use. It will override all other sources
	AccessKeyID string
	// SecretKey is the optional secret key to use. It will override all other sources
	SecretKey string

	AMI           string
	InstanceType  string
	SecurityGroup string
}

type API struct {
	session client.ConfigProvider
	ec2     *ec2.EC2
	s3      *s3.S3
	opts    *Options
}

// New creates a new AWS API wrapper. It uses credentials from any of the
// standard credentials sources, including the environment and the profile
// configured in ~/.aws.
// No validation is done that credentials exist and before using the API a
// preflight check is recommended via api.PreflightCheck
func New(opts *Options) (*API, error) {
	sess, err := session.NewSessionWithOptions(session.Options{
		Profile: opts.Profile,
		Config:  aws.Config{Region: aws.String(opts.Region)},
	})
	if err != nil {
		return nil, err
	}

	if opts.AccessKeyID != "" {
		sess.Config.WithCredentials(credentials.NewStaticCredentials(opts.AccessKeyID, opts.SecretKey, ""))
	}

	api := &API{
		session: sess,
		ec2:     ec2.New(sess),
		s3:      s3.New(sess),
		opts:    opts,
	}

	return api, nil
}

// PreflightCheck validates that the aws configuration provided has valid
// credentials
func (a *API) PreflightCheck() error {
	stsClient := sts.New(a.session)
	_, err := stsClient.GetCallerIdentity(&sts.GetCallerIdentityInput{})

	return err
}
<|endoftext|>


<commit_before>package handlers

// type FileInfo interface {
//         Name() string       // base name of the file
//         Size() int64        // length in bytes for regular files; system-dependent for others
//         Mode() FileMode     // file mode bits
//         ModTime() time.Time // modification time
//         IsDir() bool        // abbreviation for Mode().IsDir()
//         Sys() interface{}   // underlying data source (can return nil)
// }

import (
	"encoding/json"
	"net/http"
	"net/url"
	"os"
	"strings"
	"terrigenesis/fileserver/utils"
)

/*
DownloadFile Handles requests to download a file
*/
func DownloadFile(w http.ResponseWriter, r *http.Request, session utils.Session) {
	var fileName []string
	if fileName = r.URL.Query()["filename"]; fileName == nil {
		IllegalArgumentsError(w)
	}

	pathToFile := session.CWD + "/" + strings.Join(fileName, "")
	var entry os.FileInfo
	var err error
	if entry, err = os.Stat(pathToFile); err == nil {
		// entry exists
		if !entry.IsDir() {
			// is not directory
			http.ServeFile(w, r, pathToFile)
		}
		FileTypeError(w)
	}

	FileNotFoundError(w)
}

/*
UploadFile Handles requests to upload file
*/
func UploadFile(w http.ResponseWriter, r *http.Request, session utils.Session) {
	// TODO:

}

/*
RemoveFile Handles requests to remove file
*/
func RemoveFile(w http.ResponseWriter, r *http.Request, form url.Values, session utils.Session) {
	if form["filename"] == nil {
		IllegalArgumentsError(w)
		return
	}

	pathToFile := session.CWD + "/" + strings.Join(form["filename"], "")
	var entry os.FileInfo
	var err error
	if entry, err = os.Stat(pathToFile); err == nil {
		// entry exists
		if !entry.IsDir() {
			// is not directory
			if removeErr := os.Remove(pathToFile); removeErr != nil {
				GeneralError(w, 500, "Error removing file")
			} else {
				w.WriteHeader(200)
				m := utils.Response{Status: 200, Message: "Successfully removed file"}
				json.NewEncoder(w).Encode(m)
			}
		} else {
			// is directory
			GeneralError(w, 500, "File is a directory")
		}
	} else {
		FileNotFoundError(w) // other options not very possible
	}
}
<commit_msg>Implemented upload file request<commit_after>package handlers

// type FileInfo interface {
//         Name() string       // base name of the file
//         Size() int64        // length in bytes for regular files; system-dependent for others
//         Mode() FileMode     // file mode bits
//         ModTime() time.Time // modification time
//         IsDir() bool        // abbreviation for Mode().IsDir()
//         Sys() interface{}   // underlying data source (can return nil)
// }

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"terrigenesis/fileserver/utils"
)

/*
DownloadFile Handles requests to download a file
*/
func DownloadFile(w http.ResponseWriter, r *http.Request, session utils.Session) {
	var fileName []string
	if fileName = r.URL.Query()["filename"]; fileName == nil {
		IllegalArgumentsError(w)
	}

	pathToFile := session.CWD + "/" + strings.Join(fileName, "")
	var entry os.FileInfo
	var err error
	if entry, err = os.Stat(pathToFile); err == nil {
		// entry exists
		if !entry.IsDir() {
			// is not directory
			http.ServeFile(w, r, pathToFile)
		}
		FileTypeError(w)
	}

	FileNotFoundError(w)
}

/*
UploadFile Handles requests to upload file
*/
func UploadFile(w http.ResponseWriter, r *http.Request, session utils.Session) {
	file, handler, err := r.FormFile("file")
	defer file.Close()
	if err != nil {
		GeneralError(w, 500, "Error parsing file")
	} else {
		f, err := os.OpenFile(session.CWD+"/"+handler.Filename, os.O_WRONLY|os.O_CREATE, 0666)
		defer f.Close()
		if err != nil {
			GeneralError(w, 500, "Error writing file")
		} else {
			io.Copy(f, file)
			m := utils.Response{Status: 200, Message: "Successfully uploaded file: " + handler.Filename}
			json.NewEncoder(w).Encode(m)
		}
	}
}

/*
RemoveFile Handles requests to remove file
*/
func RemoveFile(w http.ResponseWriter, r *http.Request, form url.Values, session utils.Session) {
	if form["filename"] == nil {
		IllegalArgumentsError(w)
		return
	}

	pathToFile := session.CWD + "/" + strings.Join(form["filename"], "")
	var entry os.FileInfo
	var err error
	if entry, err = os.Stat(pathToFile); err == nil {
		// entry exists
		if !entry.IsDir() {
			// is not directory
			if removeErr := os.Remove(pathToFile); removeErr != nil {
				GeneralError(w, 500, "Error removing file")
			} else {
				w.WriteHeader(200)
				m := utils.Response{Status: 200, Message: "Successfully removed file"}
				json.NewEncoder(w).Encode(m)
			}
		} else {
			// is directory
			GeneralError(w, 500, "File is a directory")
		}
	} else {
		FileNotFoundError(w) // other options not very possible
	}
}
<|endoftext|>


<commit_before>package asciidocgo

import (
	"reflect"
	"testing"
	. "github.com/smartystreets/goconvey/convey"
)

var dm = new(Document).Monitor()
var dnm = new(Document)
var notMonitoredError = &NotMonitoredError{"test"}
var monitorFNames = [1]string{"ReadTime"}

func TestDocumentMonitor(t *testing.T) {
	Convey("A Document can be monitored", t, func() {
		Convey("By default, a Document is not monitored", func() {
			So(dnm.IsMonitored(), ShouldBeFalse)
		})
		Convey("A monitored Document is monitored", func() {
			So(dm.IsMonitored(), ShouldBeTrue)
		})
	})
	Convey("A non-monitored Document should return error when accessing times", t, func() {
		_, err := dnm.ReadTime()
		So(err, ShouldNotBeNil)
		So(err, ShouldHaveSameTypeAs, notMonitoredError)
		So(err.Error(), ShouldContainSubstring, "not monitored")
	})
	Convey("A monitored empty Document should return 0 when accessing times", t, func() {
		dtype := reflect.ValueOf(dm)
		for _, fname := range monitorFNames {
			dfunc := dtype.MethodByName(fname)
			ret := dfunc.Call([]reflect.Value{})
			time := ret[0].Int()
			err := ret[1].Interface()
			So(err, ShouldBeNil)
			So(time, ShouldBeZeroValue)
		}
	})
}
<commit_msg>Uses reflect for non-monitored Document time access tests.<commit_after>package asciidocgo

import (
	"reflect"
	"testing"
	. "github.com/smartystreets/goconvey/convey"
)

var dm = new(Document).Monitor()
var dnm = new(Document)
var notMonitoredError = &NotMonitoredError{"test"}
var monitorFNames = [1]string{"ReadTime"}

func TestDocumentMonitor(t *testing.T) {
	Convey("A Document can be monitored", t, func() {
		Convey("By default, a Document is not monitored", func() {
			So(dnm.IsMonitored(), ShouldBeFalse)
		})
		Convey("A monitored Document is monitored", func() {
			So(dm.IsMonitored(), ShouldBeTrue)
		})
	})
	Convey("A non-monitored Document should return error when accessing times", t, func() {
		dtype := reflect.ValueOf(dnm)
		for _, fname := range monitorFNames {
			dfunc := dtype.MethodByName(fname)
			ret := dfunc.Call([]reflect.Value{})
			err := ret[1].Interface().(error)
			So(err, ShouldNotBeNil)
			So(err, ShouldHaveSameTypeAs, notMonitoredError)
			So(err.Error(), ShouldContainSubstring, "not monitored")
		}
	})
	Convey("A monitored empty Document should return 0 when accessing times", t, func() {
		dtype := reflect.ValueOf(dm)
		for _, fname := range monitorFNames {
			dfunc := dtype.MethodByName(fname)
			ret := dfunc.Call([]reflect.Value{})
			time := ret[0].Int()
			err := ret[1].Interface()
			So(err, ShouldBeNil)
			So(time, ShouldBeZeroValue)
		}
	})
}
<|endoftext|>


<commit_before>package main

import (
	cache "code.google.com/p/vitess/go/cache"
	"fmt"
	zmq "github.com/alecthomas/gozmq"
	"hash"
	"hash/fnv"
	"os"
	"time"
	"unsafe"
)

type CacheValue struct {
	expires time.Time
}

const cache_value_size = uint64(unsafe.Sizeof(CacheValue{}))

func (self *CacheValue) Size() int {
	return int(cache_value_size)
}

func main() {
	cache := cache.NewLRUCache(cache_value_size * 1000)

	context, _ := zmq.NewContext()

	receiver, _ := context.NewSocket(zmq.SUB)
	receiver.SetSockOptString(zmq.SUBSCRIBE, "")
	receiver.Connect("tcp://master.eve-emdr.com:8050")
	receiver.Connect("tcp://secondary.eve-emdr.com:8050")
	//receiver.Connect("tcp://relay-us-central-1.eve-emdr.com:8050")

	sender, _ := context.NewSocket(zmq.PUB)
	sender.Bind("tcp://0.0.0.0:8050")

	cache_duration, err := time.ParseDuration("5m")
	if err != nil {
		println(err.Error())
		os.Exit(1)
	}

	println("Listening on port 8050...")

	for {
		msg, zmq_err := receiver.Recv(0)
		now := time.Now()

		if zmq_err != nil {
			println("RECV ERROR:", zmq_err.Error())
		}

		var h hash.Hash = fnv.New32()
		h.Write(msg)

		checksum := h.Sum([]byte{})
		cache_key := fmt.Sprintf("%x", checksum)

		cache_item, cache_hit := cache.Get(cache_key)
		item_expired := false
		if cache_item != nil {
			item_expired = now.After(cache_item.(*CacheValue).expires)
		}
		cache_item = &CacheValue{expires: now.Add(cache_duration)}
		// Insert (or reset) the cache entry to prevent future re-sends of this message.
		cache.Set(cache_key, cache_item)

		if !cache_hit {
			// A cache miss means that the incoming message is not a dupe.
			// Send the message to subscribers.
			sender.Send(msg, 0)
		} else {
			// Cache hit, dupe if not expired.
			if !item_expired {
				// Not expired - dupe.
			} else {
				// Expired - valid. Send to subscribers.
				sender.Send(msg, 0)
			}
		}

	}
}
<commit_msg>Potential fixes for the CPU issues we see on some relays. Simplified things a good deal as well.<commit_after>package main

import (
	cache "code.google.com/p/vitess/go/cache"
	"fmt"
	zmq "github.com/alecthomas/gozmq"
	"hash"
	"hash/fnv"
	"unsafe"
)

// The presence of the cache value is all we need, so keep this super simple.
type CacheValue struct {
	found bool
}

// Calculate the size (in bytes) of our struct.
const cache_value_size = uint64(unsafe.Sizeof(CacheValue{}))

// Determines the max cache size, in bytes.
const cache_size_limit = cache_value_size * 1000

// Satisfies the Value interface.
func (self *CacheValue) Size() int {
	return int(cache_value_size)
}

func main() {
	cache := cache.NewLRUCache(cache_size_limit)

	context, _ := zmq.NewContext()
	receiver, _ := context.NewSocket(zmq.SUB)
	receiver.SetSockOptString(zmq.SUBSCRIBE, "")
	receiver.Connect("tcp://master.eve-emdr.com:8050")
	receiver.Connect("tcp://secondary.eve-emdr.com:8050")
	sender, _ := context.NewSocket(zmq.PUB)
	sender.Bind("tcp://0.0.0.0:8050")

	println("Listening on port 8050...")

	for {
		msg, zmq_err := receiver.Recv(0)

		if zmq_err != nil {
			println("RECV ERROR:", zmq_err.Error())
		}

		var h hash.Hash = fnv.New32()
		h.Write(msg)

		checksum := h.Sum([]byte{})
		cache_key := fmt.Sprintf("%x", checksum)

		cache_item, cache_hit := cache.Get(cache_key)
		if cache_hit {
			// We've already seen this before, ignore it.
			continue
		}

		// At this point, we know we've encountered a message we haven't
		// seen in the recent past.
		cache_item = &CacheValue{found: true}
		// Insert the cache entry to prevent future re-sends of this message.
		cache.Set(cache_key, cache_item)

		// A cache miss means that the incoming message is not a dupe.
		// Send the message to subscribers.
		sender.Send(msg, 0)
	}
}
<|endoftext|>