<commit_before>package lfs

import (
	"sync"

	"github.com/git-lfs/git-lfs/api"
	"github.com/git-lfs/git-lfs/config"
	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/progress"
	"github.com/git-lfs/git-lfs/transfer"
	"github.com/rubyist/tracerx"
)

const (
	batchSize         = 100
	defaultMaxRetries = 1
)

type Transferable interface {
	Oid() string
	Size() int64
	Name() string
	Path() string
	Object() *api.ObjectResource
	SetObject(*api.ObjectResource)
}

type retryCounter struct {
	// MaxRetries is the maximum number of retries a single object can
	// attempt to make before it will be dropped.
	MaxRetries int `git:"lfs.transfer.maxretries"`

	// cmu guards count
	cmu sync.Mutex
	// count maps OIDs to number of retry attempts
	count map[string]int
}

// newRetryCounter instantiates a new *retryCounter. It parses the gitconfig
// value: `lfs.transfer.maxretries`, and falls back to defaultMaxRetries if none
// was provided.
//
// If it encountered an error in Unmarshaling the *config.Configuration, it will
// be returned, otherwise nil.
func newRetryCounter(cfg *config.Configuration) *retryCounter {
	rc := &retryCounter{
		MaxRetries: defaultMaxRetries,

		count: make(map[string]int),
	}

	if err := cfg.Unmarshal(rc); err != nil {
		tracerx.Printf("rc: error parsing config, falling back to default values...: %v", err)
		rc.MaxRetries = 1
	}

	if rc.MaxRetries < 1 {
		tracerx.Printf("rc: invalid retry count: %d, defaulting to %d", rc.MaxRetries, 1)
		rc.MaxRetries = 1
	}

	return rc
}

// Increment increments the number of retries for a given OID. It is safe to
// call across multiple goroutines.
func (r *retryCounter) Increment(oid string) {
	r.cmu.Lock()
	defer r.cmu.Unlock()

	r.count[oid]++
}

// CountFor returns the current number of retries for a given OID. It is safe to
// call across multiple goroutines.
func (r *retryCounter) CountFor(oid string) int {
	r.cmu.Lock()
	defer r.cmu.Unlock()

	return r.count[oid]
}

// CanRetry returns the current number of retries, and whether or not it exceeds
// the maximum number of retries (see: retryCounter.MaxRetries).
func (r *retryCounter) CanRetry(oid string) (int, bool) {
	count := r.CountFor(oid)
	return count, count < r.MaxRetries
}

// TransferQueue organises the wider process of uploading and downloading,
// including calling the API, passing the actual transfer request to transfer
// adapters, and dealing with progress, errors and retries.
type TransferQueue struct {
	direction         transfer.Direction
	adapter           transfer.TransferAdapter
	adapterInProgress bool
	adapterResultChan chan transfer.TransferResult
	adapterInitMutex  sync.Mutex
	dryRun            bool
	meter             *progress.ProgressMeter
	errors            []error
	transferables     map[string]Transferable
	batcher           *Batcher
	retriesc          chan Transferable // Channel for processing retries
	errorc            chan error        // Channel for processing errors
	watchers          []chan string
	trMutex           *sync.Mutex
	errorwait         sync.WaitGroup
	retrywait         sync.WaitGroup
	// wait is used to keep track of pending transfers. It is incremented
	// once per unique OID on Add(), and is decremented when that transfer
	// is marked as completed or failed, but not retried.
	wait          sync.WaitGroup
	oldApiWorkers int // Number of non-batch API workers to spawn (deprecated)
	manifest      *transfer.Manifest
	rc            *retryCounter
}

// newTransferQueue builds a TransferQueue, direction and underlying mechanism determined by adapter
func newTransferQueue(files int, size int64, dryRun bool, dir transfer.Direction) *TransferQueue {
	cfg := config.Config

	logPath, _ := cfg.Os.Get("GIT_LFS_PROGRESS")

	q := &TransferQueue{
		direction:     dir,
		dryRun:        dryRun,
		meter:         progress.NewProgressMeter(files, size, dryRun, logPath),
		retriesc:      make(chan Transferable, batchSize),
		errorc:        make(chan error),
		oldApiWorkers: config.Config.ConcurrentTransfers(),
		transferables: make(map[string]Transferable),
		trMutex:       &sync.Mutex{},
		manifest:      transfer.ConfigureManifest(transfer.NewManifest(), config.Config),
		rc:            newRetryCounter(cfg),
	}

	q.errorwait.Add(1)
	q.retrywait.Add(1)

	q.run()

	return q
}

// Add adds a Transferable to the transfer queue. It only increments the amount
// of waiting the TransferQueue has to do if the Transferable "t" is new.
func (q *TransferQueue) Add(t Transferable) {
	q.trMutex.Lock()
	if _, ok := q.transferables[t.Oid()]; !ok {
		q.wait.Add(1)
		q.transferables[t.Oid()] = t
		q.trMutex.Unlock()
	} else {
		tracerx.Printf("already transferring %q, skipping duplicate", t)
		q.trMutex.Unlock()
		return
	}

	if q.batcher != nil {
		q.batcher.Add(t)
		return
	}

}

func (q *TransferQueue) useAdapter(name string) {
	q.adapterInitMutex.Lock()
	defer q.adapterInitMutex.Unlock()

	if q.adapter != nil {
		if q.adapter.Name() == name {
			// re-use, this is the normal path
			return
		}
		// If the adapter we're using isn't the same as the one we've been
		// told to use now, must wait for the current one to finish then switch
		// This will probably never happen but is just in case server starts
		// changing adapter support in between batches
		q.finishAdapter()
	}
	q.adapter = q.manifest.NewAdapterOrDefault(name, q.direction)
}

func (q *TransferQueue) finishAdapter() {
	if q.adapterInProgress {
		q.adapter.End()
		q.adapterInProgress = false
		q.adapter = nil
	}
}

func (q *TransferQueue) addToAdapter(t Transferable) {
	tr := transfer.NewTransfer(t.Name(), t.Object(), t.Path())

	if q.dryRun {
		// Don't actually transfer
		res := transfer.TransferResult{tr, nil}
		q.handleTransferResult(res)
		return
	}
	err := q.ensureAdapterBegun()
	if err != nil {
		q.errorc <- err
		q.Skip(t.Size())
		q.wait.Done()
		return
	}
	q.adapter.Add(tr)
}

func (q *TransferQueue) Skip(size int64) {
	q.meter.Skip(size)
}

func (q *TransferQueue) transferKind() string {
	if q.direction == transfer.Download {
		return "download"
	} else {
		return "upload"
	}
}

func (q *TransferQueue) ensureAdapterBegun() error {
	q.adapterInitMutex.Lock()
	defer q.adapterInitMutex.Unlock()

	if q.adapterInProgress {
		return nil
	}

	adapterResultChan := make(chan transfer.TransferResult, 20)

	// Progress callback - receives byte updates
	cb := func(name string, total, read int64, current int) error {
		q.meter.TransferBytes(q.transferKind(), name, read, total, current)
		return nil
	}

	tracerx.Printf("tq: starting transfer adapter %q", q.adapter.Name())
	err := q.adapter.Begin(config.Config.ConcurrentTransfers(), cb, adapterResultChan)
	if err != nil {
		return err
	}
	q.adapterInProgress = true

	// Collector for completed transfers
	// q.wait.Done() in handleTransferResult is enough to know when this is complete for all transfers
	go func() {
		for res := range adapterResultChan {
			q.handleTransferResult(res)
		}
	}()

	return nil
}

// handleTransferResult is responsible for dealing with the result of a
// successful or failed transfer.
//
// If there was an error assosicated with the given transfer, "res.Error", and
// it is retriable (see: `q.canRetryObject`), it will be placed in the next
// batch and be retried. If that error is not retriable for any reason, the
// transfer will be marked as having failed, and the error will be reported.
//
// If the transfer was successful, the watchers of this transfer queue will be
// notified, and the transfer will be marked as having been completed.
func (q *TransferQueue) handleTransferResult(res transfer.TransferResult) {
	oid := res.Transfer.Object.Oid

	if res.Error != nil {
		if q.canRetryObject(oid, res.Error) {
			tracerx.Printf("tq: retrying object %s", oid)
			q.trMutex.Lock()
			t, ok := q.transferables[oid]
			q.trMutex.Unlock()
			if ok {
				q.retry(t)
			} else {
				q.errorc <- res.Error
			}
		} else {
			q.errorc <- res.Error
			q.wait.Done()
		}
	} else {
		for _, c := range q.watchers {
			c <- oid
		}

		q.meter.FinishTransfer(res.Transfer.Name)
		q.wait.Done()
	}
}

// Wait waits for the queue to finish processing all transfers. Once Wait is
// called, Add will no longer add transferables to the queue. Any failed
// transfers will be automatically retried once.
func (q *TransferQueue) Wait() {
	if q.batcher != nil {
		q.batcher.Exit()
	}

	q.wait.Wait()

	// Handle any retries
	close(q.retriesc)
	q.retrywait.Wait()

	q.finishAdapter()
	close(q.errorc)

	for _, watcher := range q.watchers {
		close(watcher)
	}

	q.meter.Finish()
	q.errorwait.Wait()
}

// Watch returns a channel where the queue will write the OID of each transfer
// as it completes. The channel will be closed when the queue finishes processing.
func (q *TransferQueue) Watch() chan string {
	c := make(chan string, batchSize)
	q.watchers = append(q.watchers, c)
	return c
}

// batchApiRoutine processes the queue of transfers using the batch endpoint,
// making only one POST call for all objects. The results are then handed
// off to the transfer workers.
func (q *TransferQueue) batchApiRoutine() {
	var startProgress sync.Once

	transferAdapterNames := q.manifest.GetAdapterNames(q.direction)

	for {
		batch := q.batcher.Next()
		if batch == nil {
			break
		}

		tracerx.Printf("tq: sending batch of size %d", len(batch))

		transfers := make([]*api.ObjectResource, 0, len(batch))
		for _, i := range batch {
			t := i.(Transferable)
			transfers = append(transfers, &api.ObjectResource{Oid: t.Oid(), Size: t.Size()})
		}

		if len(transfers) == 0 {
			continue
		}

		objs, adapterName, err := api.Batch(config.Config, transfers, q.transferKind(), transferAdapterNames)
		if err != nil {
			var errOnce sync.Once
			for _, o := range batch {
				t := o.(Transferable)

				if q.canRetryObject(t.Oid(), err) {
					q.retry(t)
				} else {
					errOnce.Do(func() { q.errorc <- err })
					q.wait.Done()
				}
			}

			continue
		}

		q.useAdapter(adapterName)
		startProgress.Do(q.meter.Start)

		for _, o := range objs {
			if o.Error != nil {
				q.errorc <- errors.Wrapf(o.Error, "[%v] %v", o.Oid, o.Error.Message)
				q.Skip(o.Size)
				q.wait.Done()
				continue
			}

			if _, ok := o.Rel(q.transferKind()); ok {
				// This object needs to be transferred
				q.trMutex.Lock()
				transfer, ok := q.transferables[o.Oid]
				q.trMutex.Unlock()

				if ok {
					transfer.SetObject(o)
					q.meter.Add(transfer.Name())
					q.addToAdapter(transfer)
				} else {
					q.Skip(transfer.Size())
					q.wait.Done()
				}
			} else {
				q.Skip(o.Size)
				q.wait.Done()
			}
		}
	}
}

// This goroutine collects errors returned from transfers
func (q *TransferQueue) errorCollector() {
	for err := range q.errorc {
		q.errors = append(q.errors, err)
	}
	q.errorwait.Done()
}

// retryCollector collects objects to retry, increments the number of times that
// they have been retried, and then enqueues them in the next batch, or legacy
// API channel. If the transfer queue is using a batcher, the batch will be
// flushed immediately.
//
// retryCollector runs in its own goroutine.
func (q *TransferQueue) retryCollector() {
	for t := range q.retriesc {
		q.rc.Increment(t.Oid())
		count := q.rc.CountFor(t.Oid())

		tracerx.Printf("tq: enqueue retry #%d for %q (size: %d)", count, t.Oid(), t.Size())

		// XXX(taylor): reuse some of the logic in
		// `*TransferQueue.Add(t)` here to circumvent banned duplicate
		// OIDs
		if q.batcher != nil {
			tracerx.Printf("tq: flushing batch in response to retry #%d for %q (size: %d)", count, t.Oid(), t.Size())

			q.batcher.Add(t)
			q.batcher.Flush()
		}
	}
	q.retrywait.Done()
}

// run starts the transfer queue, doing individual or batch transfers depending
// on the Config.BatchTransfer() value. run will transfer files sequentially or
// concurrently depending on the Config.ConcurrentTransfers() value.
func (q *TransferQueue) run() {
	go q.errorCollector()
	go q.retryCollector()

	tracerx.Printf("tq: running as batched queue, batch size of %d", batchSize)
	q.batcher = NewBatcher(batchSize)
	go q.batchApiRoutine()
}

func (q *TransferQueue) retry(t Transferable) {
	q.retriesc <- t
}

// canRetry returns whether or not the given error "err" is retriable.
func (q *TransferQueue) canRetry(err error) bool {
	return errors.IsRetriableError(err)
}

// canRetryObject returns whether the given error is retriable for the object
// given by "oid". If the an OID has met its retry limit, then it will not be
// able to be retried again. If so, canRetryObject returns whether or not that
// given error "err" is retriable.
func (q *TransferQueue) canRetryObject(oid string, err error) bool {
	if count, ok := q.rc.CanRetry(oid); !ok {
		tracerx.Printf("tq: refusing to retry %q, too many retries (%d)", oid, count)
		return false
	}

	return q.canRetry(err)
}

// Errors returns any errors encountered during transfer.
func (q *TransferQueue) Errors() []error {
	return q.errors
}
<commit_msg>lfs/tq: remove deprecated "oldApiWorkers"<commit_after>package lfs

import (
	"sync"

	"github.com/git-lfs/git-lfs/api"
	"github.com/git-lfs/git-lfs/config"
	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/progress"
	"github.com/git-lfs/git-lfs/transfer"
	"github.com/rubyist/tracerx"
)

const (
	batchSize         = 100
	defaultMaxRetries = 1
)

type Transferable interface {
	Oid() string
	Size() int64
	Name() string
	Path() string
	Object() *api.ObjectResource
	SetObject(*api.ObjectResource)
}

type retryCounter struct {
	// MaxRetries is the maximum number of retries a single object can
	// attempt to make before it will be dropped.
	MaxRetries int `git:"lfs.transfer.maxretries"`

	// cmu guards count
	cmu sync.Mutex
	// count maps OIDs to number of retry attempts
	count map[string]int
}

// newRetryCounter instantiates a new *retryCounter. It parses the gitconfig
// value: `lfs.transfer.maxretries`, and falls back to defaultMaxRetries if none
// was provided.
//
// If it encountered an error in Unmarshaling the *config.Configuration, it will
// be returned, otherwise nil.
func newRetryCounter(cfg *config.Configuration) *retryCounter {
	rc := &retryCounter{
		MaxRetries: defaultMaxRetries,

		count: make(map[string]int),
	}

	if err := cfg.Unmarshal(rc); err != nil {
		tracerx.Printf("rc: error parsing config, falling back to default values...: %v", err)
		rc.MaxRetries = 1
	}

	if rc.MaxRetries < 1 {
		tracerx.Printf("rc: invalid retry count: %d, defaulting to %d", rc.MaxRetries, 1)
		rc.MaxRetries = 1
	}

	return rc
}

// Increment increments the number of retries for a given OID. It is safe to
// call across multiple goroutines.
func (r *retryCounter) Increment(oid string) {
	r.cmu.Lock()
	defer r.cmu.Unlock()

	r.count[oid]++
}

// CountFor returns the current number of retries for a given OID. It is safe to
// call across multiple goroutines.
func (r *retryCounter) CountFor(oid string) int {
	r.cmu.Lock()
	defer r.cmu.Unlock()

	return r.count[oid]
}

// CanRetry returns the current number of retries, and whether or not it exceeds
// the maximum number of retries (see: retryCounter.MaxRetries).
func (r *retryCounter) CanRetry(oid string) (int, bool) {
	count := r.CountFor(oid)
	return count, count < r.MaxRetries
}

// TransferQueue organises the wider process of uploading and downloading,
// including calling the API, passing the actual transfer request to transfer
// adapters, and dealing with progress, errors and retries.
type TransferQueue struct {
	direction         transfer.Direction
	adapter           transfer.TransferAdapter
	adapterInProgress bool
	adapterResultChan chan transfer.TransferResult
	adapterInitMutex  sync.Mutex
	dryRun            bool
	meter             *progress.ProgressMeter
	errors            []error
	transferables     map[string]Transferable
	batcher           *Batcher
	retriesc          chan Transferable // Channel for processing retries
	errorc            chan error        // Channel for processing errors
	watchers          []chan string
	trMutex           *sync.Mutex
	errorwait         sync.WaitGroup
	retrywait         sync.WaitGroup
	// wait is used to keep track of pending transfers. It is incremented
	// once per unique OID on Add(), and is decremented when that transfer
	// is marked as completed or failed, but not retried.
	wait     sync.WaitGroup
	manifest *transfer.Manifest
	rc       *retryCounter
}

// newTransferQueue builds a TransferQueue, direction and underlying mechanism determined by adapter
func newTransferQueue(files int, size int64, dryRun bool, dir transfer.Direction) *TransferQueue {
	cfg := config.Config

	logPath, _ := cfg.Os.Get("GIT_LFS_PROGRESS")

	q := &TransferQueue{
		direction:     dir,
		dryRun:        dryRun,
		meter:         progress.NewProgressMeter(files, size, dryRun, logPath),
		retriesc:      make(chan Transferable, batchSize),
		errorc:        make(chan error),
		transferables: make(map[string]Transferable),
		trMutex:       &sync.Mutex{},
		manifest:      transfer.ConfigureManifest(transfer.NewManifest(), config.Config),
		rc:            newRetryCounter(cfg),
	}

	q.errorwait.Add(1)
	q.retrywait.Add(1)

	q.run()

	return q
}

// Add adds a Transferable to the transfer queue. It only increments the amount
// of waiting the TransferQueue has to do if the Transferable "t" is new.
func (q *TransferQueue) Add(t Transferable) {
	q.trMutex.Lock()
	if _, ok := q.transferables[t.Oid()]; !ok {
		q.wait.Add(1)
		q.transferables[t.Oid()] = t
		q.trMutex.Unlock()
	} else {
		tracerx.Printf("already transferring %q, skipping duplicate", t)
		q.trMutex.Unlock()
		return
	}

	if q.batcher != nil {
		q.batcher.Add(t)
		return
	}

}

func (q *TransferQueue) useAdapter(name string) {
	q.adapterInitMutex.Lock()
	defer q.adapterInitMutex.Unlock()

	if q.adapter != nil {
		if q.adapter.Name() == name {
			// re-use, this is the normal path
			return
		}
		// If the adapter we're using isn't the same as the one we've been
		// told to use now, must wait for the current one to finish then switch
		// This will probably never happen but is just in case server starts
		// changing adapter support in between batches
		q.finishAdapter()
	}
	q.adapter = q.manifest.NewAdapterOrDefault(name, q.direction)
}

func (q *TransferQueue) finishAdapter() {
	if q.adapterInProgress {
		q.adapter.End()
		q.adapterInProgress = false
		q.adapter = nil
	}
}

func (q *TransferQueue) addToAdapter(t Transferable) {
	tr := transfer.NewTransfer(t.Name(), t.Object(), t.Path())

	if q.dryRun {
		// Don't actually transfer
		res := transfer.TransferResult{tr, nil}
		q.handleTransferResult(res)
		return
	}
	err := q.ensureAdapterBegun()
	if err != nil {
		q.errorc <- err
		q.Skip(t.Size())
		q.wait.Done()
		return
	}
	q.adapter.Add(tr)
}

func (q *TransferQueue) Skip(size int64) {
	q.meter.Skip(size)
}

func (q *TransferQueue) transferKind() string {
	if q.direction == transfer.Download {
		return "download"
	} else {
		return "upload"
	}
}

func (q *TransferQueue) ensureAdapterBegun() error {
	q.adapterInitMutex.Lock()
	defer q.adapterInitMutex.Unlock()

	if q.adapterInProgress {
		return nil
	}

	adapterResultChan := make(chan transfer.TransferResult, 20)

	// Progress callback - receives byte updates
	cb := func(name string, total, read int64, current int) error {
		q.meter.TransferBytes(q.transferKind(), name, read, total, current)
		return nil
	}

	tracerx.Printf("tq: starting transfer adapter %q", q.adapter.Name())
	err := q.adapter.Begin(config.Config.ConcurrentTransfers(), cb, adapterResultChan)
	if err != nil {
		return err
	}
	q.adapterInProgress = true

	// Collector for completed transfers
	// q.wait.Done() in handleTransferResult is enough to know when this is complete for all transfers
	go func() {
		for res := range adapterResultChan {
			q.handleTransferResult(res)
		}
	}()

	return nil
}

// handleTransferResult is responsible for dealing with the result of a
// successful or failed transfer.
//
// If there was an error assosicated with the given transfer, "res.Error", and
// it is retriable (see: `q.canRetryObject`), it will be placed in the next
// batch and be retried. If that error is not retriable for any reason, the
// transfer will be marked as having failed, and the error will be reported.
//
// If the transfer was successful, the watchers of this transfer queue will be
// notified, and the transfer will be marked as having been completed.
func (q *TransferQueue) handleTransferResult(res transfer.TransferResult) {
	oid := res.Transfer.Object.Oid

	if res.Error != nil {
		if q.canRetryObject(oid, res.Error) {
			tracerx.Printf("tq: retrying object %s", oid)
			q.trMutex.Lock()
			t, ok := q.transferables[oid]
			q.trMutex.Unlock()
			if ok {
				q.retry(t)
			} else {
				q.errorc <- res.Error
			}
		} else {
			q.errorc <- res.Error
			q.wait.Done()
		}
	} else {
		for _, c := range q.watchers {
			c <- oid
		}

		q.meter.FinishTransfer(res.Transfer.Name)
		q.wait.Done()
	}
}

// Wait waits for the queue to finish processing all transfers. Once Wait is
// called, Add will no longer add transferables to the queue. Any failed
// transfers will be automatically retried once.
func (q *TransferQueue) Wait() {
	if q.batcher != nil {
		q.batcher.Exit()
	}

	q.wait.Wait()

	// Handle any retries
	close(q.retriesc)
	q.retrywait.Wait()

	q.finishAdapter()
	close(q.errorc)

	for _, watcher := range q.watchers {
		close(watcher)
	}

	q.meter.Finish()
	q.errorwait.Wait()
}

// Watch returns a channel where the queue will write the OID of each transfer
// as it completes. The channel will be closed when the queue finishes processing.
func (q *TransferQueue) Watch() chan string {
	c := make(chan string, batchSize)
	q.watchers = append(q.watchers, c)
	return c
}

// batchApiRoutine processes the queue of transfers using the batch endpoint,
// making only one POST call for all objects. The results are then handed
// off to the transfer workers.
func (q *TransferQueue) batchApiRoutine() {
	var startProgress sync.Once

	transferAdapterNames := q.manifest.GetAdapterNames(q.direction)

	for {
		batch := q.batcher.Next()
		if batch == nil {
			break
		}

		tracerx.Printf("tq: sending batch of size %d", len(batch))

		transfers := make([]*api.ObjectResource, 0, len(batch))
		for _, i := range batch {
			t := i.(Transferable)
			transfers = append(transfers, &api.ObjectResource{Oid: t.Oid(), Size: t.Size()})
		}

		if len(transfers) == 0 {
			continue
		}

		objs, adapterName, err := api.Batch(config.Config, transfers, q.transferKind(), transferAdapterNames)
		if err != nil {
			var errOnce sync.Once
			for _, o := range batch {
				t := o.(Transferable)

				if q.canRetryObject(t.Oid(), err) {
					q.retry(t)
				} else {
					errOnce.Do(func() { q.errorc <- err })
					q.wait.Done()
				}
			}

			continue
		}

		q.useAdapter(adapterName)
		startProgress.Do(q.meter.Start)

		for _, o := range objs {
			if o.Error != nil {
				q.errorc <- errors.Wrapf(o.Error, "[%v] %v", o.Oid, o.Error.Message)
				q.Skip(o.Size)
				q.wait.Done()
				continue
			}

			if _, ok := o.Rel(q.transferKind()); ok {
				// This object needs to be transferred
				q.trMutex.Lock()
				transfer, ok := q.transferables[o.Oid]
				q.trMutex.Unlock()

				if ok {
					transfer.SetObject(o)
					q.meter.Add(transfer.Name())
					q.addToAdapter(transfer)
				} else {
					q.Skip(transfer.Size())
					q.wait.Done()
				}
			} else {
				q.Skip(o.Size)
				q.wait.Done()
			}
		}
	}
}

// This goroutine collects errors returned from transfers
func (q *TransferQueue) errorCollector() {
	for err := range q.errorc {
		q.errors = append(q.errors, err)
	}
	q.errorwait.Done()
}

// retryCollector collects objects to retry, increments the number of times that
// they have been retried, and then enqueues them in the next batch, or legacy
// API channel. If the transfer queue is using a batcher, the batch will be
// flushed immediately.
//
// retryCollector runs in its own goroutine.
func (q *TransferQueue) retryCollector() {
	for t := range q.retriesc {
		q.rc.Increment(t.Oid())
		count := q.rc.CountFor(t.Oid())

		tracerx.Printf("tq: enqueue retry #%d for %q (size: %d)", count, t.Oid(), t.Size())

		// XXX(taylor): reuse some of the logic in
		// `*TransferQueue.Add(t)` here to circumvent banned duplicate
		// OIDs
		if q.batcher != nil {
			tracerx.Printf("tq: flushing batch in response to retry #%d for %q (size: %d)", count, t.Oid(), t.Size())

			q.batcher.Add(t)
			q.batcher.Flush()
		}
	}
	q.retrywait.Done()
}

// run starts the transfer queue, doing individual or batch transfers depending
// on the Config.BatchTransfer() value. run will transfer files sequentially or
// concurrently depending on the Config.ConcurrentTransfers() value.
func (q *TransferQueue) run() {
	go q.errorCollector()
	go q.retryCollector()

	tracerx.Printf("tq: running as batched queue, batch size of %d", batchSize)
	q.batcher = NewBatcher(batchSize)
	go q.batchApiRoutine()
}

func (q *TransferQueue) retry(t Transferable) {
	q.retriesc <- t
}

// canRetry returns whether or not the given error "err" is retriable.
func (q *TransferQueue) canRetry(err error) bool {
	return errors.IsRetriableError(err)
}

// canRetryObject returns whether the given error is retriable for the object
// given by "oid". If the an OID has met its retry limit, then it will not be
// able to be retried again. If so, canRetryObject returns whether or not that
// given error "err" is retriable.
func (q *TransferQueue) canRetryObject(oid string, err error) bool {
	if count, ok := q.rc.CanRetry(oid); !ok {
		tracerx.Printf("tq: refusing to retry %q, too many retries (%d)", oid, count)
		return false
	}

	return q.canRetry(err)
}

// Errors returns any errors encountered during transfer.
func (q *TransferQueue) Errors() []error {
	return q.errors
}
<|endoftext|>


<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package servicedefinition

import (
	"github.com/control-center/serviced/domain"
	"github.com/control-center/serviced/utils"

	"errors"
	"strings"
	"time"
)

//ServiceDefinition is the definition of a service hierarchy.
type ServiceDefinition struct {
	Name              string                 // Name of the defined service
	Title             string                 // Title is a label used when describing this service in the context of a service tree
	Version           string                 // Version of the defined service
	Command           string                 // Command which runs the service
	Description       string                 // Description of the service
	Tags              []string               // Searchable service tags
	ImageID           string                 // Docker image hosting the service
	Instances         domain.MinMax          // Constraints on the number of instances
	ChangeOptions     []string               // Control options for what happens when a running service is changed
	Launch            string                 // Must be "AUTO", the default, or "MANUAL"
	HostPolicy        HostPolicy             // Policy for starting up instances
	Hostname          string                 // Optional hostname which should be set on run
	Privileged        bool                   // Whether to run the container with extended privileges
	ConfigFiles       map[string]ConfigFile  // Config file templates
	Context           map[string]interface{} // Context information for the service
	Endpoints         []EndpointDefinition   // Comms endpoints used by the service
	Services          []ServiceDefinition    // Supporting subservices
	Tasks             []Task                 // Scheduled tasks for celery to find
	LogFilters        map[string]string      // map of log filter name to log filter definitions
	Volumes           []Volume               // list of volumes to bind into containers
	LogConfigs        []LogConfig
	Snapshot          SnapshotCommands              // Snapshot quiesce info for the service: Pause/Resume bash commands
	RAMCommitment     utils.EngNotation             // expected RAM commitment to use for scheduling
	CPUCommitment     uint64                        // expected CPU commitment (#cores) to use for scheduling
	Runs              map[string]string             // Map of commands that can be executed with 'serviced run ...'
	Actions           map[string]string             // Map of commands that can be executed with 'serviced action ...'
	HealthChecks      map[string]domain.HealthCheck // HealthChecks for a service.
	Prereqs           []domain.Prereq               // Optional list of scripts that must be successfully run before kicking off the service command.
	MonitoringProfile domain.MonitorProfile         // An optional list of queryable metrics, graphs, and thresholds
	MemoryLimit       float64
	CPUShares         int64
	PIDFile           string // An optional path or command to generate a path for a PID file to which signals are relayed.
}

// SnapshotCommands commands to be called during and after a snapshot
type SnapshotCommands struct {
	Pause  string // bash command to pause the volume  (quiesce)
	Resume string // bash command to resume the volume (unquiesce)
}

// EndpointDefinition An endpoint that a Service exposes.
type EndpointDefinition struct {
	Name                string // Human readable name of the endpoint. Unique per service definition
	Purpose             string
	Protocol            string
	PortNumber          uint16
	PortTemplate        string // A template which, if specified, is used to calculate the port number
	VirtualAddress      string // An address by which an imported endpoint may be accessed within the container, e.g. "mysqlhost:1234"
	Application         string
	ApplicationTemplate string
	AddressConfig       AddressResourceConfig
	VHosts              []string // VHost is used to request named vhost for this endpoint. Should be the name of a
	// subdomain, i.e "myapplication"  not "myapplication.host.com"
}

// Task A scheduled task
type Task struct {
	Name          string
	Schedule      string
	Command       string
	LastRunAt     time.Time
	TotalRunCount int
}

// Volume import defines a file system directory underneath an export directory
type Volume struct {
	Owner             string //Resource Path Owner
	Permission        string //Resource Path permissions, eg what you pass to chmod
	ResourcePath      string //Resource Pool Path, shared across all hosts in a resource pool
	ContainerPath     string //Container bind-mount path
	Type              string //Path use, i.e. "dfs" or "tmp"
	InitContainerPath string //Path to initialize the volume from at creation time, optional
}

// ConfigFile config file for a service
type ConfigFile struct {
	Filename    string // complete path of file
	Owner       string // owner of file within the container, root:root or 0:0 for root owned file, what you would pass to chown
	Permissions string // permission of file, eg 0664, what you would pass to chmod
	Content     string // content of config file
}

//AddressResourceConfig defines an external facing port for a service definition
type AddressResourceConfig struct {
	Port     uint16
	Protocol string
}

// LogConfig represents the configuration for a logfile for a service.
type LogConfig struct {
	Path    string   // The location on the container's filesystem of the log, can be a directory
	Type    string   // Arbitrary string that identifies the "types" of logs that come from this source. This will be
	Filters []string // A list of filters that must be contained in either the LogFilters or a parent's LogFilter,
	LogTags []LogTag // Key value pair of tags that are sent to logstash for all entries coming out of this logfile
}

// LogTag  no clue what this is. Maybe someone actually reads this
type LogTag struct {
	Name  string
	Value string
}

// HostPolicy represents the optional policy used to determine which hosts on
// which to run instances of a service. Default is to run on the available
// host with the most uncommitted RAM.
type HostPolicy string

const (
	//DEFAULT policy for scheduling a service instance
	DEFAULT HostPolicy = ""
	//LeastCommitted run on host w/ least committed memory
	LeastCommitted = "LEAST_COMMITTED"
	//PreferSeparate attempt to schedule instances of a service on separate hosts
	PreferSeparate = "PREFER_SEPARATE"
	//RequireSeparate schedule instances of a service on separate hosts
	RequireSeparate = "REQUIRE_SEPARATE"
)

// UnmarshalText implements the encoding/TextUnmarshaler interface
func (p *HostPolicy) UnmarshalText(b []byte) error {
	s := strings.Trim(string(b), `"`)
	switch s {
	case LeastCommitted, PreferSeparate, RequireSeparate:
		*p = HostPolicy(s)
	case "":
		*p = DEFAULT
	default:
		return errors.New("Invalid HostPolicy: " + s)
	}
	return nil
}

func (s ServiceDefinition) String() string {
	return s.Name
}

//BuildFromPath given a path will create a ServiceDefintion
func BuildFromPath(path string) (*ServiceDefinition, error) {
	sd, err := getServiceDefinition(path)
	if err != nil {
		return nil, err
	}
	return sd, sd.ValidEntity()
}
<commit_msg>add vhost type<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package servicedefinition

import (
	"github.com/control-center/serviced/domain"
	"github.com/control-center/serviced/utils"

	"errors"
	"strings"
	"time"
)

//ServiceDefinition is the definition of a service hierarchy.
type ServiceDefinition struct {
	Name              string                 // Name of the defined service
	Title             string                 // Title is a label used when describing this service in the context of a service tree
	Version           string                 // Version of the defined service
	Command           string                 // Command which runs the service
	Description       string                 // Description of the service
	Tags              []string               // Searchable service tags
	ImageID           string                 // Docker image hosting the service
	Instances         domain.MinMax          // Constraints on the number of instances
	ChangeOptions     []string               // Control options for what happens when a running service is changed
	Launch            string                 // Must be "AUTO", the default, or "MANUAL"
	HostPolicy        HostPolicy             // Policy for starting up instances
	Hostname          string                 // Optional hostname which should be set on run
	Privileged        bool                   // Whether to run the container with extended privileges
	ConfigFiles       map[string]ConfigFile  // Config file templates
	Context           map[string]interface{} // Context information for the service
	Endpoints         []EndpointDefinition   // Comms endpoints used by the service
	Services          []ServiceDefinition    // Supporting subservices
	Tasks             []Task                 // Scheduled tasks for celery to find
	LogFilters        map[string]string      // map of log filter name to log filter definitions
	Volumes           []Volume               // list of volumes to bind into containers
	LogConfigs        []LogConfig
	Snapshot          SnapshotCommands              // Snapshot quiesce info for the service: Pause/Resume bash commands
	RAMCommitment     utils.EngNotation             // expected RAM commitment to use for scheduling
	CPUCommitment     uint64                        // expected CPU commitment (#cores) to use for scheduling
	Runs              map[string]string             // Map of commands that can be executed with 'serviced run ...'
	Actions           map[string]string             // Map of commands that can be executed with 'serviced action ...'
	HealthChecks      map[string]domain.HealthCheck // HealthChecks for a service.
	Prereqs           []domain.Prereq               // Optional list of scripts that must be successfully run before kicking off the service command.
	MonitoringProfile domain.MonitorProfile         // An optional list of queryable metrics, graphs, and thresholds
	MemoryLimit       float64
	CPUShares         int64
	PIDFile           string // An optional path or command to generate a path for a PID file to which signals are relayed.
}

// SnapshotCommands commands to be called during and after a snapshot
type SnapshotCommands struct {
	Pause  string // bash command to pause the volume  (quiesce)
	Resume string // bash command to resume the volume (unquiesce)
}

// EndpointDefinition An endpoint that a Service exposes.
type EndpointDefinition struct {
	Name                string   // Human readable name of the endpoint. Unique per service definition
	Purpose             string
	Protocol            string
	PortNumber          uint16
	PortTemplate        string   // A template which, if specified, is used to calculate the port number
	VirtualAddress      string   // An address by which an imported endpoint may be accessed within the container, e.g. "mysqlhost:1234"
	Application         string
	ApplicationTemplate string
	AddressConfig       AddressResourceConfig
//	VHosts              []string // VHost is used to request named vhost for this endpoint. Should be the name of a
								 // subdomain, i.e "myapplication"  not "myapplication.host.com"
	VHostList           []VHost  // VHost is used to request named vhost(s) for this endpoint.
}

// VHost is the configuration for an application endpoint that wants an http VHost endpoint provided by Control Center
type VHost struct{
	Name string // name of the vhost subdomain subdomain, i.e "myapplication"  not "myapplication.host.com
	enabled bool // whether the vhost should be enabled or disabled.
}

// Task A scheduled task
type Task struct {
	Name          string
	Schedule      string
	Command       string
	LastRunAt     time.Time
	TotalRunCount int
}

// Volume import defines a file system directory underneath an export directory
type Volume struct {
	Owner             string //Resource Path Owner
	Permission        string //Resource Path permissions, eg what you pass to chmod
	ResourcePath      string //Resource Pool Path, shared across all hosts in a resource pool
	ContainerPath     string //Container bind-mount path
	Type              string //Path use, i.e. "dfs" or "tmp"
	InitContainerPath string //Path to initialize the volume from at creation time, optional
}

// ConfigFile config file for a service
type ConfigFile struct {
	Filename    string // complete path of file
	Owner       string // owner of file within the container, root:root or 0:0 for root owned file, what you would pass to chown
	Permissions string // permission of file, eg 0664, what you would pass to chmod
	Content     string // content of config file
}

//AddressResourceConfig defines an external facing port for a service definition
type AddressResourceConfig struct {
	Port     uint16
	Protocol string
}

// LogConfig represents the configuration for a logfile for a service.
type LogConfig struct {
	Path    string   // The location on the container's filesystem of the log, can be a directory
	Type    string   // Arbitrary string that identifies the "types" of logs that come from this source. This will be
	Filters []string // A list of filters that must be contained in either the LogFilters or a parent's LogFilter,
	LogTags []LogTag // Key value pair of tags that are sent to logstash for all entries coming out of this logfile
}

// LogTag  no clue what this is. Maybe someone actually reads this
type LogTag struct {
	Name  string
	Value string
}

// HostPolicy represents the optional policy used to determine which hosts on
// which to run instances of a service. Default is to run on the available
// host with the most uncommitted RAM.
type HostPolicy string

const (
	//DEFAULT policy for scheduling a service instance
	DEFAULT HostPolicy = ""
	//LeastCommitted run on host w/ least committed memory
	LeastCommitted = "LEAST_COMMITTED"
	//PreferSeparate attempt to schedule instances of a service on separate hosts
	PreferSeparate = "PREFER_SEPARATE"
	//RequireSeparate schedule instances of a service on separate hosts
	RequireSeparate = "REQUIRE_SEPARATE"
)

// UnmarshalText implements the encoding/TextUnmarshaler interface
func (p *HostPolicy) UnmarshalText(b []byte) error {
	s := strings.Trim(string(b), `"`)
	switch s {
	case LeastCommitted, PreferSeparate, RequireSeparate:
		*p = HostPolicy(s)
	case "":
		*p = DEFAULT
	default:
		return errors.New("Invalid HostPolicy: " + s)
	}
	return nil
}

func (s ServiceDefinition) String() string {
	return s.Name
}

//BuildFromPath given a path will create a ServiceDefintion
func BuildFromPath(path string) (*ServiceDefinition, error) {
	sd, err := getServiceDefinition(path)
	if err != nil {
		return nil, err
	}
	return sd, sd.ValidEntity()
}
<|endoftext|>


<commit_before>package rbac

import (
	"context"
	"strings"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	corev1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	kuser "k8s.io/apiserver/pkg/authentication/user"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/tools/cache"
	kauthenticationapi "k8s.io/kubernetes/pkg/apis/authentication"
	kauthorizationapi "k8s.io/kubernetes/pkg/apis/authorization"
	"k8s.io/kubernetes/pkg/apis/rbac"
	rbacv1helpers "k8s.io/kubernetes/pkg/apis/rbac/v1"
	"k8s.io/kubernetes/pkg/apis/storage"
	"k8s.io/kubernetes/pkg/registry/rbac/validation"
	e2e "k8s.io/kubernetes/test/e2e/framework"

	"github.com/openshift/api/authorization"
	"github.com/openshift/api/build"
	"github.com/openshift/api/console"
	"github.com/openshift/api/image"
	"github.com/openshift/api/oauth"
	"github.com/openshift/api/project"
	"github.com/openshift/api/template"
	"github.com/openshift/api/user"

	exutil "github.com/openshift/origin/test/extended/util"
)

// copied from bootstrap policy
var read = []string{"get", "list", "watch"}

// copied from bootstrap policy
const (
	rbacGroup    = rbac.GroupName
	storageGroup = storage.GroupName
	kAuthzGroup  = kauthorizationapi.GroupName
	kAuthnGroup  = kauthenticationapi.GroupName

	authzGroup    = authorization.GroupName
	buildGroup    = build.GroupName
	imageGroup    = image.GroupName
	oauthGroup    = oauth.GroupName
	projectGroup  = project.GroupName
	templateGroup = template.GroupName
	userGroup     = user.GroupName
	consoleGroup  = console.GroupName

	legacyGroup         = ""
	legacyAuthzGroup    = ""
	legacyBuildGroup    = ""
	legacyImageGroup    = ""
	legacyProjectGroup  = ""
	legacyTemplateGroup = ""
	legacyUserGroup     = ""
	legacyOauthGroup    = ""
)

// Do not change any of these lists without approval from the auth and master teams
// Most rules are copied from various cluster roles in bootstrap policy
var (
	allUnauthenticatedRules = []rbacv1.PolicyRule{
		rbacv1helpers.NewRule("get", "create").Groups(buildGroup, legacyBuildGroup).Resources("buildconfigs/webhooks").RuleOrDie(),

		rbacv1helpers.NewRule("impersonate").Groups(kAuthnGroup).Resources("userextras/scopes.authorization.openshift.io").RuleOrDie(),

		rbacv1helpers.NewRule("create").Groups(authzGroup, legacyAuthzGroup).Resources("selfsubjectrulesreviews").RuleOrDie(),

		rbacv1helpers.NewRule("create").Groups(kAuthzGroup).Resources("selfsubjectaccessreviews", "selfsubjectrulesreviews").RuleOrDie(),

		rbacv1helpers.NewRule("delete").Groups(oauthGroup, legacyOauthGroup).Resources("oauthaccesstokens", "oauthauthorizetokens").RuleOrDie(),

		// this is openshift specific
		rbacv1helpers.NewRule("get").URLs(
			"/version/openshift",
			"/.well-known",
			"/.well-known/*",
			"/.well-known/oauth-authorization-server",
		).RuleOrDie(),

		// TODO: remove with after 1.15 rebase
		rbacv1helpers.NewRule("get").URLs(
			"/readyz",
		).RuleOrDie(),

		// this is from upstream kube
		rbacv1helpers.NewRule("get").URLs(
			"/healthz", "/livez",
			"/version",
			"/version/",
		).RuleOrDie(),
	}

	allAuthenticatedRules = append(
		[]rbacv1.PolicyRule{
			rbacv1helpers.NewRule("create").Groups(buildGroup, legacyBuildGroup).Resources("builds/docker", "builds/optimizeddocker").RuleOrDie(),
			rbacv1helpers.NewRule("create").Groups(buildGroup, legacyBuildGroup).Resources("builds/jenkinspipeline").RuleOrDie(),
			rbacv1helpers.NewRule("create").Groups(buildGroup, legacyBuildGroup).Resources("builds/source").RuleOrDie(),

			rbacv1helpers.NewRule("get").Groups(userGroup, legacyUserGroup).Resources("users").Names("~").RuleOrDie(),
			rbacv1helpers.NewRule("list").Groups(projectGroup, legacyProjectGroup).Resources("projectrequests").RuleOrDie(),
			rbacv1helpers.NewRule("get", "list").Groups(authzGroup, legacyAuthzGroup).Resources("clusterroles").RuleOrDie(),
			rbacv1helpers.NewRule(read...).Groups(rbacGroup).Resources("clusterroles").RuleOrDie(),
			rbacv1helpers.NewRule("get", "list").Groups(storageGroup).Resources("storageclasses").RuleOrDie(),
			rbacv1helpers.NewRule("list", "watch").Groups(projectGroup, legacyProjectGroup).Resources("projects").RuleOrDie(),

			// These custom resources are used to extend console functionality
			// The console team is working on eliminating this exception in the near future
			rbacv1helpers.NewRule(read...).Groups(consoleGroup).Resources("consoleclidownloads", "consolelinks", "consoleexternalloglinks", "consolenotifications").RuleOrDie(),

			// TODO: remove when openshift-apiserver has removed these
			rbacv1helpers.NewRule("get").URLs(
				"/healthz/",
				"/oapi", "/oapi/*",
				"/osapi", "/osapi/",
				"/swaggerapi", "/swaggerapi/*", "/swagger.json", "/swagger-2.0.0.pb-v1",
				"/version/*",
				"/",
			).RuleOrDie(),

			// this is from upstream kube
			rbacv1helpers.NewRule("get").URLs(
				"/",
				"/openapi", "/openapi/*",
				"/api", "/api/*",
				"/apis", "/apis/*",
			).RuleOrDie(),
		},
		allUnauthenticatedRules...,
	)

	// group -> namespace -> rules
	groupNamespaceRules = map[string]map[string][]rbacv1.PolicyRule{
		kuser.AllAuthenticated: {
			"openshift": {
				rbacv1helpers.NewRule(read...).Groups(templateGroup, legacyTemplateGroup).Resources("templates").RuleOrDie(),
				rbacv1helpers.NewRule(read...).Groups(imageGroup, legacyImageGroup).Resources("imagestreams", "imagestreamtags", "imagestreamimages").RuleOrDie(),
				rbacv1helpers.NewRule("get").Groups(imageGroup, legacyImageGroup).Resources("imagestreams/layers").RuleOrDie(),
			},
			"openshift-config-managed": {
				rbacv1helpers.NewRule("get").Groups(legacyGroup).Resources("configmaps").Names("console-public").RuleOrDie(),
			},
		},
		kuser.AllUnauthenticated:     {}, // no rules expect the cluster wide ones
		"system:authenticated:oauth": {}, // no rules expect the cluster wide ones
	}
)

var _ = g.Describe("[Feature:OpenShiftAuthorization] The default cluster RBAC policy", func() {
	defer g.GinkgoRecover()

	oc := exutil.NewCLI("default-rbac-policy", exutil.KubeConfigPath())

	g.It("should have correct RBAC rules", func() {
		kubeInformers := informers.NewSharedInformerFactory(oc.AdminKubeClient(), 20*time.Minute)
		ruleResolver := exutil.NewRuleResolver(kubeInformers.Rbac().V1()) // signal what informers we want to use early

		stopCh := make(chan struct{})
		defer func() { close(stopCh) }()
		kubeInformers.Start(stopCh)

		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()

		if ok := cache.WaitForCacheSync(ctx.Done(),
			kubeInformers.Rbac().V1().ClusterRoles().Informer().HasSynced,
			kubeInformers.Rbac().V1().ClusterRoleBindings().Informer().HasSynced,
			kubeInformers.Rbac().V1().Roles().Informer().HasSynced,
			kubeInformers.Rbac().V1().RoleBindings().Informer().HasSynced,
		); !ok {
			exutil.FatalErr("failed to sync RBAC cache")
		}

		namespaces, err := oc.AdminKubeClient().CoreV1().Namespaces().List(metav1.ListOptions{})
		if err != nil {
			exutil.FatalErr(err)
		}

		g.By("should only allow the system:authenticated group to access certain policy rules", func() {
			testAllGroupRules(ruleResolver, kuser.AllAuthenticated, allAuthenticatedRules, namespaces.Items)
		})

		g.By("should only allow the system:unauthenticated group to access certain policy rules", func() {
			testAllGroupRules(ruleResolver, kuser.AllUnauthenticated, allUnauthenticatedRules, namespaces.Items)
		})

		g.By("should only allow the system:authenticated:oauth group to access certain policy rules", func() {
			testAllGroupRules(ruleResolver, "system:authenticated:oauth", []rbacv1.PolicyRule{
				rbacv1helpers.NewRule("create").Groups(projectGroup, legacyProjectGroup).Resources("projectrequests").RuleOrDie(),
			}, namespaces.Items)
		})

	})
})

func testAllGroupRules(ruleResolver validation.AuthorizationRuleResolver, group string, expectedClusterRules []rbacv1.PolicyRule, namespaces []corev1.Namespace) {
	testGroupRules(ruleResolver, group, metav1.NamespaceNone, expectedClusterRules)

	for _, namespace := range namespaces {
		// merge the namespace scoped and cluster wide rules
		rules := append([]rbacv1.PolicyRule{}, groupNamespaceRules[group][namespace.Name]...)
		rules = append(rules, expectedClusterRules...)

		testGroupRules(ruleResolver, group, namespace.Name, rules)
	}
}

func testGroupRules(ruleResolver validation.AuthorizationRuleResolver, group, namespace string, expectedRules []rbacv1.PolicyRule) {
	actualRules, err := ruleResolver.RulesFor(&kuser.DefaultInfo{Groups: []string{group}}, namespace)
	o.Expect(err).NotTo(o.HaveOccurred()) // our default RBAC policy should never have rule resolution errors

	if cover, missing := validation.Covers(expectedRules, actualRules); !cover {
		e2e.Failf("%s has extra permissions in namespace %q:\n%s", group, namespace, rulesToString(missing))
	}

	// force test data to be cleaned up every so often but allow extra rules to not deadlock new changes
	if cover, missing := validation.Covers(actualRules, expectedRules); !cover {
		log := e2e.Logf
		if len(missing) > 15 {
			log = e2e.Failf
		}
		log("test data for %s has too many unnecessary permissions:\n%s", group, rulesToString(missing))
	}
}

func rulesToString(rules []rbacv1.PolicyRule) string {
	compactRules := rules
	if compact, err := validation.CompactRules(rules); err == nil {
		compactRules = compact
	}

	missingDescriptions := sets.NewString()
	for _, missing := range compactRules {
		missingDescriptions.Insert(rbacv1helpers.CompactString(missing))
	}

	return strings.Join(missingDescriptions.List(), "\n")
}
<commit_msg>Add consoleyamlsamples to list of console resource exceptions<commit_after>package rbac

import (
	"context"
	"strings"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	corev1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	kuser "k8s.io/apiserver/pkg/authentication/user"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/tools/cache"
	kauthenticationapi "k8s.io/kubernetes/pkg/apis/authentication"
	kauthorizationapi "k8s.io/kubernetes/pkg/apis/authorization"
	"k8s.io/kubernetes/pkg/apis/rbac"
	rbacv1helpers "k8s.io/kubernetes/pkg/apis/rbac/v1"
	"k8s.io/kubernetes/pkg/apis/storage"
	"k8s.io/kubernetes/pkg/registry/rbac/validation"
	e2e "k8s.io/kubernetes/test/e2e/framework"

	"github.com/openshift/api/authorization"
	"github.com/openshift/api/build"
	"github.com/openshift/api/console"
	"github.com/openshift/api/image"
	"github.com/openshift/api/oauth"
	"github.com/openshift/api/project"
	"github.com/openshift/api/template"
	"github.com/openshift/api/user"

	exutil "github.com/openshift/origin/test/extended/util"
)

// copied from bootstrap policy
var read = []string{"get", "list", "watch"}

// copied from bootstrap policy
const (
	rbacGroup    = rbac.GroupName
	storageGroup = storage.GroupName
	kAuthzGroup  = kauthorizationapi.GroupName
	kAuthnGroup  = kauthenticationapi.GroupName

	authzGroup    = authorization.GroupName
	buildGroup    = build.GroupName
	imageGroup    = image.GroupName
	oauthGroup    = oauth.GroupName
	projectGroup  = project.GroupName
	templateGroup = template.GroupName
	userGroup     = user.GroupName
	consoleGroup  = console.GroupName

	legacyGroup         = ""
	legacyAuthzGroup    = ""
	legacyBuildGroup    = ""
	legacyImageGroup    = ""
	legacyProjectGroup  = ""
	legacyTemplateGroup = ""
	legacyUserGroup     = ""
	legacyOauthGroup    = ""
)

// Do not change any of these lists without approval from the auth and master teams
// Most rules are copied from various cluster roles in bootstrap policy
var (
	allUnauthenticatedRules = []rbacv1.PolicyRule{
		rbacv1helpers.NewRule("get", "create").Groups(buildGroup, legacyBuildGroup).Resources("buildconfigs/webhooks").RuleOrDie(),

		rbacv1helpers.NewRule("impersonate").Groups(kAuthnGroup).Resources("userextras/scopes.authorization.openshift.io").RuleOrDie(),

		rbacv1helpers.NewRule("create").Groups(authzGroup, legacyAuthzGroup).Resources("selfsubjectrulesreviews").RuleOrDie(),

		rbacv1helpers.NewRule("create").Groups(kAuthzGroup).Resources("selfsubjectaccessreviews", "selfsubjectrulesreviews").RuleOrDie(),

		rbacv1helpers.NewRule("delete").Groups(oauthGroup, legacyOauthGroup).Resources("oauthaccesstokens", "oauthauthorizetokens").RuleOrDie(),

		// this is openshift specific
		rbacv1helpers.NewRule("get").URLs(
			"/version/openshift",
			"/.well-known",
			"/.well-known/*",
			"/.well-known/oauth-authorization-server",
		).RuleOrDie(),

		// TODO: remove with after 1.15 rebase
		rbacv1helpers.NewRule("get").URLs(
			"/readyz",
		).RuleOrDie(),

		// this is from upstream kube
		rbacv1helpers.NewRule("get").URLs(
			"/healthz", "/livez",
			"/version",
			"/version/",
		).RuleOrDie(),
	}

	allAuthenticatedRules = append(
		[]rbacv1.PolicyRule{
			rbacv1helpers.NewRule("create").Groups(buildGroup, legacyBuildGroup).Resources("builds/docker", "builds/optimizeddocker").RuleOrDie(),
			rbacv1helpers.NewRule("create").Groups(buildGroup, legacyBuildGroup).Resources("builds/jenkinspipeline").RuleOrDie(),
			rbacv1helpers.NewRule("create").Groups(buildGroup, legacyBuildGroup).Resources("builds/source").RuleOrDie(),

			rbacv1helpers.NewRule("get").Groups(userGroup, legacyUserGroup).Resources("users").Names("~").RuleOrDie(),
			rbacv1helpers.NewRule("list").Groups(projectGroup, legacyProjectGroup).Resources("projectrequests").RuleOrDie(),
			rbacv1helpers.NewRule("get", "list").Groups(authzGroup, legacyAuthzGroup).Resources("clusterroles").RuleOrDie(),
			rbacv1helpers.NewRule(read...).Groups(rbacGroup).Resources("clusterroles").RuleOrDie(),
			rbacv1helpers.NewRule("get", "list").Groups(storageGroup).Resources("storageclasses").RuleOrDie(),
			rbacv1helpers.NewRule("list", "watch").Groups(projectGroup, legacyProjectGroup).Resources("projects").RuleOrDie(),

			// These custom resources are used to extend console functionality
			// The console team is working on eliminating this exception in the near future
			rbacv1helpers.NewRule(read...).Groups(consoleGroup).Resources("consoleclidownloads", "consolelinks", "consoleexternalloglinks", "consolenotifications", "consoleyamlsamples").RuleOrDie(),

			// TODO: remove when openshift-apiserver has removed these
			rbacv1helpers.NewRule("get").URLs(
				"/healthz/",
				"/oapi", "/oapi/*",
				"/osapi", "/osapi/",
				"/swaggerapi", "/swaggerapi/*", "/swagger.json", "/swagger-2.0.0.pb-v1",
				"/version/*",
				"/",
			).RuleOrDie(),

			// this is from upstream kube
			rbacv1helpers.NewRule("get").URLs(
				"/",
				"/openapi", "/openapi/*",
				"/api", "/api/*",
				"/apis", "/apis/*",
			).RuleOrDie(),
		},
		allUnauthenticatedRules...,
	)

	// group -> namespace -> rules
	groupNamespaceRules = map[string]map[string][]rbacv1.PolicyRule{
		kuser.AllAuthenticated: {
			"openshift": {
				rbacv1helpers.NewRule(read...).Groups(templateGroup, legacyTemplateGroup).Resources("templates").RuleOrDie(),
				rbacv1helpers.NewRule(read...).Groups(imageGroup, legacyImageGroup).Resources("imagestreams", "imagestreamtags", "imagestreamimages").RuleOrDie(),
				rbacv1helpers.NewRule("get").Groups(imageGroup, legacyImageGroup).Resources("imagestreams/layers").RuleOrDie(),
			},
			"openshift-config-managed": {
				rbacv1helpers.NewRule("get").Groups(legacyGroup).Resources("configmaps").Names("console-public").RuleOrDie(),
			},
		},
		kuser.AllUnauthenticated:     {}, // no rules expect the cluster wide ones
		"system:authenticated:oauth": {}, // no rules expect the cluster wide ones
	}
)

var _ = g.Describe("[Feature:OpenShiftAuthorization] The default cluster RBAC policy", func() {
	defer g.GinkgoRecover()

	oc := exutil.NewCLI("default-rbac-policy", exutil.KubeConfigPath())

	g.It("should have correct RBAC rules", func() {
		kubeInformers := informers.NewSharedInformerFactory(oc.AdminKubeClient(), 20*time.Minute)
		ruleResolver := exutil.NewRuleResolver(kubeInformers.Rbac().V1()) // signal what informers we want to use early

		stopCh := make(chan struct{})
		defer func() { close(stopCh) }()
		kubeInformers.Start(stopCh)

		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()

		if ok := cache.WaitForCacheSync(ctx.Done(),
			kubeInformers.Rbac().V1().ClusterRoles().Informer().HasSynced,
			kubeInformers.Rbac().V1().ClusterRoleBindings().Informer().HasSynced,
			kubeInformers.Rbac().V1().Roles().Informer().HasSynced,
			kubeInformers.Rbac().V1().RoleBindings().Informer().HasSynced,
		); !ok {
			exutil.FatalErr("failed to sync RBAC cache")
		}

		namespaces, err := oc.AdminKubeClient().CoreV1().Namespaces().List(metav1.ListOptions{})
		if err != nil {
			exutil.FatalErr(err)
		}

		g.By("should only allow the system:authenticated group to access certain policy rules", func() {
			testAllGroupRules(ruleResolver, kuser.AllAuthenticated, allAuthenticatedRules, namespaces.Items)
		})

		g.By("should only allow the system:unauthenticated group to access certain policy rules", func() {
			testAllGroupRules(ruleResolver, kuser.AllUnauthenticated, allUnauthenticatedRules, namespaces.Items)
		})

		g.By("should only allow the system:authenticated:oauth group to access certain policy rules", func() {
			testAllGroupRules(ruleResolver, "system:authenticated:oauth", []rbacv1.PolicyRule{
				rbacv1helpers.NewRule("create").Groups(projectGroup, legacyProjectGroup).Resources("projectrequests").RuleOrDie(),
			}, namespaces.Items)
		})

	})
})

func testAllGroupRules(ruleResolver validation.AuthorizationRuleResolver, group string, expectedClusterRules []rbacv1.PolicyRule, namespaces []corev1.Namespace) {
	testGroupRules(ruleResolver, group, metav1.NamespaceNone, expectedClusterRules)

	for _, namespace := range namespaces {
		// merge the namespace scoped and cluster wide rules
		rules := append([]rbacv1.PolicyRule{}, groupNamespaceRules[group][namespace.Name]...)
		rules = append(rules, expectedClusterRules...)

		testGroupRules(ruleResolver, group, namespace.Name, rules)
	}
}

func testGroupRules(ruleResolver validation.AuthorizationRuleResolver, group, namespace string, expectedRules []rbacv1.PolicyRule) {
	actualRules, err := ruleResolver.RulesFor(&kuser.DefaultInfo{Groups: []string{group}}, namespace)
	o.Expect(err).NotTo(o.HaveOccurred()) // our default RBAC policy should never have rule resolution errors

	if cover, missing := validation.Covers(expectedRules, actualRules); !cover {
		e2e.Failf("%s has extra permissions in namespace %q:\n%s", group, namespace, rulesToString(missing))
	}

	// force test data to be cleaned up every so often but allow extra rules to not deadlock new changes
	if cover, missing := validation.Covers(actualRules, expectedRules); !cover {
		log := e2e.Logf
		if len(missing) > 15 {
			log = e2e.Failf
		}
		log("test data for %s has too many unnecessary permissions:\n%s", group, rulesToString(missing))
	}
}

func rulesToString(rules []rbacv1.PolicyRule) string {
	compactRules := rules
	if compact, err := validation.CompactRules(rules); err == nil {
		compactRules = compact
	}

	missingDescriptions := sets.NewString()
	for _, missing := range compactRules {
		missingDescriptions.Insert(rbacv1helpers.CompactString(missing))
	}

	return strings.Join(missingDescriptions.List(), "\n")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/nf/reddit" // HL
	"log"
)

func main() {
	items, err := reddit.Get("golang") // HL
	if err != nil {
		log.Fatal(err)
	}
	for _, item := range items {
		fmt.Println(item)
	}
}
<commit_msg>go.talks: fix build by removing dependency<commit_after>// +build ignore,OMIT

package main

import (
	"fmt"
	"github.com/nf/reddit" // HL
	"log"
)

func main() {
	items, err := reddit.Get("golang") // HL
	if err != nil {
		log.Fatal(err)
	}
	for _, item := range items {
		fmt.Println(item)
	}
}
<|endoftext|>


<commit_before>package authenticators

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"

	"code.cloudfoundry.org/lager"
	"github.com/dgrijalva/jwt-go"
	"golang.org/x/crypto/ssh"
)

type CFAuthenticator struct {
	logger             lager.Logger
	httpClient         *http.Client
	ccURL              string
	uaaTokenURL        string
	uaaPassword        string
	uaaUsername        string
	permissionsBuilder PermissionsBuilder
}

type AppSSHResponse struct {
	ProcessGuid string `json:"process_guid"`
}

type UAAAuthTokenResponse struct {
	AccessToken string `json:"access_token"`
	TokenType   string `json:"token_type"`
}

var CFUserRegex *regexp.Regexp = regexp.MustCompile(`cf:([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})/(\d+)`)

func NewCFAuthenticator(
	logger lager.Logger,
	httpClient *http.Client,
	ccURL string,
	uaaTokenURL string,
	uaaUsername string,
	uaaPassword string,
	permissionsBuilder PermissionsBuilder,
) *CFAuthenticator {
	return &CFAuthenticator{
		logger:             logger,
		httpClient:         httpClient,
		ccURL:              ccURL,
		uaaTokenURL:        uaaTokenURL,
		uaaUsername:        uaaUsername,
		uaaPassword:        uaaPassword,
		permissionsBuilder: permissionsBuilder,
	}
}

func (cfa *CFAuthenticator) UserRegexp() *regexp.Regexp {
	return CFUserRegex
}

func (cfa *CFAuthenticator) Authenticate(metadata ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) {
	logger := cfa.logger.Session("cf-authenticate")
	logger.Info("authenticate-starting")
	defer logger.Info("authenticate-finished")

	if !CFUserRegex.MatchString(metadata.User()) {
		logger.Error("regex-match-fail", InvalidCredentialsErr)
		return nil, InvalidCredentialsErr
	}

	guidAndIndex := CFUserRegex.FindStringSubmatch(metadata.User())

	appGuid := guidAndIndex[1]

	index, err := strconv.Atoi(guidAndIndex[2])
	if err != nil {
		logger.Error("atoi-failed", err)
		return nil, InvalidCredentialsErr
	}

	cred, err := cfa.exchangeAccessCodeForToken(logger, string(password))
	if err != nil {
		return nil, err
	}

	parts := strings.Split(cred, " ")
	if len(parts) != 2 {
		return nil, AuthenticationFailedErr
	}
	tokenString := parts[1]
	// When parsing the certificate validating the signature is not required and we don't readily have the
	// certificate to validate the signature.  This is just to parse the second information part of the token anyway.
	token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) {
		return []byte("Doesntmatter"), nil
	})

	username, ok := token.Claims.(jwt.MapClaims)["user_name"].(string)
	if !ok {
		username = "unknown"
	}
	principal, ok := token.Claims.(jwt.MapClaims)["user_id"].(string)
	if !ok {
		principal = "unknown"
	}

	logger = logger.WithData(lager.Data{
		"app":       fmt.Sprintf("%s/%d", appGuid, index),
		"principal": principal,
		"username":  username,
	})

	processGuid, err := cfa.checkAccess(logger, appGuid, index, string(cred))
	if err != nil {
		return nil, err
	}

	permissions, err := cfa.permissionsBuilder.Build(logger, processGuid, index, metadata)
	if err != nil {
		logger.Error("building-ssh-permissions-failed", err)
	}

	logger.Info("app-access-success")

	return permissions, err
}

func (cfa *CFAuthenticator) exchangeAccessCodeForToken(logger lager.Logger, code string) (string, error) {
	logger = logger.Session("exchange-access-code-for-token")

	formValues := make(url.Values)
	formValues.Set("grant_type", "authorization_code")
	formValues.Set("code", code)

	req, err := http.NewRequest("POST", cfa.uaaTokenURL, strings.NewReader(formValues.Encode()))
	if err != nil {
		return "", err
	}

	req.SetBasicAuth(cfa.uaaUsername, cfa.uaaPassword)
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	resp, err := cfa.httpClient.Do(req)
	if err != nil {
		logger.Error("request-failed", err)
		return "", AuthenticationFailedErr
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		logger.Error("response-status-not-ok", AuthenticationFailedErr, lager.Data{
			"status-code": resp.StatusCode,
		})
		return "", AuthenticationFailedErr
	}

	var tokenResponse UAAAuthTokenResponse
	err = json.NewDecoder(resp.Body).Decode(&tokenResponse)
	if err != nil {
		logger.Error("decode-token-response-failed", err)
		return "", AuthenticationFailedErr
	}

	return fmt.Sprintf("%s %s", tokenResponse.TokenType, tokenResponse.AccessToken), nil
}

func (cfa *CFAuthenticator) checkAccess(logger lager.Logger, appGuid string, index int, token string) (string, error) {
	path := fmt.Sprintf("%s/internal/apps/%s/ssh_access/%d", cfa.ccURL, appGuid, index)

	req, err := http.NewRequest("GET", path, nil)
	if err != nil {
		logger.Error("creating-request-failed", InvalidRequestErr)
		return "", InvalidRequestErr
	}
	req.Header.Add("Authorization", token)

	resp, err := cfa.httpClient.Do(req)
	if err != nil {
		logger.Error("fetching-app-failed", err)
		return "", err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		logger.Error("fetching-app-failed", FetchAppFailedErr, lager.Data{
			"StatusCode":   resp.Status,
			"ResponseBody": resp.Body,
		})
		return "", FetchAppFailedErr
	}

	var app AppSSHResponse
	err = json.NewDecoder(resp.Body).Decode(&app)
	if err != nil {
		logger.Error("invalid-cc-response", err)
		return "", InvalidCCResponse
	}

	return app.ProcessGuid, nil
}
<commit_msg>Switch to golang-jwt/jwt instead of dgrijalva/jwt<commit_after>package authenticators

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"

	"code.cloudfoundry.org/lager"
	"github.com/golang-jwt/jwt/v4"
	"golang.org/x/crypto/ssh"
)

type CFAuthenticator struct {
	logger             lager.Logger
	httpClient         *http.Client
	ccURL              string
	uaaTokenURL        string
	uaaPassword        string
	uaaUsername        string
	permissionsBuilder PermissionsBuilder
}

type AppSSHResponse struct {
	ProcessGuid string `json:"process_guid"`
}

type UAAAuthTokenResponse struct {
	AccessToken string `json:"access_token"`
	TokenType   string `json:"token_type"`
}

var CFUserRegex *regexp.Regexp = regexp.MustCompile(`cf:([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})/(\d+)`)

func NewCFAuthenticator(
	logger lager.Logger,
	httpClient *http.Client,
	ccURL string,
	uaaTokenURL string,
	uaaUsername string,
	uaaPassword string,
	permissionsBuilder PermissionsBuilder,
) *CFAuthenticator {
	return &CFAuthenticator{
		logger:             logger,
		httpClient:         httpClient,
		ccURL:              ccURL,
		uaaTokenURL:        uaaTokenURL,
		uaaUsername:        uaaUsername,
		uaaPassword:        uaaPassword,
		permissionsBuilder: permissionsBuilder,
	}
}

func (cfa *CFAuthenticator) UserRegexp() *regexp.Regexp {
	return CFUserRegex
}

func (cfa *CFAuthenticator) Authenticate(metadata ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) {
	logger := cfa.logger.Session("cf-authenticate")
	logger.Info("authenticate-starting")
	defer logger.Info("authenticate-finished")

	if !CFUserRegex.MatchString(metadata.User()) {
		logger.Error("regex-match-fail", InvalidCredentialsErr)
		return nil, InvalidCredentialsErr
	}

	guidAndIndex := CFUserRegex.FindStringSubmatch(metadata.User())

	appGuid := guidAndIndex[1]

	index, err := strconv.Atoi(guidAndIndex[2])
	if err != nil {
		logger.Error("atoi-failed", err)
		return nil, InvalidCredentialsErr
	}

	cred, err := cfa.exchangeAccessCodeForToken(logger, string(password))
	if err != nil {
		return nil, err
	}

	parts := strings.Split(cred, " ")
	if len(parts) != 2 {
		return nil, AuthenticationFailedErr
	}
	tokenString := parts[1]
	// When parsing the certificate validating the signature is not required and we don't readily have the
	// certificate to validate the signature.  This is just to parse the second information part of the token anyway.
	token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) {
		return []byte("Doesntmatter"), nil
	})

	username, ok := token.Claims.(jwt.MapClaims)["user_name"].(string)
	if !ok {
		username = "unknown"
	}
	principal, ok := token.Claims.(jwt.MapClaims)["user_id"].(string)
	if !ok {
		principal = "unknown"
	}

	logger = logger.WithData(lager.Data{
		"app":       fmt.Sprintf("%s/%d", appGuid, index),
		"principal": principal,
		"username":  username,
	})

	processGuid, err := cfa.checkAccess(logger, appGuid, index, string(cred))
	if err != nil {
		return nil, err
	}

	permissions, err := cfa.permissionsBuilder.Build(logger, processGuid, index, metadata)
	if err != nil {
		logger.Error("building-ssh-permissions-failed", err)
	}

	logger.Info("app-access-success")

	return permissions, err
}

func (cfa *CFAuthenticator) exchangeAccessCodeForToken(logger lager.Logger, code string) (string, error) {
	logger = logger.Session("exchange-access-code-for-token")

	formValues := make(url.Values)
	formValues.Set("grant_type", "authorization_code")
	formValues.Set("code", code)

	req, err := http.NewRequest("POST", cfa.uaaTokenURL, strings.NewReader(formValues.Encode()))
	if err != nil {
		return "", err
	}

	req.SetBasicAuth(cfa.uaaUsername, cfa.uaaPassword)
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	resp, err := cfa.httpClient.Do(req)
	if err != nil {
		logger.Error("request-failed", err)
		return "", AuthenticationFailedErr
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		logger.Error("response-status-not-ok", AuthenticationFailedErr, lager.Data{
			"status-code": resp.StatusCode,
		})
		return "", AuthenticationFailedErr
	}

	var tokenResponse UAAAuthTokenResponse
	err = json.NewDecoder(resp.Body).Decode(&tokenResponse)
	if err != nil {
		logger.Error("decode-token-response-failed", err)
		return "", AuthenticationFailedErr
	}

	return fmt.Sprintf("%s %s", tokenResponse.TokenType, tokenResponse.AccessToken), nil
}

func (cfa *CFAuthenticator) checkAccess(logger lager.Logger, appGuid string, index int, token string) (string, error) {
	path := fmt.Sprintf("%s/internal/apps/%s/ssh_access/%d", cfa.ccURL, appGuid, index)

	req, err := http.NewRequest("GET", path, nil)
	if err != nil {
		logger.Error("creating-request-failed", InvalidRequestErr)
		return "", InvalidRequestErr
	}
	req.Header.Add("Authorization", token)

	resp, err := cfa.httpClient.Do(req)
	if err != nil {
		logger.Error("fetching-app-failed", err)
		return "", err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		logger.Error("fetching-app-failed", FetchAppFailedErr, lager.Data{
			"StatusCode":   resp.Status,
			"ResponseBody": resp.Body,
		})
		return "", FetchAppFailedErr
	}

	var app AppSSHResponse
	err = json.NewDecoder(resp.Body).Decode(&app)
	if err != nil {
		logger.Error("invalid-cc-response", err)
		return "", InvalidCCResponse
	}

	return app.ProcessGuid, nil
}
<|endoftext|>


<commit_before>package html

import (
	"fmt"
	"github.com/vlifesystems/rulehuntersrv/config"
	"github.com/vlifesystems/rulehuntersrv/html/cmd"
	"github.com/vlifesystems/rulehuntersrv/internal/testhelpers"
	"github.com/vlifesystems/rulehuntersrv/logger"
	"github.com/vlifesystems/rulehuntersrv/progress"
	"github.com/vlifesystems/rulehuntersrv/quitter"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"testing"
	"time"
)

// This checks if Run will quit properly when told to
func TestRun_quit(t *testing.T) {
	config := &config.Config{
		ExperimentsDir:   "experiments",
		WWWDir:           "www",
		BuildDir:         "build",
		NumRulesInReport: 100,
	}
	q := quitter.New()
	l := logger.NewTestLogger(q)
	htmlCmds := make(chan cmd.Cmd)
	wd, err := os.Getwd()
	if err != nil {
		t.Fatalf("Getwd() err: ", err)
	}
	defer os.Chdir(wd)
	configDir, err := testhelpers.BuildConfigDirs()
	defer os.RemoveAll(configDir)
	pm := progress.NewMonitor(
		filepath.Join(configDir, "build", "progress"),
		htmlCmds,
	)
	go Run(config, pm, l, q, htmlCmds)
	time.Sleep(1 * time.Second)
	htmlCmds <- cmd.Flush

	go func() {
		quitTime := time.Now()
		const secsWait = 2.0
		for {
			durationSinceQuit := time.Since(quitTime)
			if durationSinceQuit.Seconds() > secsWait {
				t.Fatalf("Run() didn't quit within %d seconds", secsWait)
			}
		}
	}()
	q.Quit()
}

func TestGenReportFilename(t *testing.T) {
	wwwDir := "/var/wwww"
	cases := []struct {
		stamp        time.Time
		title        string
		wantFilename string
	}{
		{time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC),
			"This could be very interesting",
			filepath.Join(wwwDir, "reports", "2009", "11", "10",
				fmt.Sprintf("%s_this-could-be-very-interesting",
					genStampMagicString(
						time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC),
					),
				),
				"index.html")},
	}
	for _, c := range cases {
		got := genReportFilename(wwwDir, c.stamp, c.title)
		if got != c.wantFilename {
			t.Errorf("genReportFilename(%s, %s) got: %s, want: %s",
				c.stamp, c.title, got, c.wantFilename)
		}
	}
}

func TestGenReportURLDir(t *testing.T) {
	cases := []struct {
		stamp   time.Time
		title   string
		wantDir string
	}{
		{time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC),
			"This could be very interesting",
			fmt.Sprintf("/reports/2009/11/10/%s_this-could-be-very-interesting/",
				genStampMagicString(
					time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC),
				),
			),
		},
	}
	for _, c := range cases {
		got := genReportURLDir(c.stamp, c.title)
		if got != c.wantDir {
			t.Errorf("genReportFilename(%s, %s) got: %s, want: %s",
				c.stamp, c.title, got, c.wantDir)
		}
	}
}

func TestMakeReportURLDir(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "html_test")
	if err != nil {
		t.Errorf("TempDir() err: %s", err)
		return
	}
	defer os.RemoveAll(tempDir)
	cases := []struct {
		stamp         time.Time
		title         string
		wantDirExists string
		wantURLDir    string
	}{
		{time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC),
			"This could be very interesting",
			filepath.Join(tempDir, "reports", "2009", "11", "10",
				fmt.Sprintf("%s_this-could-be-very-interesting/",
					genStampMagicString(
						time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC),
					),
				),
			),
			fmt.Sprintf("/reports/2009/11/10/%s_this-could-be-very-interesting/",
				genStampMagicString(
					time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC),
				),
			),
		},
	}
	for _, c := range cases {
		got, err := makeReportURLDir(tempDir, c.stamp, c.title)
		if err != nil {
			t.Errorf("makeReportURLDir(%s, %s, %s) err: %s",
				tempDir, c.stamp, c.title, err)
		}
		if got != c.wantURLDir {
			t.Errorf("makeReportURLDir(%s, %s, %s) got: %s, want: %s",
				tempDir, c.stamp, c.title, got, c.wantURLDir)
		}
		if !dirExists(c.wantDirExists) {
			t.Errorf("makeReportURLDir(%s, %s, %s)  - directory doesn't exist: %s",
				tempDir, c.stamp, c.title, c.wantDirExists)
		}
	}
}

func TestEscapeString(t *testing.T) {
	cases := []struct {
		in   string
		want string
	}{
		{"This is a TITLE",
			"this-is-a-title"},
		{"  hello how are % you423 33  today __ --",
			"hello-how-are-you423-33-today"},
		{"--  hello how are %^& you423 33  today __ --",
			"hello-how-are-you423-33-today"},
		{"hello((_ how are % you423 33  today",
			"hello-how-are-you423-33-today"},
		{"", ""},
	}
	for _, c := range cases {
		got := escapeString(c.in)
		if got != c.want {
			t.Errorf("escapeString(%s) got: %s, want: %s", c.in, got, c.want)
		}
	}
}

func TestGenStampMagicString(t *testing.T) {
	cases := []struct {
		in       time.Time
		wantDiff uint64
	}{
		{time.Date(2009, time.November, 10, 22, 19, 18, 200, time.UTC), 0},
		{time.Date(2009, time.November, 11, 22, 19, 18, 200, time.UTC), 0},
		{time.Date(2009, time.December, 11, 22, 19, 18, 200, time.UTC), 0},
		{time.Date(2010, time.December, 11, 22, 19, 18, 200, time.UTC), 0},
		{time.Date(2009, time.November, 10, 22, 19, 19, 17, time.UTC), 1},
		{time.Date(2009, time.November, 10, 22, 19, 29, 17, time.UTC), 11},
		{time.Date(2009, time.November, 10, 22, 20, 18, 17, time.UTC), 60},
		{time.Date(2009, time.November, 10, 23, 19, 18, 17, time.UTC), 3600},
	}

	initStamp := time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC)
	initMagicStr := genStampMagicString(initStamp)

	initMagicNum, err := strconv.ParseUint(initMagicStr, 36, 64)
	if err != nil {
		t.Errorf("ParseUint(%s, 36, 64) err: %s", initMagicStr, err)
		return
	}

	for _, c := range cases {
		magicStr := genStampMagicString(c.in)
		magicNum, err := strconv.ParseUint(magicStr, 36, 64)
		if err != nil {
			t.Errorf("ParseUint(%s, 36, 64) err: %s", magicStr, err)
			return
		}
		diff := magicNum - initMagicNum
		if diff != c.wantDiff {
			t.Errorf("diff != wantDiff for stamp: %s got: %d, want: %d",
				c.in, diff, c.wantDiff)
		}
	}
}

func dirExists(path string) bool {
	fi, err := os.Stat(path)
	if err != nil {
		return false
	}
	return fi.IsDir()
}
<commit_msg>Correct html test passing Quitter to NewLogger<commit_after>package html

import (
	"fmt"
	"github.com/vlifesystems/rulehuntersrv/config"
	"github.com/vlifesystems/rulehuntersrv/html/cmd"
	"github.com/vlifesystems/rulehuntersrv/internal/testhelpers"
	"github.com/vlifesystems/rulehuntersrv/logger"
	"github.com/vlifesystems/rulehuntersrv/progress"
	"github.com/vlifesystems/rulehuntersrv/quitter"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"testing"
	"time"
)

// This checks if Run will quit properly when told to
func TestRun_quit(t *testing.T) {
	config := &config.Config{
		ExperimentsDir:   "experiments",
		WWWDir:           "www",
		BuildDir:         "build",
		NumRulesInReport: 100,
	}
	q := quitter.New()
	l := logger.NewTestLogger()
	htmlCmds := make(chan cmd.Cmd)
	wd, err := os.Getwd()
	if err != nil {
		t.Fatalf("Getwd() err: ", err)
	}
	defer os.Chdir(wd)
	configDir, err := testhelpers.BuildConfigDirs()
	defer os.RemoveAll(configDir)
	pm := progress.NewMonitor(
		filepath.Join(configDir, "build", "progress"),
		htmlCmds,
	)
	go Run(config, pm, l, q, htmlCmds)
	time.Sleep(1 * time.Second)
	htmlCmds <- cmd.Flush

	go func() {
		quitTime := time.Now()
		const secsWait = 2.0
		for {
			durationSinceQuit := time.Since(quitTime)
			if durationSinceQuit.Seconds() > secsWait {
				t.Fatalf("Run() didn't quit within %d seconds", secsWait)
			}
		}
	}()
	q.Quit()
}

func TestGenReportFilename(t *testing.T) {
	wwwDir := "/var/wwww"
	cases := []struct {
		stamp        time.Time
		title        string
		wantFilename string
	}{
		{time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC),
			"This could be very interesting",
			filepath.Join(wwwDir, "reports", "2009", "11", "10",
				fmt.Sprintf("%s_this-could-be-very-interesting",
					genStampMagicString(
						time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC),
					),
				),
				"index.html")},
	}
	for _, c := range cases {
		got := genReportFilename(wwwDir, c.stamp, c.title)
		if got != c.wantFilename {
			t.Errorf("genReportFilename(%s, %s) got: %s, want: %s",
				c.stamp, c.title, got, c.wantFilename)
		}
	}
}

func TestGenReportURLDir(t *testing.T) {
	cases := []struct {
		stamp   time.Time
		title   string
		wantDir string
	}{
		{time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC),
			"This could be very interesting",
			fmt.Sprintf("/reports/2009/11/10/%s_this-could-be-very-interesting/",
				genStampMagicString(
					time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC),
				),
			),
		},
	}
	for _, c := range cases {
		got := genReportURLDir(c.stamp, c.title)
		if got != c.wantDir {
			t.Errorf("genReportFilename(%s, %s) got: %s, want: %s",
				c.stamp, c.title, got, c.wantDir)
		}
	}
}

func TestMakeReportURLDir(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "html_test")
	if err != nil {
		t.Errorf("TempDir() err: %s", err)
		return
	}
	defer os.RemoveAll(tempDir)
	cases := []struct {
		stamp         time.Time
		title         string
		wantDirExists string
		wantURLDir    string
	}{
		{time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC),
			"This could be very interesting",
			filepath.Join(tempDir, "reports", "2009", "11", "10",
				fmt.Sprintf("%s_this-could-be-very-interesting/",
					genStampMagicString(
						time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC),
					),
				),
			),
			fmt.Sprintf("/reports/2009/11/10/%s_this-could-be-very-interesting/",
				genStampMagicString(
					time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC),
				),
			),
		},
	}
	for _, c := range cases {
		got, err := makeReportURLDir(tempDir, c.stamp, c.title)
		if err != nil {
			t.Errorf("makeReportURLDir(%s, %s, %s) err: %s",
				tempDir, c.stamp, c.title, err)
		}
		if got != c.wantURLDir {
			t.Errorf("makeReportURLDir(%s, %s, %s) got: %s, want: %s",
				tempDir, c.stamp, c.title, got, c.wantURLDir)
		}
		if !dirExists(c.wantDirExists) {
			t.Errorf("makeReportURLDir(%s, %s, %s)  - directory doesn't exist: %s",
				tempDir, c.stamp, c.title, c.wantDirExists)
		}
	}
}

func TestEscapeString(t *testing.T) {
	cases := []struct {
		in   string
		want string
	}{
		{"This is a TITLE",
			"this-is-a-title"},
		{"  hello how are % you423 33  today __ --",
			"hello-how-are-you423-33-today"},
		{"--  hello how are %^& you423 33  today __ --",
			"hello-how-are-you423-33-today"},
		{"hello((_ how are % you423 33  today",
			"hello-how-are-you423-33-today"},
		{"", ""},
	}
	for _, c := range cases {
		got := escapeString(c.in)
		if got != c.want {
			t.Errorf("escapeString(%s) got: %s, want: %s", c.in, got, c.want)
		}
	}
}

func TestGenStampMagicString(t *testing.T) {
	cases := []struct {
		in       time.Time
		wantDiff uint64
	}{
		{time.Date(2009, time.November, 10, 22, 19, 18, 200, time.UTC), 0},
		{time.Date(2009, time.November, 11, 22, 19, 18, 200, time.UTC), 0},
		{time.Date(2009, time.December, 11, 22, 19, 18, 200, time.UTC), 0},
		{time.Date(2010, time.December, 11, 22, 19, 18, 200, time.UTC), 0},
		{time.Date(2009, time.November, 10, 22, 19, 19, 17, time.UTC), 1},
		{time.Date(2009, time.November, 10, 22, 19, 29, 17, time.UTC), 11},
		{time.Date(2009, time.November, 10, 22, 20, 18, 17, time.UTC), 60},
		{time.Date(2009, time.November, 10, 23, 19, 18, 17, time.UTC), 3600},
	}

	initStamp := time.Date(2009, time.November, 10, 22, 19, 18, 17, time.UTC)
	initMagicStr := genStampMagicString(initStamp)

	initMagicNum, err := strconv.ParseUint(initMagicStr, 36, 64)
	if err != nil {
		t.Errorf("ParseUint(%s, 36, 64) err: %s", initMagicStr, err)
		return
	}

	for _, c := range cases {
		magicStr := genStampMagicString(c.in)
		magicNum, err := strconv.ParseUint(magicStr, 36, 64)
		if err != nil {
			t.Errorf("ParseUint(%s, 36, 64) err: %s", magicStr, err)
			return
		}
		diff := magicNum - initMagicNum
		if diff != c.wantDiff {
			t.Errorf("diff != wantDiff for stamp: %s got: %d, want: %d",
				c.in, diff, c.wantDiff)
		}
	}
}

func dirExists(path string) bool {
	fi, err := os.Stat(path)
	if err != nil {
		return false
	}
	return fi.IsDir()
}
<|endoftext|>


<commit_before>package raindrops

import "testing"

const targetTestVersion = 2

func TestConvert(t *testing.T) {
	if testVersion != targetTestVersion {
		t.Fatalf("Found testVersion = %v, want %v", testVersion, targetTestVersion)
	}
	for _, test := range tests {
		if actual := Convert(test.input); actual != test.expected {
			t.Errorf("Convert(%d) = %q, expected %q.",
				test.input, actual, test.expected)
		}
	}
}

func BenchmarkConvert(b *testing.B) {
	for i := 0; i < b.N; i++ {
		for _, test := range tests {
			Convert(test.input)
		}
	}
}
<commit_msg>raindrops: Ensure test versioning consistency with other exercises (#578)<commit_after>package raindrops

import "testing"

const targetTestVersion = 2

func TestTestVersion(t *testing.T) {
	if testVersion != targetTestVersion {
		t.Fatalf("Found testVersion = %v, want %v", testVersion, targetTestVersion)
	}
}

func TestConvert(t *testing.T) {
	for _, test := range tests {
		if actual := Convert(test.input); actual != test.expected {
			t.Errorf("Convert(%d) = %q, expected %q.",
				test.input, actual, test.expected)
		}
	}
}

func BenchmarkConvert(b *testing.B) {
	for i := 0; i < b.N; i++ {
		for _, test := range tests {
			Convert(test.input)
		}
	}
}
<|endoftext|>


<commit_before>package redis

import (
	"github.com/garyburd/redigo/redis"
)

// Get will retrieve a key
func (c *RedisStore) Get(key string) (result []byte, err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	raw, err := conn.Do("GET", key)
	if raw == nil {
		return nil, ErrCacheMiss
	}
	result, err = redis.Bytes(raw, err)
	if err != nil {
		return
	}

	return
}

// HGet will retrieve a hash
func (c *RedisStore) HGet(key string, value string) (result []byte, err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	raw, err := conn.Do("HGET", key, value)
	if raw == nil {
		return nil, ErrCacheMiss
	}
	result, err = redis.Bytes(raw, err)
	if err != nil {
		return
	}

	return
}

// Set will set a single record
func (c *RedisStore) Set(key string, result []byte) (err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	_, err = conn.Do("SET", key, result)

	return
}

// Set will set a single record
func (c *RedisStore) SetEx(key string, timeout uint, result []byte) (err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	_, err = conn.Do("SETEX", key, timeout, result)

	return
}

// HMSet will set a hash
func (c *RedisStore) HMSet(key string, value string, result []byte) (err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	_, err = conn.Do("HMSET", key, value, result)

	return
}

// Delete will delete a key
func (c *RedisStore) Delete(key ...interface{}) (err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	_, err = conn.Do("DEL", key...)

	return
}

// Flush will call flushall and delete all keys
func (c *RedisStore) Flush() (err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	_, err = conn.Do("FLUSHALL")

	return
}

// will increment a redis key
func (c *RedisStore) Incr(key string) (result int, err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	raw, err := conn.Do("INCR", key)
	if raw == nil {
		return 0, ErrCacheMiss
	}
	result, err = redis.Int(raw, err)
	if err != nil {
		return
	}

	return
}

// will set expire on a redis key
func (c *RedisStore) Expire(key string, timeout uint) (err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	_, err = conn.Do("EXPIRE", key, timeout)

	return
}
<commit_msg>fix up redis lib<commit_after>package redis

import (
	"github.com/garyburd/redigo/redis"
)

// Get will retrieve a key
func (c *RedisStore) Get(key string) (result []byte, err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	return redis.Bytes(conn.Do("GET", key))

}

// HGet will retrieve a hash
func (c *RedisStore) HGet(key string, value string) (result []byte, err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	return redis.Bytes(conn.Do("HGET", key, value))

}

// Set will set a single record
func (c *RedisStore) Set(key string, result []byte) (err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	_, err = conn.Do("SET", key, result)

	return
}

// Set will set a single record
func (c *RedisStore) SetEx(key string, timeout uint, result []byte) (err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	_, err = conn.Do("SETEX", key, timeout, result)

	return
}

// HMSet will set a hash
func (c *RedisStore) HMSet(key string, value string, result []byte) (err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	_, err = conn.Do("HMSET", key, value, result)

	return
}

// Delete will delete a key
func (c *RedisStore) Delete(key ...interface{}) (err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	_, err = conn.Do("DEL", key...)

	return
}

// Flush will call flushall and delete all keys
func (c *RedisStore) Flush() (err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	_, err = conn.Do("FLUSHALL")

	return
}

// will increment a redis key
func (c *RedisStore) Incr(key string) (result int, err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	return redis.Int(conn.Do("INCR", key))
}

// will set expire on a redis key
func (c *RedisStore) Expire(key string, timeout uint) (err error) {
	conn := c.Pool.Get()
	defer conn.Close()

	_, err = conn.Do("EXPIRE", key, timeout)

	return
}
<|endoftext|>