<commit_before>package endly

import (
	"github.com/lunixbochs/vtclean"
	"strings"
)

const commandNotFound = "command not found"
const noSuchFileOrDirectory = "no such file or directory"
const programCanBeFound = "can be found in the following packages"
const errorIsNotRecoverable = "Error is not recoverable"
const notInstalled = "not installed"

//CheckNoSuchFileOrDirectory checks for no such file or directory message in the provided stdout.
func CheckNoSuchFileOrDirectory(stdout ...string) bool {
	if len(stdout) == 0 {
		return false
	}
	candidate := strings.ToLower(strings.Join(stdout, "\n"))
	return strings.Contains(candidate, noSuchFileOrDirectory)
}

//CheckCommandNotFound checks for command not found message in the provided stdout.
func CheckCommandNotFound(stdout ...string) bool {
	if len(stdout) == 0 {
		return false
	}
	candidate := strings.ToLower(strings.Join(stdout, "\n"))
	return strings.Contains(candidate, commandNotFound) || strings.Contains(candidate, notInstalled)
}

//ExtractColumn extract a column from the line for provided index
func ExtractColumn(line string, columnIndex int) (string, bool) {
	var columns, has = ExtractColumns(line)
	if !has {
		return "", has
	}
	if columnIndex < len(columns) {
		return columns[columnIndex], true
	}
	return "", false
}

//ExtractColumns extract all column from the line
func ExtractColumns(line string) ([]string, bool) {
	if line == "" {
		return []string{}, false
	}
	line = vtclean.Clean(line, false)
	var index = -1
	var expectColumn = true
	var result = make([]string, 0)

	for i := 0; i < len(line); i++ {
		var aChar = string(line[i : i+1])
		if aChar == " " || aChar == "\t" {
			expectColumn = true
			continue
		}
		if expectColumn {
			index++
			result = append(result, "")
			expectColumn = false
		}
		result[index] += aChar
	}
	return result, true
}

func escapeStdout(stdout string) string {
	return vtclean.Clean(stdout, true)
}
<commit_msg>expanded not fund definition<commit_after>package endly

import (
	"github.com/lunixbochs/vtclean"
	"strings"
)

const commandNotFound = "command not found"
const noSuchFileOrDirectory = "no such file or directory"
const programCanBeFound = "can be found in the following packages"
const errorIsNotRecoverable = "Error is not recoverable"
const notInstalled = "not installed"
const canNotOpen = "Can't open"

//CheckNoSuchFileOrDirectory checks for no such file or directory message in the provided stdout.
func CheckNoSuchFileOrDirectory(stdout ...string) bool {
	if len(stdout) == 0 {
		return false
	}
	candidate := strings.ToLower(strings.Join(stdout, "\n"))
	return strings.Contains(candidate, noSuchFileOrDirectory)
}

//CheckCommandNotFound checks for command not found message in the provided stdout.
func CheckCommandNotFound(stdout ...string) bool {
	if len(stdout) == 0 {
		return false
	}
	candidate := strings.ToLower(strings.Join(stdout, "\n"))
	return strings.Contains(candidate, commandNotFound) || strings.Contains(candidate, notInstalled) || strings.Contains(candidate, canNotOpen)
}

//ExtractColumn extract a column from the line for provided index
func ExtractColumn(line string, columnIndex int) (string, bool) {
	var columns, has = ExtractColumns(line)
	if !has {
		return "", has
	}
	if columnIndex < len(columns) {
		return columns[columnIndex], true
	}
	return "", false
}

//ExtractColumns extract all column from the line
func ExtractColumns(line string) ([]string, bool) {
	if line == "" {
		return []string{}, false
	}
	line = vtclean.Clean(line, false)
	var index = -1
	var expectColumn = true
	var result = make([]string, 0)

	for i := 0; i < len(line); i++ {
		var aChar = string(line[i : i+1])
		if aChar == " " || aChar == "\t" {
			expectColumn = true
			continue
		}
		if expectColumn {
			index++
			result = append(result, "")
			expectColumn = false
		}
		result[index] += aChar
	}
	return result, true
}

func escapeStdout(stdout string) string {
	return vtclean.Clean(stdout, true)
}
<|endoftext|>


<commit_before>package stringutil

import (
	"strings"
)

func StringAfter(content string, find string) string {
	if len(find) == 0 {
		return content
	}
	pos := strings.Index(content, find)
	if pos == -1 {
		return ""
	}
	return content[pos+len(find):]
}

func StringBefore(content string, find string) string {
	if len(find) == 0 {
		return content
	}
	pos := strings.Index(content, find)
	if pos == -1 {
		return ""
	}
	return content[:pos]
}

func Trim(content string) string {
	runes := []rune(content)
	if len(runes) > 0 {
		if trimableChar(runes[0]) {
			return Trim(content[1:])
		}
		if trimableChar(runes[len(content)-1]) {
			return Trim(content[:len(content)-1])
		}
	}
	return content
}

func trimableChar(c rune) bool {
	return c == ' ' || c == '\n' || c == '\r'
}

func StringLess(a, b string) bool {
	for i := 0; i < len(a) && i < len(b); i++ {
		if a[i] < b[i] {
			return true
		}
		if a[i] > b[i] {
			return false
		}
	}
	return len(a) < len(b)
}
<commit_msg>handle version not found error<commit_after>package stringutil

import (
	"fmt"
	"strings"
)

func StringAfter(content string, find string) string {
	match, err := StringAfterError(content, find)
	if err != nil {
		return ""
	}
	return match
}

func StringAfterError(content string, find string) (string, error) {
	if len(find) == 0 {
		return content, nil
	}
	pos := strings.Index(content, find)
	if pos == -1 {
		return "", fmt.Errorf("can't find '%s' in content")
	}
	return content[pos+len(find):], nil
}

func StringBefore(content string, find string) string {
	match, err := StringBeforeError(content, find)
	if err != nil {
		return ""
	}
	return match
}

func StringBeforeError(content string, find string) (string, error) {
	if len(find) == 0 {
		return content, nil
	}
	pos := strings.Index(content, find)
	if pos == -1 {
		return "", fmt.Errorf("can't find '%s' in content")
	}
	return content[:pos], nil
}

func Trim(content string) string {
	runes := []rune(content)
	if len(runes) > 0 {
		if trimableChar(runes[0]) {
			return Trim(content[1:])
		}
		if trimableChar(runes[len(content)-1]) {
			return Trim(content[:len(content)-1])
		}
	}
	return content
}

func trimableChar(c rune) bool {
	return c == ' ' || c == '\n' || c == '\r'
}

func StringLess(a, b string) bool {
	for i := 0; i < len(a) && i < len(b); i++ {
		if a[i] < b[i] {
			return true
		}
		if a[i] > b[i] {
			return false
		}
	}
	return len(a) < len(b)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/technoweenie/grohl"
	"io"
	"os"
	"path/filepath"
	"time"
)

type Supervisor struct {
	Files []FileConfiguration
	Client
	Snapshotter
	SpoolSize    int
	SpoolTimeout time.Duration

	// How frequently to glob for new files that may have appeared
	GlobRefresh time.Duration
}

const (
	supervisorBackoffMinimum = 500 * time.Millisecond
	supervisorBackoffMaximum = 5 * time.Second

	supervisorEOFTimeout = 5 * time.Minute
)

// Pulls the entire program together. Connects file readers to a spooler to
// a client, snapshotting progress after a successful acknowledgement from
// the server.
//
// To stop the supervisor, send a message to the done channel.
func (s *Supervisor) Serve(done chan interface{}) {
	logger := grohl.NewContext(grohl.Data{"ns": "Supervisor"})

	spooler := NewSpooler(s.SpoolSize, s.SpoolTimeout)
	go spooler.Spool()
	defer func() { close(spooler.In) }()

	readers := new(FileReaderCollection)
	s.startFileReaders(spooler.In, readers)

	// In the case that a chunk fails, we retry it by setting it as the
	// retryChunk.  We keep retrying the chunk until it sends correctly, then
	// move on to the normal queues.
	var retryChunk []*FileData
	retryBackoff := &ExponentialBackoff{Minimum: supervisorBackoffMinimum, Maximum: supervisorBackoffMaximum}

	globTicker := time.NewTicker(s.GlobRefresh)
	for {
		var chunkToSend []*FileData
		if retryChunk != nil {
			// Retry case: after the retry timer, select retryChunk as the chunk to
			// send. Also monitor the other channels so we can do work in the
			// background if needed.
			select {
			case <-done:
				return
			case <-time.After(retryBackoff.Current()):
				chunkToSend = retryChunk
				retryBackoff.Next()
			case <-globTicker.C:
				s.startFileReaders(spooler.In, readers)
			}
		} else {
			select {
			case <-done:
				return
			case chunkToSend = <-spooler.Out:
				// got a chunk; we'll send it below
			case <-globTicker.C:
				s.startFileReaders(spooler.In, readers)
			}
		}

		if chunkToSend != nil {
			err := s.sendChunk(chunkToSend)
			if err != nil {
				logger.Report(err, grohl.Data{"msg": "failed to send chunk", "resolution": "retrying"})

				retryChunk = chunkToSend
			} else {
				retryChunk = nil
				retryBackoff.Reset()

				err = s.acknowledgeChunk(chunkToSend)
				if err != nil {
					// This is trickier; we've already sent the chunk to the remote system
					// successfully; retrying it would just create duplicates. The best
					// thing we can do is report the error and assume it's transient ...
					// that the next time we acknowledge a chunk, the snapshot will
					// succeed.
					logger.Report(err, grohl.Data{"msg": "failed to snapshot high water marks"})
				}
			}
		}
	}
}

func (s *Supervisor) sendChunk(chunk []*FileData) error {
	lines := make([]Data, 0, len(chunk))
	for _, fileData := range chunk {
		lines = append(lines, fileData.Data)
	}

	return s.Client.Send(lines)
}

func (s *Supervisor) acknowledgeChunk(chunk []*FileData) error {
	marks := make([]*HighWaterMark, 0, len(chunk))
	for _, fileData := range chunk {
		marks = append(marks, fileData.HighWaterMark)
	}

	return s.Snapshotter.SetHighWaterMarks(marks)
}

// startFileReaders globs the paths in each FileConfiguration, making sure
// a FileReader has been started for each one.
func (s *Supervisor) startFileReaders(spoolIn chan *FileData, readers *FileReaderCollection) {
	logger := grohl.NewContext(grohl.Data{"ns": "Supervisor", "fn": "startFileReaders"})

	for _, config := range s.Files {
		for _, path := range config.Paths {
			matches, err := filepath.Glob(path)
			if err != nil {
				logger.Report(err, grohl.Data{"path": path, "msg": "failed to glob", "resolution": "skipping path"})
				continue
			}

			for _, match := range matches {
				err = s.startFileReader(spoolIn, readers, match, config.Fields)
				if err != nil {
					logger.Report(err, grohl.Data{"path": "path", "match": match, "msg": "failed to start reader", "resolution": "skipping file"})
				}
			}
		}
	}
}

// startFileReader starts an individual file reader at a given path, if one
// isn't already running.
func (s *Supervisor) startFileReader(spoolIn chan *FileData, readers *FileReaderCollection, filePath string, fields map[string]string) error {
	if readers.Get(filePath) != nil {
		// There's already a reader for this file path. No need to do anything
		// further.
		return nil
	}

	highWaterMark, err := s.Snapshotter.HighWaterMark(filePath)
	if err != nil {
		return err
	}

	file, err := os.Open(filePath)
	if err != nil {
		return err
	}

	_, err = file.Seek(highWaterMark.Position, os.SEEK_SET)
	if err != nil {
		file.Close()
		return err
	}

	reader := &FileReader{File: file, Fields: fields}
	readers.Set(filePath, reader)
	go func() {
		s.runFileReader(spoolIn, reader)

		// When the reader is deleted from the collection, it's eligible to be
		// recreated when glob runs again.
		readers.Delete(filePath)
	}()

	return nil
}

// runFileReader reads from a FileReader until EOF is reached
func (s *Supervisor) runFileReader(spoolIn chan *FileData, reader *FileReader) {
	logger := grohl.NewContext(grohl.Data{"ns": "Supervisor", "fn": "runFileReader", "file": reader.File.Name()})

	// Track the "last position" that has been sent to the spool channel. If we
	// encounter an error, we want to make sure that position has been
	// snapshotted before we exit. Otherwise, a new file reader might be created
	// and repeat log lines.
	lastPosition := reader.Position()

	// Records the last time we receive an EOF; if we keep receiving an EOF,
	// we'll eventually exit.
	lastEOF := time.Unix(0, 0)

	for {
		fileData, err := reader.ReadLine()
		if err == io.EOF {
			if lastEOF.IsZero() {
				// Our first EOF: record it
				lastEOF = time.Now()
			} else if time.Since(lastEOF) >= supervisorEOFTimeout {
				logger.Log(grohl.Data{"status": "EOF", "resolution": "closing file"})
				break
			} else {
				// Wait a little while before trying to read from this file again
				<-time.After(5 * time.Second)
			}
		} else if err != nil {
			logger.Report(err, grohl.Data{"msg": "failed to completely read file", "resolution": "closing file"})
			break
		} else {
			lastEOF = time.Unix(0, 0)

			spoolIn <- fileData
			lastPosition = reader.Position()
		}
	}

	// Wait until our last position has been snapshotted
	for {
		highWaterMark, err := s.Snapshotter.HighWaterMark(reader.File.Name())
		if err != nil {
			logger.Report(err, grohl.Data{"msg": "failed to read high water mark", "resolution": "retrying"})
		} else if highWaterMark.Position >= lastPosition {
			// Done! We can exit cleanly now.
			break
		}

		// Try again in a second
		<-time.After(1 * time.Second)
	}
}
<commit_msg>Uses time zero value<commit_after>package main

import (
	"github.com/technoweenie/grohl"
	"io"
	"os"
	"path/filepath"
	"time"
)

type Supervisor struct {
	Files []FileConfiguration
	Client
	Snapshotter
	SpoolSize    int
	SpoolTimeout time.Duration

	// How frequently to glob for new files that may have appeared
	GlobRefresh time.Duration
}

const (
	supervisorBackoffMinimum = 500 * time.Millisecond
	supervisorBackoffMaximum = 5 * time.Second

	supervisorEOFTimeout = 5 * time.Minute
)

// Pulls the entire program together. Connects file readers to a spooler to
// a client, snapshotting progress after a successful acknowledgement from
// the server.
//
// To stop the supervisor, send a message to the done channel.
func (s *Supervisor) Serve(done chan interface{}) {
	logger := grohl.NewContext(grohl.Data{"ns": "Supervisor"})

	spooler := NewSpooler(s.SpoolSize, s.SpoolTimeout)
	go spooler.Spool()
	defer func() { close(spooler.In) }()

	readers := new(FileReaderCollection)
	s.startFileReaders(spooler.In, readers)

	// In the case that a chunk fails, we retry it by setting it as the
	// retryChunk.  We keep retrying the chunk until it sends correctly, then
	// move on to the normal queues.
	var retryChunk []*FileData
	retryBackoff := &ExponentialBackoff{Minimum: supervisorBackoffMinimum, Maximum: supervisorBackoffMaximum}

	globTicker := time.NewTicker(s.GlobRefresh)
	for {
		var chunkToSend []*FileData
		if retryChunk != nil {
			// Retry case: after the retry timer, select retryChunk as the chunk to
			// send. Also monitor the other channels so we can do work in the
			// background if needed.
			select {
			case <-done:
				return
			case <-time.After(retryBackoff.Current()):
				chunkToSend = retryChunk
				retryBackoff.Next()
			case <-globTicker.C:
				s.startFileReaders(spooler.In, readers)
			}
		} else {
			select {
			case <-done:
				return
			case chunkToSend = <-spooler.Out:
				// got a chunk; we'll send it below
			case <-globTicker.C:
				s.startFileReaders(spooler.In, readers)
			}
		}

		if chunkToSend != nil {
			err := s.sendChunk(chunkToSend)
			if err != nil {
				logger.Report(err, grohl.Data{"msg": "failed to send chunk", "resolution": "retrying"})

				retryChunk = chunkToSend
			} else {
				retryChunk = nil
				retryBackoff.Reset()

				err = s.acknowledgeChunk(chunkToSend)
				if err != nil {
					// This is trickier; we've already sent the chunk to the remote system
					// successfully; retrying it would just create duplicates. The best
					// thing we can do is report the error and assume it's transient ...
					// that the next time we acknowledge a chunk, the snapshot will
					// succeed.
					logger.Report(err, grohl.Data{"msg": "failed to snapshot high water marks"})
				}
			}
		}
	}
}

func (s *Supervisor) sendChunk(chunk []*FileData) error {
	lines := make([]Data, 0, len(chunk))
	for _, fileData := range chunk {
		lines = append(lines, fileData.Data)
	}

	return s.Client.Send(lines)
}

func (s *Supervisor) acknowledgeChunk(chunk []*FileData) error {
	marks := make([]*HighWaterMark, 0, len(chunk))
	for _, fileData := range chunk {
		marks = append(marks, fileData.HighWaterMark)
	}

	return s.Snapshotter.SetHighWaterMarks(marks)
}

// startFileReaders globs the paths in each FileConfiguration, making sure
// a FileReader has been started for each one.
func (s *Supervisor) startFileReaders(spoolIn chan *FileData, readers *FileReaderCollection) {
	logger := grohl.NewContext(grohl.Data{"ns": "Supervisor", "fn": "startFileReaders"})

	for _, config := range s.Files {
		for _, path := range config.Paths {
			matches, err := filepath.Glob(path)
			if err != nil {
				logger.Report(err, grohl.Data{"path": path, "msg": "failed to glob", "resolution": "skipping path"})
				continue
			}

			for _, match := range matches {
				err = s.startFileReader(spoolIn, readers, match, config.Fields)
				if err != nil {
					logger.Report(err, grohl.Data{"path": "path", "match": match, "msg": "failed to start reader", "resolution": "skipping file"})
				}
			}
		}
	}
}

// startFileReader starts an individual file reader at a given path, if one
// isn't already running.
func (s *Supervisor) startFileReader(spoolIn chan *FileData, readers *FileReaderCollection, filePath string, fields map[string]string) error {
	if readers.Get(filePath) != nil {
		// There's already a reader for this file path. No need to do anything
		// further.
		return nil
	}

	highWaterMark, err := s.Snapshotter.HighWaterMark(filePath)
	if err != nil {
		return err
	}

	file, err := os.Open(filePath)
	if err != nil {
		return err
	}

	_, err = file.Seek(highWaterMark.Position, os.SEEK_SET)
	if err != nil {
		file.Close()
		return err
	}

	reader := &FileReader{File: file, Fields: fields}
	readers.Set(filePath, reader)
	go func() {
		s.runFileReader(spoolIn, reader)

		// When the reader is deleted from the collection, it's eligible to be
		// recreated when glob runs again.
		readers.Delete(filePath)
	}()

	return nil
}

// runFileReader reads from a FileReader until EOF is reached
func (s *Supervisor) runFileReader(spoolIn chan *FileData, reader *FileReader) {
	logger := grohl.NewContext(grohl.Data{"ns": "Supervisor", "fn": "runFileReader", "file": reader.File.Name()})
	logger.Log(grohl.Data{"status": "opened"})

	// Track the "last position" that has been sent to the spool channel. If we
	// encounter an error, we want to make sure that position has been
	// snapshotted before we exit. Otherwise, a new file reader might be created
	// and repeat log lines.
	lastPosition := reader.Position()

	// Records the last time we receive an EOF; if we keep receiving an EOF,
	// we'll eventually exit.
	lastEOF := time.Time{}

	for {
		fileData, err := reader.ReadLine()
		if err == io.EOF {
			if lastEOF.IsZero() {
				// Our first EOF: record it
				lastEOF = time.Now()
			} else if time.Since(lastEOF) >= supervisorEOFTimeout {
				logger.Log(grohl.Data{"status": "EOF", "resolution": "closing file"})
				break
			} else {
				// Wait a little while before trying to read from this file again
				<-time.After(5 * time.Second)
			}
		} else if err != nil {
			logger.Report(err, grohl.Data{"msg": "failed to completely read file", "resolution": "closing file"})
			break
		} else {
			lastEOF = time.Time{}

			spoolIn <- fileData
			lastPosition = reader.Position()
		}
	}

	// Wait until our last position has been snapshotted
	for {
		highWaterMark, err := s.Snapshotter.HighWaterMark(reader.File.Name())
		if err != nil {
			logger.Report(err, grohl.Data{"msg": "failed to read high water mark", "resolution": "retrying"})
		} else if highWaterMark.Position >= lastPosition {
			// Done! We can exit cleanly now.
			break
		}

		// Try again in a second
		<-time.After(1 * time.Second)
	}
}
<|endoftext|>


<commit_before>/*
Package supervisor provides supervisor trees for Go applications.

This package is a clean reimplementation of github.com/thejerf/suture, aiming
to be more Go idiomatic, thus less Erlang-like.

It is built on top of context package, with all of its advantages, namely the
possibility trickle down context-related values and cancellation signals.

TheJerf's blog post about Suture is a very good and helpful read to understand
how this package has been implemented.

http://www.jerf.org/iri/post/2930
*/
package supervisor // import "cirello.io/supervisor"

import (
	"fmt"
	"log"
	"math"
	"sync"
	"time"

	"golang.org/x/net/context"
)

// Service is the public interface expected by a Supervisor.
//
// This will be internally named after the result of fmt.Stringer, if available.
// Otherwise it will going to use an internal representation for the service
// name.
type Service interface {
	// Serve is called by a Supervisor to start the service. It expects the
	// service to honor the passed context and its lifetime. Observe
	// <-ctx.Done() and ctx.Err(). If the service is stopped by anything
	// but the Supervisor, it will get started again. Be careful with shared
	// state among restarts.
	Serve(ctx context.Context)
}

// Supervisor is the basic datastructure responsible for offering a supervisor
// tree. It implements Service, therefore it can be nested if necessary. When
// passing the Supervisor around, remind to do it as reference (&supervisor).
type Supervisor struct {
	// Name for this supervisor tree, used for logging.
	Name string

	// FailureDecay is the timespan on which the current failure count will
	// be halved.
	FailureDecay float64

	// FailureThreshold is the maximum accepted number of failures, after
	// decay adjustment, that shall trigger the back-off wait.
	FailureThreshold float64

	// Backoff is the wait duration when hit threshold.
	Backoff time.Duration

	// Log is a replaceable function used for overall logging
	Log func(string)

	ready sync.Once

	startedMu sync.Mutex
	started   bool

	addedService    chan struct{}
	startedServices chan struct{}
	stoppedService  chan struct{}

	servicesMu sync.Mutex
	services   map[string]Service

	cancelationsMu sync.Mutex
	cancelations   map[string]context.CancelFunc

	backoffMu sync.Mutex
	backoff   map[string]*backoff

	runningMu sync.Mutex
	running   int
}

func (s *Supervisor) String() string {
	return s.Name
}

func (s *Supervisor) prepare() {
	s.ready.Do(func() {
		if s.Name == "" {
			s.Name = "supervisor"
		}
		s.addedService = make(chan struct{}, 1)
		s.backoff = make(map[string]*backoff)
		s.cancelations = make(map[string]context.CancelFunc)
		s.services = make(map[string]Service)
		s.startedServices = make(chan struct{}, 1)
		s.stoppedService = make(chan struct{}, 1)

		if s.Log == nil {
			s.Log = func(str string) {
				log.Println(s.Name, ":", str)
			}
		}
		if s.FailureDecay == 0 {
			s.FailureDecay = 30
		}
		if s.FailureThreshold == 0 {
			s.FailureThreshold = 5
		}
		if s.Backoff == 0 {
			s.Backoff = 15 * time.Second
		}
	})
}

// Add inserts into the Supervisor tree a new service. If the Supervisor is
// already started, it will start it automatically.
func (s *Supervisor) Add(service Service) {
	s.prepare()

	name := fmt.Sprintf("%s", service)

	s.servicesMu.Lock()
	s.backoffMu.Lock()
	s.backoff[name] = &backoff{}
	s.services[name] = service
	s.backoffMu.Unlock()
	s.servicesMu.Unlock()

	select {
	case s.addedService <- struct{}{}:
	default:
	}
}

// Remove stops the service in the Supervisor tree and remove from it.
func (s *Supervisor) Remove(name string) {
	s.prepare()

	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()
	if _, ok := s.services[name]; !ok {
		return
	}

	s.cancelationsMu.Lock()
	defer s.cancelationsMu.Unlock()
	if c, ok := s.cancelations[name]; ok {
		delete(s.cancelations, name)
		c()
	}
}

// Serve starts the Supervisor tree. It can be started only once at a time. If
// stopped (canceled), it can be restarted.
func (s *Supervisor) Serve(ctx context.Context) {
	s.prepare()

	select {
	case s.addedService <- struct{}{}:
	default:
	}

	s.startedMu.Lock()
	if !s.started {
		s.started = true
		s.startedMu.Unlock()

		s.serve(ctx)

		s.startedMu.Lock()
		s.started = false
	}
	s.startedMu.Unlock()
}

// Services return a list of services
func (s *Supervisor) Services() map[string]Service {
	svclist := make(map[string]Service)
	s.servicesMu.Lock()
	for k, v := range s.services {
		svclist[k] = v
	}
	s.servicesMu.Unlock()
	return svclist
}

// Cancelations return a list of services names and their cancellation calls
func (s *Supervisor) Cancelations() map[string]context.CancelFunc {
	svclist := make(map[string]context.CancelFunc)
	s.cancelationsMu.Lock()
	for k, v := range s.cancelations {
		svclist[k] = v
	}
	s.cancelationsMu.Unlock()
	return svclist
}

func (s *Supervisor) serve(ctx context.Context) {
	go func(ctx context.Context) {
		for {
			select {
			case <-s.addedService:
				s.startServices(ctx)
				select {
				case s.startedServices <- struct{}{}:
				default:
				}

			case <-ctx.Done():
				return
			}
		}
	}(ctx)

	<-ctx.Done()
	s.cancelationsMu.Lock()
	s.cancelations = make(map[string]context.CancelFunc)
	s.cancelationsMu.Unlock()

	for range s.stoppedService {
		s.runningMu.Lock()
		r := s.running
		s.runningMu.Unlock()
		if r == 0 {
			return
		}
	}
}

func (s *Supervisor) startServices(ctx context.Context) {
	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()

	var wg sync.WaitGroup

	for name, svc := range s.services {
		s.cancelationsMu.Lock()
		_, ok := s.cancelations[name]
		s.cancelationsMu.Unlock()
		if ok {
			continue
		}

		wg.Add(1)

		go func(name string, svc Service) {
			s.runningMu.Lock()
			s.running++
			s.runningMu.Unlock()
			wg.Done()
			for {
				retry := func() (retry bool) {
					select {
					case <-ctx.Done():
						return false
					default:
					}

					defer func() {
						if r := recover(); r != nil {
							s.Log(fmt.Sprint("trapped panic:", r))
							retry = true
						}
					}()

					c, cancel := context.WithCancel(ctx)
					s.cancelationsMu.Lock()
					s.cancelations[name] = cancel
					s.cancelationsMu.Unlock()
					svc.Serve(c)

					select {
					case <-ctx.Done():
						return false
					default:
						return true
					}
				}()
				if retry {
					s.Log(fmt.Sprintf("restarting %s", name))
					s.backoffMu.Lock()
					b := s.backoff[name]
					s.backoffMu.Unlock()
					b.wait(s.FailureDecay, s.FailureThreshold, s.Backoff)
					continue
				}

				s.runningMu.Lock()
				s.running--
				s.runningMu.Unlock()
				s.stoppedService <- struct{}{}
				return
			}
		}(name, svc)
	}
	wg.Wait()
}

type backoff struct {
	lastfail time.Time
	failures float64
}

func (b *backoff) wait(failureDecay float64, threshold float64, backoffDur time.Duration) {
	if b.lastfail.IsZero() {
		b.lastfail = time.Now()
		b.failures = 1.0
	} else {
		b.failures++
		intervals := time.Since(b.lastfail).Seconds() / failureDecay
		b.failures = b.failures*math.Pow(.5, intervals) + 1
	}

	if b.failures > threshold {
		time.Sleep(backoffDur)
	}
}
<commit_msg>refactor end of supervisor lifecyle<commit_after>/*
Package supervisor provides supervisor trees for Go applications.

This package is a clean reimplementation of github.com/thejerf/suture, aiming
to be more Go idiomatic, thus less Erlang-like.

It is built on top of context package, with all of its advantages, namely the
possibility trickle down context-related values and cancellation signals.

TheJerf's blog post about Suture is a very good and helpful read to understand
how this package has been implemented.

http://www.jerf.org/iri/post/2930
*/
package supervisor // import "cirello.io/supervisor"

import (
	"fmt"
	"log"
	"math"
	"sync"
	"time"

	"golang.org/x/net/context"
)

// Service is the public interface expected by a Supervisor.
//
// This will be internally named after the result of fmt.Stringer, if available.
// Otherwise it will going to use an internal representation for the service
// name.
type Service interface {
	// Serve is called by a Supervisor to start the service. It expects the
	// service to honor the passed context and its lifetime. Observe
	// <-ctx.Done() and ctx.Err(). If the service is stopped by anything
	// but the Supervisor, it will get started again. Be careful with shared
	// state among restarts.
	Serve(ctx context.Context)
}

// Supervisor is the basic datastructure responsible for offering a supervisor
// tree. It implements Service, therefore it can be nested if necessary. When
// passing the Supervisor around, remind to do it as reference (&supervisor).
type Supervisor struct {
	// Name for this supervisor tree, used for logging.
	Name string

	// FailureDecay is the timespan on which the current failure count will
	// be halved.
	FailureDecay float64

	// FailureThreshold is the maximum accepted number of failures, after
	// decay adjustment, that shall trigger the back-off wait.
	FailureThreshold float64

	// Backoff is the wait duration when hit threshold.
	Backoff time.Duration

	// Log is a replaceable function used for overall logging
	Log func(string)

	ready sync.Once

	startedMu sync.Mutex
	started   bool

	addedService    chan struct{}
	startedServices chan struct{}
	stoppedService  chan struct{}

	servicesMu sync.Mutex
	services   map[string]Service

	cancelationsMu sync.Mutex
	cancelations   map[string]context.CancelFunc

	backoffMu sync.Mutex
	backoff   map[string]*backoff

	runningMu sync.Mutex
	running   int
}

func (s *Supervisor) String() string {
	return s.Name
}

func (s *Supervisor) prepare() {
	s.ready.Do(func() {
		if s.Name == "" {
			s.Name = "supervisor"
		}
		s.addedService = make(chan struct{}, 1)
		s.backoff = make(map[string]*backoff)
		s.cancelations = make(map[string]context.CancelFunc)
		s.services = make(map[string]Service)
		s.startedServices = make(chan struct{}, 1)
		s.stoppedService = make(chan struct{}, 1)

		if s.Log == nil {
			s.Log = func(str string) {
				log.Println(s.Name, ":", str)
			}
		}
		if s.FailureDecay == 0 {
			s.FailureDecay = 30
		}
		if s.FailureThreshold == 0 {
			s.FailureThreshold = 5
		}
		if s.Backoff == 0 {
			s.Backoff = 15 * time.Second
		}
	})
}

// Add inserts into the Supervisor tree a new service. If the Supervisor is
// already started, it will start it automatically.
func (s *Supervisor) Add(service Service) {
	s.prepare()

	name := fmt.Sprintf("%s", service)

	s.servicesMu.Lock()
	s.backoffMu.Lock()
	s.backoff[name] = &backoff{}
	s.services[name] = service
	s.backoffMu.Unlock()
	s.servicesMu.Unlock()

	select {
	case s.addedService <- struct{}{}:
	default:
	}
}

// Remove stops the service in the Supervisor tree and remove from it.
func (s *Supervisor) Remove(name string) {
	s.prepare()

	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()
	if _, ok := s.services[name]; !ok {
		return
	}

	s.cancelationsMu.Lock()
	defer s.cancelationsMu.Unlock()
	if c, ok := s.cancelations[name]; ok {
		delete(s.cancelations, name)
		c()
	}
}

// Serve starts the Supervisor tree. It can be started only once at a time. If
// stopped (canceled), it can be restarted.
func (s *Supervisor) Serve(ctx context.Context) {
	s.prepare()

	select {
	case s.addedService <- struct{}{}:
	default:
	}

	s.startedMu.Lock()
	if !s.started {
		s.started = true
		s.startedMu.Unlock()

		s.serve(ctx)

		s.startedMu.Lock()
		s.started = false
	}
	s.startedMu.Unlock()
}

// Services return a list of services
func (s *Supervisor) Services() map[string]Service {
	svclist := make(map[string]Service)
	s.servicesMu.Lock()
	for k, v := range s.services {
		svclist[k] = v
	}
	s.servicesMu.Unlock()
	return svclist
}

// Cancelations return a list of services names and their cancellation calls
func (s *Supervisor) Cancelations() map[string]context.CancelFunc {
	svclist := make(map[string]context.CancelFunc)
	s.cancelationsMu.Lock()
	for k, v := range s.cancelations {
		svclist[k] = v
	}
	s.cancelationsMu.Unlock()
	return svclist
}

func (s *Supervisor) serve(ctx context.Context) {
	go func(ctx context.Context) {
		for {
			select {
			case <-s.addedService:
				s.startServices(ctx)
				select {
				case s.startedServices <- struct{}{}:
				default:
				}

			case <-ctx.Done():
				return
			}
		}
	}(ctx)

	<-ctx.Done()

	for range s.stoppedService {
		s.runningMu.Lock()
		r := s.running
		s.runningMu.Unlock()
		if r == 0 {
			break
		}
	}

	s.cancelationsMu.Lock()
	s.cancelations = make(map[string]context.CancelFunc)
	s.cancelationsMu.Unlock()
	return
}

func (s *Supervisor) startServices(ctx context.Context) {
	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()

	var wg sync.WaitGroup

	for name, svc := range s.services {
		s.cancelationsMu.Lock()
		_, ok := s.cancelations[name]
		s.cancelationsMu.Unlock()
		if ok {
			continue
		}

		wg.Add(1)

		go func(name string, svc Service) {
			s.runningMu.Lock()
			s.running++
			s.runningMu.Unlock()
			wg.Done()
			for {
				retry := func() (retry bool) {
					select {
					case <-ctx.Done():
						return false
					default:
					}

					defer func() {
						if r := recover(); r != nil {
							s.Log(fmt.Sprint("trapped panic:", r))
							retry = true
						}
					}()

					c, cancel := context.WithCancel(ctx)
					s.cancelationsMu.Lock()
					s.cancelations[name] = cancel
					s.cancelationsMu.Unlock()
					svc.Serve(c)

					select {
					case <-ctx.Done():
						return false
					default:
						return true
					}
				}()
				if retry {
					s.Log(fmt.Sprintf("restarting %s", name))
					s.backoffMu.Lock()
					b := s.backoff[name]
					s.backoffMu.Unlock()
					b.wait(s.FailureDecay, s.FailureThreshold, s.Backoff)
					continue
				}
				break
			}
			s.runningMu.Lock()
			s.running--
			s.runningMu.Unlock()
			s.stoppedService <- struct{}{}
			return
		}(name, svc)
	}
	wg.Wait()
}

type backoff struct {
	lastfail time.Time
	failures float64
}

func (b *backoff) wait(failureDecay float64, threshold float64, backoffDur time.Duration) {
	if b.lastfail.IsZero() {
		b.lastfail = time.Now()
		b.failures = 1.0
	} else {
		b.failures++
		intervals := time.Since(b.lastfail).Seconds() / failureDecay
		b.failures = b.failures*math.Pow(.5, intervals) + 1
	}

	if b.failures > threshold {
		time.Sleep(backoffDur)
	}
}
<|endoftext|>


<commit_before>package sync

import (
	"database/sql"
	"fmt"
	"radiusd/config"
)

var (
	acct  *sql.Stmt
	usage *sql.Stmt
)

func Init() error {
	var e error
	acct, e = config.DB.Prepare(
		`INSERT INTO
			accounting
		(user, date, bytes_in, bytes_out, hostname)
		VALUES (?, ?, ?, ?, ?)`,
	)
	if e != nil {
		return e
	}

	usage, e = config.DB.Prepare(
		`UPDATE
			user
		SET
			block_remaining = IF(CAST(block_remaining as SIGNED) - ? < 0, 0, block_remaining - ?)
		WHERE
			user = ?`,
	)
	return e
}

func SessionAcct(user string, date string, octetIn uint32, octetOut uint32, hostname string) error {
	res, e := acct.Exec(user, date, octetIn, octetOut, hostname)
	if e != nil {
		return e
	}
	affect, e := res.RowsAffected()
	if e != nil {
		return e
	}
	if affect != 1 {
		return fmt.Errorf(
			"Affect fail for user=%s",
			user,
		)
	}
	return nil
}

func UpdateRemaining(user string, remain uint32) error {
	if remain == 0 {
		return nil
	}

	res, e := usage.Exec(remain, remain, user)
	if e != nil {
		return e
	}
	affect, e := res.RowsAffected()
	if e != nil {
		return e
	}
	if affect != 1 {
		// Nothing changed, check if this behaviour is correct
		remain, e := checkRemain(user)
		if e != nil {
			return e
		}
		if !remain {
			return fmt.Errorf(
				"Affect fail for user=%s",
				user,
			)
		}
	}
	return nil
}

func checkRemain(user string) (bool, error) {
	var remain *int64
	e := config.DB.QueryRow(
		`SELECT
			block_remaining
		FROM
			user
		WHERE
			user = ?`,
		user,
	).Scan(remain)
	if remain == nil || *remain == 0 {
		return true, e
	}
	return false, e
}
<commit_msg>Bugfix. nullpointer<commit_after>package sync

import (
	"database/sql"
	"fmt"
	"radiusd/config"
)

var (
	acct  *sql.Stmt
	usage *sql.Stmt
)

func Init() error {
	var e error
	acct, e = config.DB.Prepare(
		`INSERT INTO
			accounting
		(user, date, bytes_in, bytes_out, hostname)
		VALUES (?, ?, ?, ?, ?)`,
	)
	if e != nil {
		return e
	}

	usage, e = config.DB.Prepare(
		`UPDATE
			user
		SET
			block_remaining = IF(CAST(block_remaining as SIGNED) - ? < 0, 0, block_remaining - ?)
		WHERE
			user = ?`,
	)
	return e
}

func SessionAcct(user string, date string, octetIn uint32, octetOut uint32, hostname string) error {
	res, e := acct.Exec(user, date, octetIn, octetOut, hostname)
	if e != nil {
		return e
	}
	affect, e := res.RowsAffected()
	if e != nil {
		return e
	}
	if affect != 1 {
		return fmt.Errorf(
			"Affect fail for user=%s",
			user,
		)
	}
	return nil
}

func UpdateRemaining(user string, remain uint32) error {
	if remain == 0 {
		return nil
	}

	res, e := usage.Exec(remain, remain, user)
	if e != nil {
		return e
	}
	affect, e := res.RowsAffected()
	if e != nil {
		return e
	}
	if affect != 1 {
		// Nothing changed, check if this behaviour is correct
		remain, e := checkRemain(user)
		if e != nil {
			return e
		}
		if !remain {
			return fmt.Errorf(
				"Affect fail for user=%s",
				user,
			)
		}
	}
	return nil
}

func checkRemain(user string) (bool, error) {
	var remain *int64
	n := int64(0)
	remain = &n

	e := config.DB.QueryRow(
		`SELECT
			block_remaining
		FROM
			user
		WHERE
			user = ?`,
		user,
	).Scan(remain)
	if remain == nil || *remain == 0 {
		return true, e
	}
	return false, e
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package acmeorders

import (
	"context"
	"encoding/json"
	"fmt"
	"hash/fnv"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/jetstack/cert-manager/pkg/acme"
	acmecl "github.com/jetstack/cert-manager/pkg/acme/client"
	"github.com/jetstack/cert-manager/pkg/api/util"
	cmacme "github.com/jetstack/cert-manager/pkg/apis/acme/v1"
	cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
	"github.com/jetstack/cert-manager/pkg/controller/acmeorders/selectors"
	logf "github.com/jetstack/cert-manager/pkg/logs"
)

var (
	orderGvk = cmacme.SchemeGroupVersion.WithKind("Order")
)

func buildRequiredChallenges(ctx context.Context, cl acmecl.Interface, issuer cmapi.GenericIssuer, o *cmacme.Order) ([]cmacme.Challenge, error) {
	chs := make([]cmacme.Challenge, 0)
	for _, a := range o.Status.Authorizations {
		if a.InitialState == cmacme.Valid {
			wc := false
			if a.Wildcard != nil {
				wc = *a.Wildcard
			}
			logf.FromContext(ctx).V(logf.DebugLevel).Info("Authorization already valid, not creating Challenge resource", "identifier", a.Identifier, "is_wildcard", wc)
			continue
		}
		ch, err := buildChallenge(ctx, cl, issuer, o, a)
		if err != nil {
			return nil, err
		}
		chs = append(chs, *ch)
	}
	return chs, nil
}

func buildChallenge(ctx context.Context, cl acmecl.Interface, issuer cmapi.GenericIssuer, o *cmacme.Order, authz cmacme.ACMEAuthorization) (*cmacme.Challenge, error) {
	chSpec, err := challengeSpecForAuthorization(ctx, cl, issuer, o, authz)
	if err != nil {
		// TODO: in this case, we should probably not return the error as it's
		//  unlikely we can make it succeed by retrying.
		return nil, err
	}

	chName, err := util.ComputeName(o.Name, chSpec)
	if err != nil {
		return nil, err
	}

	return &cmacme.Challenge{
		ObjectMeta: metav1.ObjectMeta{
			Name:            chName,
			Namespace:       o.Namespace,
			OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(o, orderGvk)},
			Finalizers:      []string{cmacme.ACMEFinalizer},
		},
		Spec: *chSpec,
	}, nil
}

func hashChallenge(spec cmacme.ChallengeSpec) (uint32, error) {
	specBytes, err := json.Marshal(spec)
	if err != nil {
		return 0, err
	}

	hashF := fnv.New32()
	_, err = hashF.Write(specBytes)
	if err != nil {
		return 0, err
	}

	return hashF.Sum32(), nil
}

func challengeSpecForAuthorization(ctx context.Context, cl acmecl.Interface, issuer cmapi.GenericIssuer, o *cmacme.Order, authz cmacme.ACMEAuthorization) (*cmacme.ChallengeSpec, error) {
	log := logf.FromContext(ctx, "challengeSpecForAuthorization")
	dbg := log.V(logf.DebugLevel)

	// 1. fetch solvers from issuer
	solvers := issuer.GetSpec().ACME.Solvers

	wc := false
	if authz.Wildcard != nil {
		wc = *authz.Wildcard
	}
	domainToFind := authz.Identifier
	if wc {
		domainToFind = "*." + domainToFind
	}

	var selectedSolver *cmacme.ACMEChallengeSolver
	var selectedChallenge *cmacme.ACMEChallenge
	selectedNumLabelsMatch := 0
	selectedNumDNSNamesMatch := 0
	selectedNumDNSZonesMatch := 0

	challengeForSolver := func(solver *cmacme.ACMEChallengeSolver) *cmacme.ACMEChallenge {
		for _, ch := range authz.Challenges {
			switch {
			case ch.Type == "http-01" && solver.HTTP01 != nil:
				return &ch
			case ch.Type == "dns-01" && solver.DNS01 != nil:
				return &ch
			}
		}
		return nil
	}

	// 2. filter solvers to only those that matchLabels
	for _, cfg := range solvers {
		acmech := challengeForSolver(&cfg)
		if acmech == nil {
			dbg.Info("cannot use solver as the ACME authorization does not allow solvers of this type")
			continue
		}

		if cfg.Selector == nil {
			if selectedSolver != nil {
				dbg.Info("not selecting solver as previously selected solver has a just as or more specific selector")
				continue
			}
			dbg.Info("selecting solver due to match all selector and no previously selected solver")
			selectedSolver = cfg.DeepCopy()
			selectedChallenge = acmech
			continue
		}

		labelsMatch, numLabelsMatch := selectors.Labels(*cfg.Selector).Matches(o.ObjectMeta, domainToFind)
		dnsNamesMatch, numDNSNamesMatch := selectors.DNSNames(*cfg.Selector).Matches(o.ObjectMeta, domainToFind)
		dnsZonesMatch, numDNSZonesMatch := selectors.DNSZones(*cfg.Selector).Matches(o.ObjectMeta, domainToFind)

		if !labelsMatch || !dnsNamesMatch || !dnsZonesMatch {
			dbg.Info("not selecting solver", "labels_match", labelsMatch, "dnsnames_match", dnsNamesMatch, "dnszones_match", dnsZonesMatch)
			continue
		}

		dbg.Info("selector matches")

		selectSolver := func() {
			selectedSolver = cfg.DeepCopy()
			selectedChallenge = acmech
			selectedNumLabelsMatch = numLabelsMatch
			selectedNumDNSNamesMatch = numDNSNamesMatch
			selectedNumDNSZonesMatch = numDNSZonesMatch
		}

		if selectedSolver == nil {
			dbg.Info("selecting solver as there is no previously selected solver")
			selectSolver()
			continue
		}

		dbg.Info("determining whether this match is more significant than last")

		// because we don't count multiple dnsName matches as extra 'weight'
		// in the selection process, we normalise the numDNSNamesMatch vars
		// to be either 1 or 0 (i.e. true or false)
		selectedHasMatchingDNSNames := selectedNumDNSNamesMatch > 0
		hasMatchingDNSNames := numDNSNamesMatch > 0

		// dnsName selectors have the highest precedence, so check them first
		switch {
		case !selectedHasMatchingDNSNames && hasMatchingDNSNames:
			dbg.Info("selecting solver as this solver has matching DNS names and the previous one does not")
			selectSolver()
			continue
		case selectedHasMatchingDNSNames && !hasMatchingDNSNames:
			dbg.Info("not selecting solver as the previous one has matching DNS names and this one does not")
			continue
		case !selectedHasMatchingDNSNames && !hasMatchingDNSNames:
			dbg.Info("solver does not have any matching DNS names, checking dnsZones")
			// check zones
		case selectedHasMatchingDNSNames && hasMatchingDNSNames:
			dbg.Info("both this solver and the previously selected one matches dnsNames, comparing zones")
			if numDNSZonesMatch > selectedNumDNSZonesMatch {
				dbg.Info("selecting solver as this one has a more specific dnsZone match than the previously selected one")
				selectSolver()
				continue
			}
			if selectedNumDNSZonesMatch > numDNSZonesMatch {
				dbg.Info("not selecting this solver as the previously selected one has a more specific dnsZone match")
				continue
			}
			dbg.Info("both this solver and the previously selected one match dnsZones, comparing labels")
			// choose the one with the most labels
			if numLabelsMatch > selectedNumLabelsMatch {
				dbg.Info("selecting solver as this one has more labels than the previously selected one")
				selectSolver()
				continue
			}
			dbg.Info("not selecting this solver as previous one has either the same number of or more labels")
			continue
		}

		selectedHasMatchingDNSZones := selectedNumDNSZonesMatch > 0
		hasMatchingDNSZones := numDNSZonesMatch > 0

		switch {
		case !selectedHasMatchingDNSZones && hasMatchingDNSZones:
			dbg.Info("selecting solver as this solver has matching DNS zones and the previous one does not")
			selectSolver()
			continue
		case selectedHasMatchingDNSZones && !hasMatchingDNSZones:
			dbg.Info("not selecting solver as the previous one has matching DNS zones and this one does not")
			continue
		case !selectedHasMatchingDNSZones && !hasMatchingDNSZones:
			dbg.Info("solver does not have any matching DNS zones, checking labels")
			// check labels
		case selectedHasMatchingDNSZones && hasMatchingDNSZones:
			dbg.Info("both this solver and the previously selected one matches dnsZones")
			dbg.Info("comparing number of matching domain segments")
			// choose the one with the most matching DNS zone segments
			if numDNSZonesMatch > selectedNumDNSZonesMatch {
				dbg.Info("selecting solver because this one has more matching DNS zone segments")
				selectSolver()
				continue
			}
			if selectedNumDNSZonesMatch > numDNSZonesMatch {
				dbg.Info("not selecting solver because previous one has more matching DNS zone segments")
				continue
			}
			// choose the one with the most labels
			if numLabelsMatch > selectedNumLabelsMatch {
				dbg.Info("selecting solver because this one has more labels than the previous one")
				selectSolver()
				continue
			}
			dbg.Info("not selecting solver as this one's number of matching labels is equal to or less than the last one")
			continue
		}

		if numLabelsMatch > selectedNumLabelsMatch {
			dbg.Info("selecting solver as this one has more labels than the last one")
			selectSolver()
			continue
		}

		dbg.Info("not selecting solver as this one's number of matching labels is equal to or less than the last one (reached end of loop)")
		// if we get here, the number of matches is less than or equal so we
		// fallback to choosing the first in the list
	}

	if selectedSolver == nil || selectedChallenge == nil {
		return nil, fmt.Errorf("no configured challenge solvers can be used for this challenge")
	}

	// It should never be possible for this case to be hit as earlier in this
	// method we already assert that the challenge type is one of 'http-01'
	// or 'dns-01'.
	chType, err := challengeType(selectedChallenge.Type)
	if err != nil {
		return nil, err
	}

	key, err := keyForChallenge(cl, selectedChallenge.Token, chType)
	if err != nil {
		return nil, err
	}

	// 4. handle overriding the HTTP01 ingress class and name fields using the
	//    ACMECertificateHTTP01IngressNameOverride & Class annotations
	if err := applyIngressParameterAnnotationOverrides(o, selectedSolver); err != nil {
		return nil, err
	}

	// 5. construct Challenge resource with spec.solver field set
	return &cmacme.ChallengeSpec{
		AuthorizationURL: authz.URL,
		Type:             chType,
		URL:              selectedChallenge.URL,
		DNSName:          authz.Identifier,
		Token:            selectedChallenge.Token,
		Key:              key,
		// selectedSolver cannot be nil due to the check above.
		Solver:    *selectedSolver,
		Wildcard:  wc,
		IssuerRef: o.Spec.IssuerRef,
	}, nil
}

func challengeType(t string) (cmacme.ACMEChallengeType, error) {
	switch t {
	case "http-01":
		return cmacme.ACMEChallengeTypeHTTP01, nil
	case "dns-01":
		return cmacme.ACMEChallengeTypeDNS01, nil
	default:
		return "", fmt.Errorf("unsupported challenge type: %v", t)
	}
}

func applyIngressParameterAnnotationOverrides(o *cmacme.Order, s *cmacme.ACMEChallengeSolver) error {
	if s.HTTP01 == nil || s.HTTP01.Ingress == nil || o.Annotations == nil {
		return nil
	}

	manualIngressName, hasManualIngressName := o.Annotations[cmacme.ACMECertificateHTTP01IngressNameOverride]
	manualIngressClass, hasManualIngressClass := o.Annotations[cmacme.ACMECertificateHTTP01IngressClassOverride]
	// don't allow both override annotations to be specified at once
	if hasManualIngressName && hasManualIngressClass {
		return fmt.Errorf("both ingress name and ingress class overrides specified - only one may be specified at a time")
	}
	// if an override annotation is specified, clear out the existing solver
	// config
	if hasManualIngressClass || hasManualIngressName {
		s.HTTP01.Ingress.Class = nil
		s.HTTP01.Ingress.Name = ""
	}
	if hasManualIngressName {
		s.HTTP01.Ingress.Name = manualIngressName
	}
	if hasManualIngressClass {
		s.HTTP01.Ingress.Class = &manualIngressClass
	}
	return nil
}

func keyForChallenge(cl acmecl.Interface, token string, chType cmacme.ACMEChallengeType) (string, error) {
	switch chType {
	case cmacme.ACMEChallengeTypeHTTP01:
		return cl.HTTP01ChallengeResponse(token)
	case cmacme.ACMEChallengeTypeDNS01:
		return cl.DNS01ChallengeRecord(token)
	default:
		return "", fmt.Errorf("unsupported challenge type: %v", chType)
	}
}

func anyChallengesFailed(chs []*cmacme.Challenge) bool {
	for _, ch := range chs {
		if acme.IsFailureState(ch.Status.State) {
			return true
		}
	}
	return false
}

func allChallengesFinal(chs []*cmacme.Challenge) bool {
	for _, ch := range chs {
		if !acme.IsFinalState(ch.Status.State) {
			return false
		}
	}
	return true
}
<commit_msg>spelling: normalize<commit_after>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package acmeorders

import (
	"context"
	"encoding/json"
	"fmt"
	"hash/fnv"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/jetstack/cert-manager/pkg/acme"
	acmecl "github.com/jetstack/cert-manager/pkg/acme/client"
	"github.com/jetstack/cert-manager/pkg/api/util"
	cmacme "github.com/jetstack/cert-manager/pkg/apis/acme/v1"
	cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
	"github.com/jetstack/cert-manager/pkg/controller/acmeorders/selectors"
	logf "github.com/jetstack/cert-manager/pkg/logs"
)

var (
	orderGvk = cmacme.SchemeGroupVersion.WithKind("Order")
)

func buildRequiredChallenges(ctx context.Context, cl acmecl.Interface, issuer cmapi.GenericIssuer, o *cmacme.Order) ([]cmacme.Challenge, error) {
	chs := make([]cmacme.Challenge, 0)
	for _, a := range o.Status.Authorizations {
		if a.InitialState == cmacme.Valid {
			wc := false
			if a.Wildcard != nil {
				wc = *a.Wildcard
			}
			logf.FromContext(ctx).V(logf.DebugLevel).Info("Authorization already valid, not creating Challenge resource", "identifier", a.Identifier, "is_wildcard", wc)
			continue
		}
		ch, err := buildChallenge(ctx, cl, issuer, o, a)
		if err != nil {
			return nil, err
		}
		chs = append(chs, *ch)
	}
	return chs, nil
}

func buildChallenge(ctx context.Context, cl acmecl.Interface, issuer cmapi.GenericIssuer, o *cmacme.Order, authz cmacme.ACMEAuthorization) (*cmacme.Challenge, error) {
	chSpec, err := challengeSpecForAuthorization(ctx, cl, issuer, o, authz)
	if err != nil {
		// TODO: in this case, we should probably not return the error as it's
		//  unlikely we can make it succeed by retrying.
		return nil, err
	}

	chName, err := util.ComputeName(o.Name, chSpec)
	if err != nil {
		return nil, err
	}

	return &cmacme.Challenge{
		ObjectMeta: metav1.ObjectMeta{
			Name:            chName,
			Namespace:       o.Namespace,
			OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(o, orderGvk)},
			Finalizers:      []string{cmacme.ACMEFinalizer},
		},
		Spec: *chSpec,
	}, nil
}

func hashChallenge(spec cmacme.ChallengeSpec) (uint32, error) {
	specBytes, err := json.Marshal(spec)
	if err != nil {
		return 0, err
	}

	hashF := fnv.New32()
	_, err = hashF.Write(specBytes)
	if err != nil {
		return 0, err
	}

	return hashF.Sum32(), nil
}

func challengeSpecForAuthorization(ctx context.Context, cl acmecl.Interface, issuer cmapi.GenericIssuer, o *cmacme.Order, authz cmacme.ACMEAuthorization) (*cmacme.ChallengeSpec, error) {
	log := logf.FromContext(ctx, "challengeSpecForAuthorization")
	dbg := log.V(logf.DebugLevel)

	// 1. fetch solvers from issuer
	solvers := issuer.GetSpec().ACME.Solvers

	wc := false
	if authz.Wildcard != nil {
		wc = *authz.Wildcard
	}
	domainToFind := authz.Identifier
	if wc {
		domainToFind = "*." + domainToFind
	}

	var selectedSolver *cmacme.ACMEChallengeSolver
	var selectedChallenge *cmacme.ACMEChallenge
	selectedNumLabelsMatch := 0
	selectedNumDNSNamesMatch := 0
	selectedNumDNSZonesMatch := 0

	challengeForSolver := func(solver *cmacme.ACMEChallengeSolver) *cmacme.ACMEChallenge {
		for _, ch := range authz.Challenges {
			switch {
			case ch.Type == "http-01" && solver.HTTP01 != nil:
				return &ch
			case ch.Type == "dns-01" && solver.DNS01 != nil:
				return &ch
			}
		}
		return nil
	}

	// 2. filter solvers to only those that matchLabels
	for _, cfg := range solvers {
		acmech := challengeForSolver(&cfg)
		if acmech == nil {
			dbg.Info("cannot use solver as the ACME authorization does not allow solvers of this type")
			continue
		}

		if cfg.Selector == nil {
			if selectedSolver != nil {
				dbg.Info("not selecting solver as previously selected solver has a just as or more specific selector")
				continue
			}
			dbg.Info("selecting solver due to match all selector and no previously selected solver")
			selectedSolver = cfg.DeepCopy()
			selectedChallenge = acmech
			continue
		}

		labelsMatch, numLabelsMatch := selectors.Labels(*cfg.Selector).Matches(o.ObjectMeta, domainToFind)
		dnsNamesMatch, numDNSNamesMatch := selectors.DNSNames(*cfg.Selector).Matches(o.ObjectMeta, domainToFind)
		dnsZonesMatch, numDNSZonesMatch := selectors.DNSZones(*cfg.Selector).Matches(o.ObjectMeta, domainToFind)

		if !labelsMatch || !dnsNamesMatch || !dnsZonesMatch {
			dbg.Info("not selecting solver", "labels_match", labelsMatch, "dnsnames_match", dnsNamesMatch, "dnszones_match", dnsZonesMatch)
			continue
		}

		dbg.Info("selector matches")

		selectSolver := func() {
			selectedSolver = cfg.DeepCopy()
			selectedChallenge = acmech
			selectedNumLabelsMatch = numLabelsMatch
			selectedNumDNSNamesMatch = numDNSNamesMatch
			selectedNumDNSZonesMatch = numDNSZonesMatch
		}

		if selectedSolver == nil {
			dbg.Info("selecting solver as there is no previously selected solver")
			selectSolver()
			continue
		}

		dbg.Info("determining whether this match is more significant than last")

		// because we don't count multiple dnsName matches as extra 'weight'
		// in the selection process, we normalize the numDNSNamesMatch vars
		// to be either 1 or 0 (i.e. true or false)
		selectedHasMatchingDNSNames := selectedNumDNSNamesMatch > 0
		hasMatchingDNSNames := numDNSNamesMatch > 0

		// dnsName selectors have the highest precedence, so check them first
		switch {
		case !selectedHasMatchingDNSNames && hasMatchingDNSNames:
			dbg.Info("selecting solver as this solver has matching DNS names and the previous one does not")
			selectSolver()
			continue
		case selectedHasMatchingDNSNames && !hasMatchingDNSNames:
			dbg.Info("not selecting solver as the previous one has matching DNS names and this one does not")
			continue
		case !selectedHasMatchingDNSNames && !hasMatchingDNSNames:
			dbg.Info("solver does not have any matching DNS names, checking dnsZones")
			// check zones
		case selectedHasMatchingDNSNames && hasMatchingDNSNames:
			dbg.Info("both this solver and the previously selected one matches dnsNames, comparing zones")
			if numDNSZonesMatch > selectedNumDNSZonesMatch {
				dbg.Info("selecting solver as this one has a more specific dnsZone match than the previously selected one")
				selectSolver()
				continue
			}
			if selectedNumDNSZonesMatch > numDNSZonesMatch {
				dbg.Info("not selecting this solver as the previously selected one has a more specific dnsZone match")
				continue
			}
			dbg.Info("both this solver and the previously selected one match dnsZones, comparing labels")
			// choose the one with the most labels
			if numLabelsMatch > selectedNumLabelsMatch {
				dbg.Info("selecting solver as this one has more labels than the previously selected one")
				selectSolver()
				continue
			}
			dbg.Info("not selecting this solver as previous one has either the same number of or more labels")
			continue
		}

		selectedHasMatchingDNSZones := selectedNumDNSZonesMatch > 0
		hasMatchingDNSZones := numDNSZonesMatch > 0

		switch {
		case !selectedHasMatchingDNSZones && hasMatchingDNSZones:
			dbg.Info("selecting solver as this solver has matching DNS zones and the previous one does not")
			selectSolver()
			continue
		case selectedHasMatchingDNSZones && !hasMatchingDNSZones:
			dbg.Info("not selecting solver as the previous one has matching DNS zones and this one does not")
			continue
		case !selectedHasMatchingDNSZones && !hasMatchingDNSZones:
			dbg.Info("solver does not have any matching DNS zones, checking labels")
			// check labels
		case selectedHasMatchingDNSZones && hasMatchingDNSZones:
			dbg.Info("both this solver and the previously selected one matches dnsZones")
			dbg.Info("comparing number of matching domain segments")
			// choose the one with the most matching DNS zone segments
			if numDNSZonesMatch > selectedNumDNSZonesMatch {
				dbg.Info("selecting solver because this one has more matching DNS zone segments")
				selectSolver()
				continue
			}
			if selectedNumDNSZonesMatch > numDNSZonesMatch {
				dbg.Info("not selecting solver because previous one has more matching DNS zone segments")
				continue
			}
			// choose the one with the most labels
			if numLabelsMatch > selectedNumLabelsMatch {
				dbg.Info("selecting solver because this one has more labels than the previous one")
				selectSolver()
				continue
			}
			dbg.Info("not selecting solver as this one's number of matching labels is equal to or less than the last one")
			continue
		}

		if numLabelsMatch > selectedNumLabelsMatch {
			dbg.Info("selecting solver as this one has more labels than the last one")
			selectSolver()
			continue
		}

		dbg.Info("not selecting solver as this one's number of matching labels is equal to or less than the last one (reached end of loop)")
		// if we get here, the number of matches is less than or equal so we
		// fallback to choosing the first in the list
	}

	if selectedSolver == nil || selectedChallenge == nil {
		return nil, fmt.Errorf("no configured challenge solvers can be used for this challenge")
	}

	// It should never be possible for this case to be hit as earlier in this
	// method we already assert that the challenge type is one of 'http-01'
	// or 'dns-01'.
	chType, err := challengeType(selectedChallenge.Type)
	if err != nil {
		return nil, err
	}

	key, err := keyForChallenge(cl, selectedChallenge.Token, chType)
	if err != nil {
		return nil, err
	}

	// 4. handle overriding the HTTP01 ingress class and name fields using the
	//    ACMECertificateHTTP01IngressNameOverride & Class annotations
	if err := applyIngressParameterAnnotationOverrides(o, selectedSolver); err != nil {
		return nil, err
	}

	// 5. construct Challenge resource with spec.solver field set
	return &cmacme.ChallengeSpec{
		AuthorizationURL: authz.URL,
		Type:             chType,
		URL:              selectedChallenge.URL,
		DNSName:          authz.Identifier,
		Token:            selectedChallenge.Token,
		Key:              key,
		// selectedSolver cannot be nil due to the check above.
		Solver:    *selectedSolver,
		Wildcard:  wc,
		IssuerRef: o.Spec.IssuerRef,
	}, nil
}

func challengeType(t string) (cmacme.ACMEChallengeType, error) {
	switch t {
	case "http-01":
		return cmacme.ACMEChallengeTypeHTTP01, nil
	case "dns-01":
		return cmacme.ACMEChallengeTypeDNS01, nil
	default:
		return "", fmt.Errorf("unsupported challenge type: %v", t)
	}
}

func applyIngressParameterAnnotationOverrides(o *cmacme.Order, s *cmacme.ACMEChallengeSolver) error {
	if s.HTTP01 == nil || s.HTTP01.Ingress == nil || o.Annotations == nil {
		return nil
	}

	manualIngressName, hasManualIngressName := o.Annotations[cmacme.ACMECertificateHTTP01IngressNameOverride]
	manualIngressClass, hasManualIngressClass := o.Annotations[cmacme.ACMECertificateHTTP01IngressClassOverride]
	// don't allow both override annotations to be specified at once
	if hasManualIngressName && hasManualIngressClass {
		return fmt.Errorf("both ingress name and ingress class overrides specified - only one may be specified at a time")
	}
	// if an override annotation is specified, clear out the existing solver
	// config
	if hasManualIngressClass || hasManualIngressName {
		s.HTTP01.Ingress.Class = nil
		s.HTTP01.Ingress.Name = ""
	}
	if hasManualIngressName {
		s.HTTP01.Ingress.Name = manualIngressName
	}
	if hasManualIngressClass {
		s.HTTP01.Ingress.Class = &manualIngressClass
	}
	return nil
}

func keyForChallenge(cl acmecl.Interface, token string, chType cmacme.ACMEChallengeType) (string, error) {
	switch chType {
	case cmacme.ACMEChallengeTypeHTTP01:
		return cl.HTTP01ChallengeResponse(token)
	case cmacme.ACMEChallengeTypeDNS01:
		return cl.DNS01ChallengeRecord(token)
	default:
		return "", fmt.Errorf("unsupported challenge type: %v", chType)
	}
}

func anyChallengesFailed(chs []*cmacme.Challenge) bool {
	for _, ch := range chs {
		if acme.IsFailureState(ch.Status.State) {
			return true
		}
	}
	return false
}

func allChallengesFinal(chs []*cmacme.Challenge) bool {
	for _, ch := range chs {
		if !acme.IsFinalState(ch.Status.State) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package buildlogs

import (
	"fmt"
	"io"
	"net/http"

	"github.com/spf13/cobra"

	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
	kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/kubectl/genericclioptions"

	buildapi "github.com/openshift/origin/pkg/build/apis/build"
	buildmanualclient "github.com/openshift/origin/pkg/build/client/internalversion"
	buildclientinternal "github.com/openshift/origin/pkg/build/generated/internalclientset"
	"github.com/openshift/origin/pkg/oc/cli/logs"
)

var (
	buildLogsLong = templates.LongDesc(`
		Retrieve logs for a build

		This command displays the log for the provided build. If the pod that ran the build has been deleted logs
		will no longer be available. If the build has not yet completed, the build logs will be streamed until the
		build completes or fails.`)

	buildLogsExample = templates.Examples(`
		# Stream logs from container
  	%[1]s build-logs 566bed879d2d`)
)

type BuildLogsOptions struct {
	Follow bool
	NoWait bool

	Name        string
	Namespace   string
	BuildClient buildclientinternal.Interface

	genericclioptions.IOStreams
}

func NewBuildLogsOptions(streams genericclioptions.IOStreams) *BuildLogsOptions {
	return &BuildLogsOptions{
		IOStreams: streams,
		Follow:    true,
	}
}

// NewCmdBuildLogs implements the OpenShift cli build-logs command
func NewCmdBuildLogs(fullName string, f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command {
	o := NewBuildLogsOptions(streams)
	cmd := &cobra.Command{
		Use:        "build-logs BUILD",
		Short:      "Show logs from a build",
		Long:       buildLogsLong,
		Example:    fmt.Sprintf(buildLogsExample, fullName),
		Deprecated: fmt.Sprintf("use oc %v build/<build-name>", logs.LogsRecommendedCommandName),
		Hidden:     true,
		Run: func(cmd *cobra.Command, args []string) {
			kcmdutil.CheckErr(o.Complete(f, cmd, args))
			kcmdutil.CheckErr(o.RunBuildLogs())
		},
	}
	cmd.Flags().BoolVarP(&o.Follow, "follow", "f", o.Follow, "Specify whether logs should be followed; default is true.")
	cmd.Flags().BoolVarP(&o.NoWait, "nowait", "w", o.NoWait, "Specify whether to return immediately without waiting for logs to be available; default is false.")

	return cmd
}

func (o *BuildLogsOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error {
	if len(args) != 1 {
		return fmt.Errorf("build name is required")
	}
	o.Name = args[0]

	var err error
	o.Namespace, _, err = f.ToRawKubeConfigLoader().Namespace()
	if err != nil {
		return err
	}

	clientConfig, err := f.ToRESTConfig()
	if err != nil {
		return err
	}
	o.BuildClient, err = buildclientinternal.NewForConfig(clientConfig)
	if err != nil {
		return err
	}

	return nil
}

// RunBuildLogs contains all the necessary functionality for the OpenShift cli build-logs command
func (o *BuildLogsOptions) RunBuildLogs() error {
	opts := buildapi.BuildLogOptions{
		Follow: o.Follow,
		NoWait: o.NoWait,
	}
	readCloser, err := buildmanualclient.NewBuildLogClient(o.BuildClient.Build().RESTClient(), o.Namespace).Logs(o.Name, opts).Stream()
	if err != nil {
		return err
	}
	defer readCloser.Close()

	_, err = io.Copy(o.Out, readCloser)
	if err, ok := err.(errors.APIStatus); ok {
		if err.Status().Code == http.StatusNotFound {
			switch err.Status().Details.Kind {
			case "build":
				return fmt.Errorf("the build %s could not be found, therefore build logs cannot be retrieved", err.Status().Details.Name)
			case "pod":
				return fmt.Errorf("the pod %s for build %s could not be found, therefore build logs cannot be retrieved", err.Status().Details.Name, o.Name)
			}
		}
	}
	return err
}
<commit_msg>externalize buildlogs command<commit_after>package buildlogs

import (
	"fmt"
	"io"
	"net/http"

	"github.com/spf13/cobra"

	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
	kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/kubectl/genericclioptions"

	buildv1 "github.com/openshift/api/build/v1"
	buildv1client "github.com/openshift/client-go/build/clientset/versioned"
	buildclientv1 "github.com/openshift/origin/pkg/build/client/v1"
	"github.com/openshift/origin/pkg/oc/cli/logs"
)

var (
	buildLogsLong = templates.LongDesc(`
		Retrieve logs for a build

		This command displays the log for the provided build. If the pod that ran the build has been deleted logs
		will no longer be available. If the build has not yet completed, the build logs will be streamed until the
		build completes or fails.`)

	buildLogsExample = templates.Examples(`
		# Stream logs from container
  	%[1]s build-logs 566bed879d2d`)
)

type BuildLogsOptions struct {
	Follow bool
	NoWait bool

	Name        string
	Namespace   string
	BuildClient buildv1client.Interface

	genericclioptions.IOStreams
}

func NewBuildLogsOptions(streams genericclioptions.IOStreams) *BuildLogsOptions {
	return &BuildLogsOptions{
		IOStreams: streams,
		Follow:    true,
	}
}

// NewCmdBuildLogs implements the OpenShift cli build-logs command
func NewCmdBuildLogs(fullName string, f kcmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command {
	o := NewBuildLogsOptions(streams)
	cmd := &cobra.Command{
		Use:        "build-logs BUILD",
		Short:      "Show logs from a build",
		Long:       buildLogsLong,
		Example:    fmt.Sprintf(buildLogsExample, fullName),
		Deprecated: fmt.Sprintf("use oc %v build/<build-name>", logs.LogsRecommendedCommandName),
		Hidden:     true,
		Run: func(cmd *cobra.Command, args []string) {
			kcmdutil.CheckErr(o.Complete(f, cmd, args))
			kcmdutil.CheckErr(o.RunBuildLogs())
		},
	}
	cmd.Flags().BoolVarP(&o.Follow, "follow", "f", o.Follow, "Specify whether logs should be followed; default is true.")
	cmd.Flags().BoolVarP(&o.NoWait, "nowait", "w", o.NoWait, "Specify whether to return immediately without waiting for logs to be available; default is false.")

	return cmd
}

func (o *BuildLogsOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error {
	if len(args) != 1 {
		return fmt.Errorf("build name is required")
	}
	o.Name = args[0]

	var err error
	o.Namespace, _, err = f.ToRawKubeConfigLoader().Namespace()
	if err != nil {
		return err
	}

	clientConfig, err := f.ToRESTConfig()
	if err != nil {
		return err
	}
	o.BuildClient, err = buildv1client.NewForConfig(clientConfig)
	if err != nil {
		return err
	}

	return nil
}

// RunBuildLogs contains all the necessary functionality for the OpenShift cli build-logs command
func (o *BuildLogsOptions) RunBuildLogs() error {
	opts := buildv1.BuildLogOptions{
		Follow: o.Follow,
		NoWait: o.NoWait,
	}
	readCloser, err := buildclientv1.NewBuildLogClient(o.BuildClient.BuildV1().RESTClient(), o.Namespace).Logs(o.Name, opts).Stream()
	if err != nil {
		return err
	}
	defer readCloser.Close()

	_, err = io.Copy(o.Out, readCloser)
	if err, ok := err.(errors.APIStatus); ok {
		if err.Status().Code == http.StatusNotFound {
			switch err.Status().Details.Kind {
			case "build":
				return fmt.Errorf("the build %s could not be found, therefore build logs cannot be retrieved", err.Status().Details.Name)
			case "pod":
				return fmt.Errorf("the pod %s for build %s could not be found, therefore build logs cannot be retrieved", err.Status().Details.Name, o.Name)
			}
		}
	}
	return err
}
<|endoftext|>


<commit_before>package cloudwatch

import (
	"context"
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/tsdb"
	"golang.org/x/sync/errgroup"
)

type CloudWatchExecutor struct {
	*models.DataSource
	ec2Svc  ec2iface.EC2API
	rgtaSvc resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
}

type DatasourceInfo struct {
	Profile       string
	Region        string
	AuthType      string
	AssumeRoleArn string
	Namespace     string

	AccessKey string
	SecretKey string
}

func NewCloudWatchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
	return &CloudWatchExecutor{}, nil
}

var (
	plog               log.Logger
	standardStatistics map[string]bool
	aliasFormat        *regexp.Regexp
)

func init() {
	plog = log.New("tsdb.cloudwatch")
	tsdb.RegisterTsdbQueryEndpoint("cloudwatch", NewCloudWatchExecutor)
	standardStatistics = map[string]bool{
		"Average":     true,
		"Maximum":     true,
		"Minimum":     true,
		"Sum":         true,
		"SampleCount": true,
	}
	aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
}

func (e *CloudWatchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	var result *tsdb.Response
	e.DataSource = dsInfo
	queryType := queryContext.Queries[0].Model.Get("type").MustString("")
	var err error

	switch queryType {
	case "metricFindQuery":
		result, err = e.executeMetricFindQuery(ctx, queryContext)
	case "annotationQuery":
		result, err = e.executeAnnotationQuery(ctx, queryContext)
	case "timeSeriesQuery":
		fallthrough
	default:
		result, err = e.executeTimeSeriesQuery(ctx, queryContext)
	}

	return result, err
}

func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	results := &tsdb.Response{
		Results: make(map[string]*tsdb.QueryResult),
	}
	resultChan := make(chan *tsdb.QueryResult, len(queryContext.Queries))

	eg, ectx := errgroup.WithContext(ctx)

	getMetricDataQueries := make(map[string]map[string]*CloudWatchQuery)
	for i, model := range queryContext.Queries {
		queryType := model.Model.Get("type").MustString()
		if queryType != "timeSeriesQuery" && queryType != "" {
			continue
		}

		RefId := queryContext.Queries[i].RefId
		query, err := parseQuery(queryContext.Queries[i].Model)
		if err != nil {
			results.Results[RefId] = &tsdb.QueryResult{
				Error: err,
			}
			return results, nil
		}
		query.RefId = RefId

		if query.Id != "" {
			if _, ok := getMetricDataQueries[query.Region]; !ok {
				getMetricDataQueries[query.Region] = make(map[string]*CloudWatchQuery)
			}
			getMetricDataQueries[query.Region][query.Id] = query
			continue
		}

		if query.Id == "" && query.Expression != "" {
			results.Results[query.RefId] = &tsdb.QueryResult{
				Error: fmt.Errorf("Invalid query: id should be set if using expression"),
			}
			return results, nil
		}

		eg.Go(func() error {
			defer func() {
				if err := recover(); err != nil {
					plog.Error("Execute Query Panic", "error", err, "stack", log.Stack(1))
					if theErr, ok := err.(error); ok {
						resultChan <- &tsdb.QueryResult{
							RefId: query.RefId,
							Error: theErr,
						}
					}
				}
			}()

			queryRes, err := e.executeQuery(ectx, query, queryContext)
			if ae, ok := err.(awserr.Error); ok && ae.Code() == "500" {
				return err
			}
			if err != nil {
				resultChan <- &tsdb.QueryResult{
					RefId: query.RefId,
					Error: err,
				}
				return nil
			}
			resultChan <- queryRes
			return nil
		})
	}

	if len(getMetricDataQueries) > 0 {
		for region, getMetricDataQuery := range getMetricDataQueries {
			q := getMetricDataQuery
			eg.Go(func() error {
				defer func() {
					if err := recover(); err != nil {
						plog.Error("Execute Get Metric Data Query Panic", "error", err, "stack", log.Stack(1))
						if theErr, ok := err.(error); ok {
							resultChan <- &tsdb.QueryResult{
								Error: theErr,
							}
						}
					}
				}()

				queryResponses, err := e.executeGetMetricDataQuery(ectx, region, q, queryContext)
				if ae, ok := err.(awserr.Error); ok && ae.Code() == "500" {
					return err
				}
				for _, queryRes := range queryResponses {
					if err != nil {
						queryRes.Error = err
					}
					resultChan <- queryRes
				}
				return nil
			})
		}
	}

	if err := eg.Wait(); err != nil {
		return nil, err
	}
	close(resultChan)
	for result := range resultChan {
		results.Results[result.RefId] = result
	}

	return results, nil
}

func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]string, label string) string {
	region := query.Region
	namespace := query.Namespace
	metricName := query.MetricName
	period := strconv.Itoa(query.Period)
	if len(query.Id) > 0 && len(query.Expression) > 0 {
		if strings.Index(query.Expression, "SEARCH(") == 0 {
			pIndex := strings.LastIndex(query.Expression, ",")
			period = strings.Trim(query.Expression[pIndex+1:], " )")
			sIndex := strings.LastIndex(query.Expression[:pIndex], ",")
			stat = strings.Trim(query.Expression[sIndex+1:pIndex], " '")
		} else if len(query.Alias) > 0 {
			// expand by Alias
		} else {
			return query.Id
		}
	}

	data := map[string]string{}
	data["region"] = region
	data["namespace"] = namespace
	data["metric"] = metricName
	data["stat"] = stat
	data["period"] = period
	if len(label) != 0 {
		data["label"] = label
	}
	for k, v := range dimensions {
		data[k] = v
	}

	result := aliasFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte {
		labelName := strings.Replace(string(in), "{{", "", 1)
		labelName = strings.Replace(labelName, "}}", "", 1)
		labelName = strings.TrimSpace(labelName)
		if val, exists := data[labelName]; exists {
			return []byte(val)
		}

		return in
	})

	return string(result)
}
<commit_msg>CloudWatch: Use default alias if there is no alias for metrics (#16732)<commit_after>package cloudwatch

import (
	"context"
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/tsdb"
	"golang.org/x/sync/errgroup"
)

type CloudWatchExecutor struct {
	*models.DataSource
	ec2Svc  ec2iface.EC2API
	rgtaSvc resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
}

type DatasourceInfo struct {
	Profile       string
	Region        string
	AuthType      string
	AssumeRoleArn string
	Namespace     string

	AccessKey string
	SecretKey string
}

func NewCloudWatchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
	return &CloudWatchExecutor{}, nil
}

var (
	plog               log.Logger
	standardStatistics map[string]bool
	aliasFormat        *regexp.Regexp
)

func init() {
	plog = log.New("tsdb.cloudwatch")
	tsdb.RegisterTsdbQueryEndpoint("cloudwatch", NewCloudWatchExecutor)
	standardStatistics = map[string]bool{
		"Average":     true,
		"Maximum":     true,
		"Minimum":     true,
		"Sum":         true,
		"SampleCount": true,
	}
	aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
}

func (e *CloudWatchExecutor) Query(ctx context.Context, dsInfo *models.DataSource, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	var result *tsdb.Response
	e.DataSource = dsInfo
	queryType := queryContext.Queries[0].Model.Get("type").MustString("")
	var err error

	switch queryType {
	case "metricFindQuery":
		result, err = e.executeMetricFindQuery(ctx, queryContext)
	case "annotationQuery":
		result, err = e.executeAnnotationQuery(ctx, queryContext)
	case "timeSeriesQuery":
		fallthrough
	default:
		result, err = e.executeTimeSeriesQuery(ctx, queryContext)
	}

	return result, err
}

func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) {
	results := &tsdb.Response{
		Results: make(map[string]*tsdb.QueryResult),
	}
	resultChan := make(chan *tsdb.QueryResult, len(queryContext.Queries))

	eg, ectx := errgroup.WithContext(ctx)

	getMetricDataQueries := make(map[string]map[string]*CloudWatchQuery)
	for i, model := range queryContext.Queries {
		queryType := model.Model.Get("type").MustString()
		if queryType != "timeSeriesQuery" && queryType != "" {
			continue
		}

		RefId := queryContext.Queries[i].RefId
		query, err := parseQuery(queryContext.Queries[i].Model)
		if err != nil {
			results.Results[RefId] = &tsdb.QueryResult{
				Error: err,
			}
			return results, nil
		}
		query.RefId = RefId

		if query.Id != "" {
			if _, ok := getMetricDataQueries[query.Region]; !ok {
				getMetricDataQueries[query.Region] = make(map[string]*CloudWatchQuery)
			}
			getMetricDataQueries[query.Region][query.Id] = query
			continue
		}

		if query.Id == "" && query.Expression != "" {
			results.Results[query.RefId] = &tsdb.QueryResult{
				Error: fmt.Errorf("Invalid query: id should be set if using expression"),
			}
			return results, nil
		}

		eg.Go(func() error {
			defer func() {
				if err := recover(); err != nil {
					plog.Error("Execute Query Panic", "error", err, "stack", log.Stack(1))
					if theErr, ok := err.(error); ok {
						resultChan <- &tsdb.QueryResult{
							RefId: query.RefId,
							Error: theErr,
						}
					}
				}
			}()

			queryRes, err := e.executeQuery(ectx, query, queryContext)
			if ae, ok := err.(awserr.Error); ok && ae.Code() == "500" {
				return err
			}
			if err != nil {
				resultChan <- &tsdb.QueryResult{
					RefId: query.RefId,
					Error: err,
				}
				return nil
			}
			resultChan <- queryRes
			return nil
		})
	}

	if len(getMetricDataQueries) > 0 {
		for region, getMetricDataQuery := range getMetricDataQueries {
			q := getMetricDataQuery
			eg.Go(func() error {
				defer func() {
					if err := recover(); err != nil {
						plog.Error("Execute Get Metric Data Query Panic", "error", err, "stack", log.Stack(1))
						if theErr, ok := err.(error); ok {
							resultChan <- &tsdb.QueryResult{
								Error: theErr,
							}
						}
					}
				}()

				queryResponses, err := e.executeGetMetricDataQuery(ectx, region, q, queryContext)
				if ae, ok := err.(awserr.Error); ok && ae.Code() == "500" {
					return err
				}
				for _, queryRes := range queryResponses {
					if err != nil {
						queryRes.Error = err
					}
					resultChan <- queryRes
				}
				return nil
			})
		}
	}

	if err := eg.Wait(); err != nil {
		return nil, err
	}
	close(resultChan)
	for result := range resultChan {
		results.Results[result.RefId] = result
	}

	return results, nil
}

func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]string, label string) string {
	region := query.Region
	namespace := query.Namespace
	metricName := query.MetricName
	period := strconv.Itoa(query.Period)
	if len(query.Id) > 0 && len(query.Expression) > 0 {
		if strings.Index(query.Expression, "SEARCH(") == 0 {
			pIndex := strings.LastIndex(query.Expression, ",")
			period = strings.Trim(query.Expression[pIndex+1:], " )")
			sIndex := strings.LastIndex(query.Expression[:pIndex], ",")
			stat = strings.Trim(query.Expression[sIndex+1:pIndex], " '")
		} else if len(query.Alias) > 0 {
			// expand by Alias
		} else {
			return query.Id
		}
	}

	data := map[string]string{}
	data["region"] = region
	data["namespace"] = namespace
	data["metric"] = metricName
	data["stat"] = stat
	data["period"] = period
	if len(label) != 0 {
		data["label"] = label
	}
	for k, v := range dimensions {
		data[k] = v
	}

	result := aliasFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte {
		labelName := strings.Replace(string(in), "{{", "", 1)
		labelName = strings.Replace(labelName, "}}", "", 1)
		labelName = strings.TrimSpace(labelName)
		if val, exists := data[labelName]; exists {
			return []byte(val)
		}

		return in
	})

	if string(result) == "" {
		return metricName + "_" + stat
	}

	return string(result)
}
<|endoftext|>


<commit_before>package pretty

import (
	"fmt"
	"os"
	"text/tabwriter"
)

// A TabPrinter is an object that allows printing tab-aligned words on multiple lines,
// up to a maximum number per line
type TabPrinter struct {
	w            *tabwriter.Writer
	current, max int
}

// create a TabPrinter
//
// max specifies the maximum number of 'words' per line
func NewTabPrinter(max int) *TabPrinter {
	tp := &TabPrinter{w: new(tabwriter.Writer), max: max}
	tp.w.Init(os.Stdout, 0, 8, 1, '\t', 0)

	return tp
}

// print a 'word'
//
// when the maximum number of words per lines is reached, this will print the formatted line
func (tp *TabPrinter) Print(arg interface{}) {
	if tp.current > 0 {
		if (tp.current % tp.max) == 0 {
			fmt.Fprintln(tp.w, "")
			tp.w.Flush()
                        tp.current = 0
		} else {
			fmt.Fprint(tp.w, "\t")
		}
	}

	tp.current++
	fmt.Fprint(tp.w, arg)
}

// print current line
//
// terminate current line and print - call this after all words have been printed
func (tp *TabPrinter) Println() {
	if tp.current > 0 && (tp.current%tp.max) != 0 {
		fmt.Fprintln(tp.w, "")
		tp.w.Flush()
	}

	tp.current = 0
}
<commit_msg>formatting stuff<commit_after>package pretty

import (
	"fmt"
	"os"
	"text/tabwriter"
)

// A TabPrinter is an object that allows printing tab-aligned words on multiple lines,
// up to a maximum number per line
type TabPrinter struct {
	w            *tabwriter.Writer
	current, max int
}

// create a TabPrinter
//
// max specifies the maximum number of 'words' per line
func NewTabPrinter(max int) *TabPrinter {
	tp := &TabPrinter{w: new(tabwriter.Writer), max: max}
	tp.w.Init(os.Stdout, 0, 8, 1, '\t', 0)

	return tp
}

// print a 'word'
//
// when the maximum number of words per lines is reached, this will print the formatted line
func (tp *TabPrinter) Print(arg interface{}) {
	if tp.current > 0 {
		if (tp.current % tp.max) == 0 {
			fmt.Fprintln(tp.w, "")
			tp.w.Flush()
			tp.current = 0
		} else {
			fmt.Fprint(tp.w, "\t")
		}
	}

	tp.current++
	fmt.Fprint(tp.w, arg)
}

// print current line
//
// terminate current line and print - call this after all words have been printed
func (tp *TabPrinter) Println() {
	if tp.current > 0 && (tp.current%tp.max) != 0 {
		fmt.Fprintln(tp.w, "")
		tp.w.Flush()
	}

	tp.current = 0
}
<|endoftext|>


<commit_before>// The Simple Token Bucket like HTB in Linux TC.
package tbucket

import "time"

type TokenBucket struct {
	bucket  chan bool
	sleep   time.Duration
	started bool
	stoped  bool
}

// NewTB creates a new token bucket.
// The default size of the token bucket is 1024.
func NewTokenBucket(rate uint64) *TokenBucket {
	t := &TokenBucket{}
	return t.SetRate(rate).SetBucketSize(1024)
}

func (t TokenBucket) calcSleep(rate uint64) time.Duration {
	return time.Second / time.Duration(rate)
	//return time.Duration(uint64(time.Second) / rate)
}

// Set the size of the token bucket.
//
// If the token bucket server has been started, calling this method will panic.
func (t *TokenBucket) SetBucketSize(size uint) *TokenBucket {
	if t.started {
		panic("The token bucket server has been started")
	}
	t.bucket = make(chan bool, size)
	return t
}

// Set the rate to produce the token. The unit is token/s.
//
// Allow that adjust the rate in running.
func (t *TokenBucket) SetRate(rate uint64) *TokenBucket {
	t.sleep = t.calcSleep(rate)
	return t
}

// Get the token from the bucket.
//
// This method isn't the returned value. That it returns is indicating that you
// have got the token.
//
// If the token bucket server has not been started, calling this method will panic.
func (t *TokenBucket) Get() {
	if !t.started {
		panic("The token bucket server isn't started")
	}
	<-t.bucket
	return
}

// Start to produce the token and put it to the bucket. Then you can get
// the token from the bucket by calling t.Get().
//
// If the token bucket server has been started, calling this method will panic.
func (t *TokenBucket) Start() {
	if t.started {
		panic("The token bucket server has been started")
	}

	go t.start()
	t.started = true
	t.stoped = false
}

// Stop the token bucket server. Later you can start it again.
//
// If the token bucket server has not been started, calling this method will panic.
func (t *TokenBucket) Stop() {
	if !t.started {
		panic("The token bucket server isn't started")
	}
	t.stoped = true
	t.started = false

	// In order to let the for loop ends in t.start().
	time.Sleep(t.sleep)
}

func (t *TokenBucket) start() {
	for !t.stoped {
		t.bucket <- true
		time.Sleep(t.sleep)
	}
}
<commit_msg>Update token bucket<commit_after>// The Simple Token Bucket like HTB in Linux TC.
package tbucket

import "time"

type TokenBucket struct {
	bucket  chan bool
	sleep   time.Duration
	started bool
	stoped  bool
	num     int64
}

// NewTB creates a new token bucket.
// The default size of the token bucket is 1024.
func NewTokenBucket(rate uint64) *TokenBucket {
	t := &TokenBucket{}
	return t.SetRate(rate).SetBucketSize(1024)
}

// Set the size of the token bucket.
//
// If the token bucket server has been started, calling this method will panic.
func (t *TokenBucket) SetBucketSize(size uint) *TokenBucket {
	if t.started {
		panic("The token bucket server has been started")
	}
	t.bucket = make(chan bool, size)
	return t
}

// Set the rate to produce the token. The unit is token/s.
//
// Allow that adjust the rate in running.
func (t *TokenBucket) SetRate(rate uint64) *TokenBucket {
	t.num = 1
	min_sleep := time.Millisecond * time.Duration(10)
	sleep := time.Second / time.Duration(rate)
	if sleep < min_sleep {
		t.num = min_sleep / sleep
	} else {
		t.sleep = sleep
	}

	return t
}

// Get the token from the bucket.
//
// This method isn't the returned value. That it returns is indicating that you
// have got the token.
//
// If the token bucket server has not been started, calling this method will panic.
func (t *TokenBucket) Get() {
	if !t.started {
		panic("The token bucket server isn't started")
	}
	<-t.bucket
	return
}

// Start to produce the token and put it to the bucket. Then you can get
// the token from the bucket by calling t.Get().
//
// If the token bucket server has been started, calling this method will panic.
func (t *TokenBucket) Start() {
	if t.started {
		panic("The token bucket server has been started")
	}

	go t.start()
	t.started = true
	t.stoped = false
}

// Stop the token bucket server. Later you can start it again.
//
// If the token bucket server has not been started, calling this method will panic.
func (t *TokenBucket) Stop() {
	if !t.started {
		panic("The token bucket server isn't started")
	}
	t.stoped = true
	t.started = false

	// In order to let the for loop ends in t.start().
	time.Sleep(t.sleep)
}

func (t *TokenBucket) start() {
	for !t.stoped {
		for i := 0; i < t.num; i++ {
			t.bucket <- true
		}

		time.Sleep(t.sleep)
	}
}
<|endoftext|>