<commit_before>package statictest

import (
	"reflect"
	"strings"
)

// Error implements error and holds a list of failures.
type Error struct {
	Errors []string
}

func (e *Error) Error() string {
	return strings.Join(e.Errors, "\n")
}

// AsError returns nil if Errors is empty or the pointer to Error otherwise.
func (e *Error) AsError() error {
	if e.Errors == nil {
		return nil
	}
	return e
}

// Skipper is used to skip errors. Skip returns true if the error in
// str must be skipped.
type Skipper interface {
	Skip(str string) bool
}

// SkipFunc is an adapter to allow the use of functions as Skippers. If f is a function, SkipFunc(f) is a Skipper that calls f.
type SkipFunc func(str string) bool

// Skip calls s(str)
func (s SkipFunc) Skip(str string) bool { return s(str) }

// StringSkipper implements Skipper and skips an error if Matcher(err, str) == true for
// any of Strings
type StringSkipper struct {
	Strings []string
	Matcher func(err, str string) bool
}

// Skip returns true if Matcher(check, str) == true for any of Strings.
func (s StringSkipper) Skip(check string) bool {
	for _, str := range s.Strings {
		if s.Matcher(check, str) {
			return true
		}
	}
	return false
}

func skip(check string, skippers []Skipper) bool {
	for _, s := range skippers {
		if s.Skip(check) {
			return true
		}
	}
	return false
}

// Skip returns a Checker that executes checkers and filters errors skipped by
// the provided skippers. If checker returns an Error instance the filters are
// applied on Errors.Errors. Each skipper is run in the order provided and a single
// skipper returning true will result in that error being skipped.
func Skip(checker Checker, skippers ...Skipper) Checker {
	return CheckFunc(func(pkg ...string) error {
		switch err := checker.Check(pkg...).(type) {
		case nil:
			return nil
		case *Error:
			var n []string
			for _, e := range err.Errors {
				if !skip(e, skippers) {
					n = append(n, e)
				}
			}
			err.Errors = n
			return err.AsError()
		default:
			if skip(err.Error(), skippers) {
				return nil
			}
			return err
		}
	})
}

// Checker performs a static check of a list of packages.
// Use Verify(pkg, checker...) to perform the actual static checks.
type Checker interface {
	// Check performs a static check of all files in pkgs, which may be fully
	// qualified import paths, relative import paths or paths with the wildcard
	// suffix ...
	Check(pkg ...string) error
}

// CheckFunc is a function that implements Checker
type CheckFunc func(pkgs ...string) error

// Check calls c with pkgs
func (c CheckFunc) Check(pkgs ...string) error { return c(pkgs...) }

// Group returns a Checker that applies each of checkers in the order provided.
//
// The error returned is either nil or of type *Errors, containing errors returned
// by each Checker. These are prefixed with the type of the Checker that generated
// the error. For example, an error generated by the govet Checker such as:
//
//    file.go:23: err is unintentionally shadowed.
//
// is converted to:
//
//    govet.Checker: file.go:23: err is unintentionally shadowed.
//
// If an error returned by a checker is of type *Errors, these will be listed
// separately in the final returned error. A checker is not shorted-circuited
// by a previous checker returning an error.
func Group(checkers ...Checker) Checker {
	return CheckFunc(func(pkgs ...string) error {
		errs := &Error{}
		for _, checker := range checkers {
			name := reflect.TypeOf(checker).String()
			switch err := checker.Check(pkgs...).(type) {
			case nil:
				continue
			case *Error:
				for _, e := range err.Errors {
					errs.Errors = append(errs.Errors, name+": "+e)
				}
			default:
				errs.Errors = append(errs.Errors, name+": "+err.Error())
			}
		}
		return errs.AsError()
	})
}
<commit_msg>Add package doc with an example<commit_after>// Package statictest runs static analysis tools as go tests.
//
// It is intended to be used as a substitute for an external build
// step that runs tools such as go vet or golint.
//
// Example Usage
//
//    func TestStaticChecks(t *testing.T) {
//    	basic := statictest.Group(
//    		gofmt.Check{},	// Verify that all files are properly formatted
//    		govet.Shadow,	// go vet
//    		golint.Check{},	// golint
//    		gosimple.Check{},	// gosimple
//    		gostaticcheck.Check{},	// gostaticcheck
//    	)
//
//    	// Ignore some lint errors that we're not interested in.
//    	skipper := statictest.SkipRegexpMatch(
//     		"should omit type InterfaceType from declaration of var Handle",
//    		`settings.go:.*can probably use "var res \[\]byte" instead`)
//
//    	basic = statictest.Skip(basic, skipper)
//
//     	// Verify all files under this package recursively.
//    	if err := skipped.Check("./..."); err != nil {
//    		t.Fatal(err)
//    	}
//    }
package statictest

import (
	"reflect"
	"regexp"
	"strings"
)

// Error implements error and holds a list of failures.
type Error struct {
	Errors []string
}

func (e *Error) Error() string {
	return strings.Join(e.Errors, "\n")
}

// AsError returns nil if Errors is empty or the pointer to Error otherwise.
func (e *Error) AsError() error {
	if e.Errors == nil {
		return nil
	}
	return e
}

// Skipper is used to skip errors. Skip returns true if the error in
// str must be skipped.
type Skipper interface {
	Skip(str string) bool
}

// SkipFunc is an adapter to allow the use of functions as Skippers. If f is a function, SkipFunc(f) is a Skipper that calls f.
type SkipFunc func(str string) bool

// Skip calls s(str)
func (s SkipFunc) Skip(str string) bool { return s(str) }

// StringSkipper implements Skipper and skips an error if Matcher(err, str) == true for
// any of Strings
type StringSkipper struct {
	Strings []string
	Matcher func(err, str string) bool
}

// Skip returns true if Matcher(check, str) == true for any of Strings.
func (s StringSkipper) Skip(check string) bool {
	for _, str := range s.Strings {
		if s.Matcher(check, str) {
			return true
		}
	}
	return false
}

func skip(check string, skippers []Skipper) bool {
	for _, s := range skippers {
		if s.Skip(check) {
			return true
		}
	}
	return false
}

// Skip returns a Checker that executes checkers and filters errors skipped by
// the provided skippers. If checker returns an Error instance the filters are
// applied on Errors.Errors. Each skipper is run in the order provided and a single
// skipper returning true will result in that error being skipped.
func Skip(checker Checker, skippers ...Skipper) Checker {
	return CheckFunc(func(pkg ...string) error {
		switch err := checker.Check(pkg...).(type) {
		case nil:
			return nil
		case *Error:
			var n []string
			for _, e := range err.Errors {
				if !skip(e, skippers) {
					n = append(n, e)
				}
			}
			err.Errors = n
			return err.AsError()
		default:
			if skip(err.Error(), skippers) {
				return nil
			}
			return err
		}
	})
}

// Checker is the interface that wraps the Check method.
//
// Check performs a static check of all files in pkgs, which may be fully
// qualified import paths, relative import paths or paths with the wildcard
// suffix ...
type Checker interface {
	Check(pkgs ...string) error
}

// CheckFunc is a function that implements Checker
type CheckFunc func(pkgs ...string) error

// Check calls c with pkgs
func (c CheckFunc) Check(pkgs ...string) error { return c(pkgs...) }

// Group returns a Checker that applies each of checkers in the order provided.
//
// The error returned is either nil or of type *Errors, containing errors returned
// by each Checker. These are prefixed with the type of the Checker that generated
// the error. For example, an error generated by the govet Checker such as:
//
//    file.go:23: err is unintentionally shadowed.
//
// is converted to:
//
//    govet.Checker: file.go:23: err is unintentionally shadowed.
//
// If an error returned by a checker is of type *Errors, these will be listed
// separately in the final returned error. A checker is not shorted-circuited
// by a previous checker returning an error.
func Group(checkers ...Checker) Checker {
	return CheckFunc(func(pkgs ...string) error {
		errs := &Error{}
		for _, checker := range checkers {
			name := reflect.TypeOf(checker).String()
			switch err := checker.Check(pkgs...).(type) {
			case nil:
				continue
			case *Error:
				for _, e := range err.Errors {
					errs.Errors = append(errs.Errors, name+": "+e)
				}
			default:
				errs.Errors = append(errs.Errors, name+": "+err.Error())
			}
		}
		return errs.AsError()
	})
}

// SkipRegexpMatch returns a Skipper that skips all errors which match
// any of the provided regular expression patterns. SkipRegexpMatch expects
// all patterns to be valid regexps and panics otherwise.
func SkipRegexpMatch(regexps ...string) Skipper {
	return StringSkipper{
		Strings: regexps,
		Matcher: func(errstr, pattern string) bool {
			matched, err := regexp.MatchString(pattern, errstr)
			if err != nil {
				panic(err)
			}
			return matched
		},
	}
}
<|endoftext|>


<commit_before>package stats

import (
	"encoding/json"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"testing"
)

var testHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(200)
	w.Write([]byte("bar"))
})

func TestSimple(t *testing.T) {
	s := New()

	res := httptest.NewRecorder()

	req, _ := http.NewRequest("GET", "http://example.com/foo", nil)

	s.Handler(testHandler).ServeHTTP(res, req)

	assert.Equal(t, res.Code, 200)
	assert.Equal(t, s.ResponseCounts, map[string]int{"200": 1})
}

func TestGetStats(t *testing.T) {
	s := New()

	var stats = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

		stats := s.Data()

		b, _ := json.Marshal(stats)

		w.Write(b)
		w.WriteHeader(200)
		w.Header().Add("Content-Type", "application/json")
	})

	res := httptest.NewRecorder()

	req, _ := http.NewRequest("GET", "http://example.com/foo", nil)

	s.Handler(testHandler).ServeHTTP(res, req)

	res = httptest.NewRecorder()

	s.Handler(stats).ServeHTTP(res, req)

	assert.Equal(t, res.Header().Get("Content-Type"), "application/json")

	var data map[string]interface{}

	err := json.Unmarshal(res.Body.Bytes(), &data)

	assert.Nil(t, err)

	assert.Equal(t, data["total_count"].(float64), 1)
}
<commit_msg>Fix type conversion assertion<commit_after>package stats

import (
	"encoding/json"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"testing"
)

var testHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(200)
	w.Write([]byte("bar"))
})

func TestSimple(t *testing.T) {
	s := New()

	res := httptest.NewRecorder()

	req, _ := http.NewRequest("GET", "http://example.com/foo", nil)

	s.Handler(testHandler).ServeHTTP(res, req)

	assert.Equal(t, res.Code, 200)
	assert.Equal(t, s.ResponseCounts, map[string]int{"200": 1})
}

func TestGetStats(t *testing.T) {
	s := New()

	var stats = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

		stats := s.Data()

		b, _ := json.Marshal(stats)

		w.Write(b)
		w.WriteHeader(200)
		w.Header().Add("Content-Type", "application/json")
	})

	res := httptest.NewRecorder()

	req, _ := http.NewRequest("GET", "http://example.com/foo", nil)

	s.Handler(testHandler).ServeHTTP(res, req)

	res = httptest.NewRecorder()

	s.Handler(stats).ServeHTTP(res, req)

	assert.Equal(t, res.Header().Get("Content-Type"), "application/json")

	var data map[string]interface{}

	err := json.Unmarshal(res.Body.Bytes(), &data)

	assert.Nil(t, err)

	assert.Equal(t, data["total_count"].(float64), float64(1))
}
<|endoftext|>


<commit_before>// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	appsV1 "k8s.io/api/apps/v1"

	"github.com/pkg/errors"
	"github.com/prometheus/prombench/pkg/provider/k8s"
	"gopkg.in/alecthomas/kingpin.v2"
	"k8s.io/apimachinery/pkg/runtime"
)

type scale struct {
	k8sClient *k8s.K8s
	min       int32
	max       int32
	interval  time.Duration
}

func new() *scale {
	k, err := k8s.New(context.Background(), nil)
	if err != nil {
		fmt.Fprintln(os.Stderr, errors.Wrapf(err, "Error creating k8s client inside the k8s cluster"))
		os.Exit(2)
	}
	return &scale{
		k8sClient: k,
	}
}

func (s *scale) updateReplicas(replicas *int32) []k8s.Resource {
	var k8sResource []k8s.Resource
	for _, deployment := range s.k8sClient.GetResources() {
		k8sObjects := make([]runtime.Object, 0)

		for _, resource := range deployment.Objects {
			if kind := strings.ToLower(resource.GetObjectKind().GroupVersionKind().Kind); kind == "deployment" {
				req := resource.(*appsV1.Deployment)
				req.Spec.Replicas = replicas
				k8sObjects = append(k8sObjects, req.DeepCopyObject())
			}
		}
		if len(k8sObjects) > 0 {
			k8sResource = append(k8sResource, k8s.Resource{FileName: deployment.FileName, Objects: k8sObjects})
		}
	}
	return k8sResource
}

func (s *scale) scale(*kingpin.ParseContext) error {
	log.Printf("Starting Prombench-Scaler:\n\t max: %d\n\t min: %d\n\t interval: %s", s.max, s.min, s.interval)

	maxResourceObjects := s.updateReplicas(&s.max)
	minResourceObjects := s.updateReplicas(&s.min)

	for {
		log.Printf("Scaling Deployment to %d", s.max)
		if err := s.k8sClient.ResourceApply(maxResourceObjects); err != nil {
			fmt.Fprintln(os.Stderr, errors.Wrapf(err, "Error scaling deployment"))
		}

		time.Sleep(s.interval)

		log.Printf("Scaling Deployment to %d", s.min)
		if err := s.k8sClient.ResourceApply(minResourceObjects); err != nil {
			fmt.Fprintln(os.Stderr, errors.Wrapf(err, "Error scaling deployment"))
		}

		time.Sleep(s.interval)
	}
}

func main() {

	app := kingpin.New(filepath.Base(os.Args[0]), "The Prombench-Scaler tool")
	app.HelpFlag.Short('h')

	s := new()

	k8sApp := app.Command("scale", "Scale a Kubernetes deployment object periodically up and down. \nex: ./scaler scale -v NAMESPACE:scale -f fake-webserver.yaml 20 1 15m").
		Action(s.k8sClient.DeploymentsParse).
		Action(s.scale)
	k8sApp.Flag("file", "yaml file or folder that describes the parameters for the deployment.").
		Required().
		Short('f').
		ExistingFilesOrDirsVar(&s.k8sClient.DeploymentFiles)
	k8sApp.Flag("vars", "When provided it will substitute the token holders in the yaml file. Follows the standard golang template formating - {{ .hashStable }}.").
		Short('v').
		StringMapVar(&s.k8sClient.DeploymentVars)
	k8sApp.Arg("max", "Number of Replicas to scale up.").
		Required().
		Int32Var(&s.max)
	k8sApp.Arg("min", "Number of Replicas to scale down.").
		Required().
		Int32Var(&s.min)
	k8sApp.Arg("interval", "Time to wait before changing the number of replicas.").
		Required().
		DurationVar(&s.interval)

	if _, err := app.Parse(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, errors.Wrapf(err, "Error parsing commandline arguments"))
		app.Usage(os.Args[1:])
		os.Exit(2)
	}
}
<commit_msg>rename new() function in scaler (#312)<commit_after>// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	appsV1 "k8s.io/api/apps/v1"

	"github.com/pkg/errors"
	"github.com/prometheus/prombench/pkg/provider/k8s"
	"gopkg.in/alecthomas/kingpin.v2"
	"k8s.io/apimachinery/pkg/runtime"
)

type scale struct {
	k8sClient *k8s.K8s
	min       int32
	max       int32
	interval  time.Duration
}

func newScaler() *scale {
	k, err := k8s.New(context.Background(), nil)
	if err != nil {
		fmt.Fprintln(os.Stderr, errors.Wrapf(err, "Error creating k8s client inside the k8s cluster"))
		os.Exit(2)
	}
	return &scale{
		k8sClient: k,
	}
}

func (s *scale) updateReplicas(replicas *int32) []k8s.Resource {
	var k8sResource []k8s.Resource
	for _, deployment := range s.k8sClient.GetResources() {
		k8sObjects := make([]runtime.Object, 0)

		for _, resource := range deployment.Objects {
			if kind := strings.ToLower(resource.GetObjectKind().GroupVersionKind().Kind); kind == "deployment" {
				req := resource.(*appsV1.Deployment)
				req.Spec.Replicas = replicas
				k8sObjects = append(k8sObjects, req.DeepCopyObject())
			}
		}
		if len(k8sObjects) > 0 {
			k8sResource = append(k8sResource, k8s.Resource{FileName: deployment.FileName, Objects: k8sObjects})
		}
	}
	return k8sResource
}

func (s *scale) scale(*kingpin.ParseContext) error {
	log.Printf("Starting Prombench-Scaler:\n\t max: %d\n\t min: %d\n\t interval: %s", s.max, s.min, s.interval)

	maxResourceObjects := s.updateReplicas(&s.max)
	minResourceObjects := s.updateReplicas(&s.min)

	for {
		log.Printf("Scaling Deployment to %d", s.max)
		if err := s.k8sClient.ResourceApply(maxResourceObjects); err != nil {
			fmt.Fprintln(os.Stderr, errors.Wrapf(err, "Error scaling deployment"))
		}

		time.Sleep(s.interval)

		log.Printf("Scaling Deployment to %d", s.min)
		if err := s.k8sClient.ResourceApply(minResourceObjects); err != nil {
			fmt.Fprintln(os.Stderr, errors.Wrapf(err, "Error scaling deployment"))
		}

		time.Sleep(s.interval)
	}
}

func main() {

	app := kingpin.New(filepath.Base(os.Args[0]), "The Prombench-Scaler tool")
	app.HelpFlag.Short('h')

	s := newScaler()

	k8sApp := app.Command("scale", "Scale a Kubernetes deployment object periodically up and down. \nex: ./scaler scale -v NAMESPACE:scale -f fake-webserver.yaml 20 1 15m").
		Action(s.k8sClient.DeploymentsParse).
		Action(s.scale)
	k8sApp.Flag("file", "yaml file or folder that describes the parameters for the deployment.").
		Required().
		Short('f').
		ExistingFilesOrDirsVar(&s.k8sClient.DeploymentFiles)
	k8sApp.Flag("vars", "When provided it will substitute the token holders in the yaml file. Follows the standard golang template formating - {{ .hashStable }}.").
		Short('v').
		StringMapVar(&s.k8sClient.DeploymentVars)
	k8sApp.Arg("max", "Number of Replicas to scale up.").
		Required().
		Int32Var(&s.max)
	k8sApp.Arg("min", "Number of Replicas to scale down.").
		Required().
		Int32Var(&s.min)
	k8sApp.Arg("interval", "Time to wait before changing the number of replicas.").
		Required().
		DurationVar(&s.interval)

	if _, err := app.Parse(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, errors.Wrapf(err, "Error parsing commandline arguments"))
		app.Usage(os.Args[1:])
		os.Exit(2)
	}
}
<|endoftext|>


<commit_before>package clock

import (
	"testing"
	"time"
	. "github.com/101loops/bdd"
)

var (
	delay     = 200 * time.Millisecond
	threshold = 100 * time.Millisecond
)

func TestSuite(t *testing.T) {
	RunSpecs(t, "Clock Suite")
}

func now() time.Time {
	return time.Now()
}

func timeDiff(c Clock) time.Duration {
	return time.Now().Sub(c.Now())
}

func durationOf(fn func()) time.Duration {
	beforeSleep := now()
	fn()
	return now().Sub(beforeSleep)
}
<commit_msg>go fmt<commit_after>package clock

import (
	. "github.com/101loops/bdd"
	"testing"
	"time"
)

var (
	delay     = 200 * time.Millisecond
	threshold = 100 * time.Millisecond
)

func TestSuite(t *testing.T) {
	RunSpecs(t, "Clock Suite")
}

func now() time.Time {
	return time.Now()
}

func timeDiff(c Clock) time.Duration {
	return time.Now().Sub(c.Now())
}

func durationOf(fn func()) time.Duration {
	beforeSleep := now()
	fn()
	return now().Sub(beforeSleep)
}
<|endoftext|>


<commit_before>/*
Package supervisor provides supervisor trees for Go applications.

This package is a clean reimplementation of github.com/thejerf/suture, aiming
to be more Go idiomatic, thus less Erlang-like.

It is built on top of context package, with all of its advantages, namely the
possibility trickle down context-related values and cancellation signals.

TheJerf's blog post about Suture is a very good and helpful read to understand
how this package has been implemented.

http://www.jerf.org/iri/post/2930
*/
package supervisor // import "cirello.io/supervisor"

import (
	"fmt"
	"log"
	"math"
	"sync"
	"time"

	"golang.org/x/net/context"
)

// Service is the public interface expected by a Supervisor.
//
// This will be internally named after the result of fmt.Stringer, if available.
// Otherwise it will going to use an internal representation for the service
// name.
type Service interface {
	// Serve is called by a Supervisor to start the service. It expects the
	// service to honor the passed context and its lifetime. Observe
	// <-ctx.Done() and ctx.Err(). If the service is stopped by anything
	// but the Supervisor, it will get started again. Be careful with shared
	// state among restarts.
	Serve(ctx context.Context)
}

// Supervisor is the basic datastructure responsible for offering a supervisor
// tree. It implements Service, therefore it can be nested if necessary. When
// passing the Supervisor around, remind to do it as reference (&supervisor).
type Supervisor struct {
	// Name for this supervisor tree, used for logging.
	Name string

	// FailureDecay is the timespan on which the current failure count will
	// be halved.
	FailureDecay float64

	// FailureThreshold is the maximum accepted number of failures, after
	// decay adjustment, that shall trigger the back-off wait.
	FailureThreshold float64

	// Backoff is the wait duration when hit threshold.
	Backoff time.Duration

	// Log is a replaceable function used for overall logging
	Log func(interface{})

	ready sync.Once

	addedService    chan struct{}
	startedServices chan struct{}

	runningServices sync.WaitGroup

	servicesMu sync.Mutex
	services   map[string]Service

	cancelationsMu sync.Mutex
	cancelations   map[string]context.CancelFunc

	backoffMu sync.Mutex
	backoff   map[string]*backoff

	runningMu sync.Mutex
}

func (s *Supervisor) String() string {
	return s.Name
}

func (s *Supervisor) prepare() {
	s.ready.Do(func() {
		if s.Name == "" {
			s.Name = "supervisor"
		}
		s.addedService = make(chan struct{}, 1)
		s.backoff = make(map[string]*backoff)
		s.cancelations = make(map[string]context.CancelFunc)
		s.services = make(map[string]Service)
		s.startedServices = make(chan struct{}, 1)

		if s.Log == nil {
			s.Log = func(msg interface{}) {
				log.Printf("%s: %v", s.Name, msg)
			}
		}
		if s.FailureDecay == 0 {
			s.FailureDecay = 30
		}
		if s.FailureThreshold == 0 {
			s.FailureThreshold = 5
		}
		if s.Backoff == 0 {
			s.Backoff = 15 * time.Second
		}
	})
}

// Add inserts into the Supervisor tree a new service. If the Supervisor is
// already started, it will start it automatically.
func (s *Supervisor) Add(service Service) {
	s.prepare()

	name := fmt.Sprintf("%s", service)

	s.servicesMu.Lock()
	s.backoffMu.Lock()
	s.backoff[name] = &backoff{}
	s.services[name] = service
	s.backoffMu.Unlock()
	s.servicesMu.Unlock()

	select {
	case s.addedService <- struct{}{}:
	default:
	}
}

// Remove stops the service in the Supervisor tree and remove from it.
func (s *Supervisor) Remove(name string) {
	s.prepare()

	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()
	if _, ok := s.services[name]; !ok {
		return
	}

	s.cancelationsMu.Lock()
	defer s.cancelationsMu.Unlock()
	if c, ok := s.cancelations[name]; ok {
		delete(s.cancelations, name)
		c()
	}
}

// Serve starts the Supervisor tree. It can be started only once at a time. If
// stopped (canceled), it can be restarted. In case of concurrent calls, it will
// hang until the current call is completed.
func (s *Supervisor) Serve(ctx context.Context) {
	s.prepare()

	s.runningMu.Lock()
	s.serve(ctx)
	s.runningMu.Unlock()
}

// Services return a list of services
func (s *Supervisor) Services() map[string]Service {
	svclist := make(map[string]Service)
	s.servicesMu.Lock()
	for k, v := range s.services {
		svclist[k] = v
	}
	s.servicesMu.Unlock()
	return svclist
}

// Cancelations return a list of services names and their cancellation calls
func (s *Supervisor) Cancelations() map[string]context.CancelFunc {
	svclist := make(map[string]context.CancelFunc)
	s.cancelationsMu.Lock()
	for k, v := range s.cancelations {
		svclist[k] = v
	}
	s.cancelationsMu.Unlock()
	return svclist
}

func (s *Supervisor) startServices(ctx context.Context) {
	s.startAllServices(ctx)
	select {
	case s.startedServices <- struct{}{}:
	default:
	}
}

func (s *Supervisor) serve(ctx context.Context) {
	select {
	case <-s.addedService:
	default:
	}
	s.startServices(ctx)

	go func(ctx context.Context) {
		for {
			select {
			case <-s.addedService:
				s.startServices(ctx)

			case <-ctx.Done():
				return
			}
		}
	}(ctx)
	<-ctx.Done()

	s.runningServices.Wait()

	s.cancelationsMu.Lock()
	s.cancelations = make(map[string]context.CancelFunc)
	s.cancelationsMu.Unlock()
	return
}

func (s *Supervisor) startAllServices(ctx context.Context) {
	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()
	s.cancelationsMu.Lock()
	defer s.cancelationsMu.Unlock()

	var wg sync.WaitGroup
	for name, svc := range s.services {
		if _, ok := s.cancelations[name]; ok {
			continue
		}

		wg.Add(1)
		go func(name string, svc Service) {
			s.runningServices.Add(1)
			wg.Done()
			for {
				retry := func() (retry bool) {
					defer func() {
						if r := recover(); r != nil {
							s.Log(fmt.Sprintf("trapped panic: %v", r))
							retry = true
						}
					}()

					c, cancel := context.WithCancel(ctx)
					s.cancelationsMu.Lock()
					s.cancelations[name] = cancel
					s.cancelationsMu.Unlock()
					svc.Serve(c)

					select {
					case <-ctx.Done():
						return false
					default:
						return true
					}
				}()
				if retry {
					s.Log(fmt.Sprintf("restarting %s", name))
					s.backoffMu.Lock()
					b := s.backoff[name]
					s.backoffMu.Unlock()
					b.wait(s.FailureDecay, s.FailureThreshold, s.Backoff, func(msg interface{}) {
						s.Log(fmt.Sprintf("backoff %s: %v", name, msg))
					})
					continue
				}
				break
			}
			s.runningServices.Done()
		}(name, svc)
	}

	wg.Wait()
}

type backoff struct {
	lastfail time.Time
	failures float64
}

func (b *backoff) wait(failureDecay float64, threshold float64, backoffDur time.Duration, log func(str interface{})) {
	if b.lastfail.IsZero() {
		b.lastfail = time.Now()
		b.failures = 1.0
		return
	}

	b.failures++
	intervals := time.Since(b.lastfail).Seconds() / failureDecay
	b.failures = b.failures*math.Pow(.5, intervals) + 1

	if b.failures > threshold {
		log(backoffDur)
		time.Sleep(backoffDur)
	}
}
<commit_msg>Avoid a nil pointer between exchange of supervisor and service supervisor<commit_after>/*
Package supervisor provides supervisor trees for Go applications.

This package is a clean reimplementation of github.com/thejerf/suture, aiming
to be more Go idiomatic, thus less Erlang-like.

It is built on top of context package, with all of its advantages, namely the
possibility trickle down context-related values and cancellation signals.

TheJerf's blog post about Suture is a very good and helpful read to understand
how this package has been implemented.

http://www.jerf.org/iri/post/2930
*/
package supervisor // import "cirello.io/supervisor"

import (
	"fmt"
	"log"
	"math"
	"sync"
	"time"

	"golang.org/x/net/context"
)

// Service is the public interface expected by a Supervisor.
//
// This will be internally named after the result of fmt.Stringer, if available.
// Otherwise it will going to use an internal representation for the service
// name.
type Service interface {
	// Serve is called by a Supervisor to start the service. It expects the
	// service to honor the passed context and its lifetime. Observe
	// <-ctx.Done() and ctx.Err(). If the service is stopped by anything
	// but the Supervisor, it will get started again. Be careful with shared
	// state among restarts.
	Serve(ctx context.Context)
}

// Supervisor is the basic datastructure responsible for offering a supervisor
// tree. It implements Service, therefore it can be nested if necessary. When
// passing the Supervisor around, remind to do it as reference (&supervisor).
type Supervisor struct {
	// Name for this supervisor tree, used for logging.
	Name string

	// FailureDecay is the timespan on which the current failure count will
	// be halved.
	FailureDecay float64

	// FailureThreshold is the maximum accepted number of failures, after
	// decay adjustment, that shall trigger the back-off wait.
	FailureThreshold float64

	// Backoff is the wait duration when hit threshold.
	Backoff time.Duration

	// Log is a replaceable function used for overall logging
	Log func(interface{})

	ready sync.Once

	addedService    chan struct{}
	startedServices chan struct{}

	runningServices sync.WaitGroup

	servicesMu sync.Mutex
	services   map[string]Service

	cancelationsMu sync.Mutex
	cancelations   map[string]context.CancelFunc

	backoffMu sync.Mutex
	backoff   map[string]*backoff

	runningMu sync.Mutex
}

func (s *Supervisor) String() string {
	return s.Name
}

func (s *Supervisor) prepare() {
	s.ready.Do(func() {
		if s.Name == "" {
			s.Name = "supervisor"
		}
		s.addedService = make(chan struct{}, 1)
		s.backoff = make(map[string]*backoff)
		s.cancelations = make(map[string]context.CancelFunc)
		s.services = make(map[string]Service)
		s.startedServices = make(chan struct{}, 1)

		if s.Log == nil {
			s.Log = func(msg interface{}) {
				log.Printf("%s: %v", s.Name, msg)
			}
		}
		if s.FailureDecay == 0 {
			s.FailureDecay = 30
		}
		if s.FailureThreshold == 0 {
			s.FailureThreshold = 5
		}
		if s.Backoff == 0 {
			s.Backoff = 15 * time.Second
		}
	})
}

// Add inserts into the Supervisor tree a new service. If the Supervisor is
// already started, it will start it automatically.
func (s *Supervisor) Add(service Service) {
	s.prepare()

	name := fmt.Sprintf("%s", service)

	s.servicesMu.Lock()
	s.backoffMu.Lock()
	s.backoff[name] = &backoff{}
	s.services[name] = service
	s.backoffMu.Unlock()
	s.servicesMu.Unlock()

	select {
	case s.addedService <- struct{}{}:
	default:
	}
}

// Remove stops the service in the Supervisor tree and remove from it.
func (s *Supervisor) Remove(name string) {
	s.prepare()

	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()
	if _, ok := s.services[name]; !ok {
		return
	}

	s.cancelationsMu.Lock()
	defer s.cancelationsMu.Unlock()
	if c, ok := s.cancelations[name]; ok {
		delete(s.cancelations, name)
		c()
	}
}

// Serve starts the Supervisor tree. It can be started only once at a time. If
// stopped (canceled), it can be restarted. In case of concurrent calls, it will
// hang until the current call is completed.
func (s *Supervisor) Serve(ctx context.Context) {
	s.prepare()

	s.runningMu.Lock()
	s.serve(ctx)
	s.runningMu.Unlock()
}

// Services return a list of services
func (s *Supervisor) Services() map[string]Service {
	svclist := make(map[string]Service)
	s.servicesMu.Lock()
	for k, v := range s.services {
		svclist[k] = v
	}
	s.servicesMu.Unlock()
	return svclist
}

// Cancelations return a list of services names and their cancellation calls
func (s *Supervisor) Cancelations() map[string]context.CancelFunc {
	svclist := make(map[string]context.CancelFunc)
	s.cancelationsMu.Lock()
	for k, v := range s.cancelations {
		svclist[k] = v
	}
	s.cancelationsMu.Unlock()
	return svclist
}

func (s *Supervisor) startServices(ctx context.Context) {
	s.startAllServices(ctx)
	select {
	case s.startedServices <- struct{}{}:
	default:
	}
}

func (s *Supervisor) serve(ctx context.Context) {
	select {
	case <-s.addedService:
	default:
	}
	s.startServices(ctx)

	go func(ctx context.Context) {
		for {
			select {
			case <-s.addedService:
				s.startServices(ctx)

			case <-ctx.Done():
				return
			}
		}
	}(ctx)
	<-ctx.Done()

	s.runningServices.Wait()

	s.cancelationsMu.Lock()
	s.cancelations = make(map[string]context.CancelFunc)
	s.cancelationsMu.Unlock()
	return
}

func (s *Supervisor) startAllServices(ctx context.Context) {
	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()
	s.cancelationsMu.Lock()
	defer s.cancelationsMu.Unlock()

	var wg sync.WaitGroup
	for name, svc := range s.services {
		if _, ok := s.cancelations[name]; ok {
			continue
		}

		intermediateCtx, cancel := context.WithCancel(ctx)
		s.cancelations[name] = cancel

		wg.Add(1)
		go func(name string, svc Service) {
			s.runningServices.Add(1)
			wg.Done()
			for {
				retry := func() (retry bool) {
					defer func() {
						if r := recover(); r != nil {
							s.Log(fmt.Sprintf("trapped panic: %v", r))
							retry = true
						}
					}()

					c, cancel := context.WithCancel(intermediateCtx)
					s.cancelationsMu.Lock()
					s.cancelations[name] = cancel
					s.cancelationsMu.Unlock()
					svc.Serve(c)

					select {
					case <-ctx.Done():
						return false
					default:
						return true
					}
				}()
				if retry {
					s.Log(fmt.Sprintf("restarting %s", name))
					s.backoffMu.Lock()
					b := s.backoff[name]
					s.backoffMu.Unlock()
					b.wait(s.FailureDecay, s.FailureThreshold, s.Backoff, func(msg interface{}) {
						s.Log(fmt.Sprintf("backoff %s: %v", name, msg))
					})
					continue
				}
				break
			}
			s.runningServices.Done()
		}(name, svc)
	}

	wg.Wait()
}

type backoff struct {
	lastfail time.Time
	failures float64
}

func (b *backoff) wait(failureDecay float64, threshold float64, backoffDur time.Duration, log func(str interface{})) {
	if b.lastfail.IsZero() {
		b.lastfail = time.Now()
		b.failures = 1.0
		return
	}

	b.failures++
	intervals := time.Since(b.lastfail).Seconds() / failureDecay
	b.failures = b.failures*math.Pow(.5, intervals) + 1

	if b.failures > threshold {
		log(backoffDur)
		time.Sleep(backoffDur)
	}
}
<|endoftext|>


<commit_before>package lib

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"github.com/BytemarkHosting/bytemark-client/util/log"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// RequestAlreadyRunError is returned if the Run method was already called for this Request.
type RequestAlreadyRunError struct {
	Request *Request
}

// InsecureConnectionError is returned if the endpoint isn't https but AllowInsecure was not called.
type InsecureConnectionError struct {
	Request *Request
}

func (e RequestAlreadyRunError) Error() string {
	return "A Request was Run twice"
}

func (e InsecureConnectionError) Error() string {
	return "A Request to an insecure endpoint was attempted when AllowInsecure had not been called."
}

// Request is the workhorse of the bytemark-client/lib - it builds up a request, then Run can be called to get its results.
type Request struct {
	authenticate  bool
	client        Client
	endpoint      Endpoint
	url           *url.URL
	method        string
	body          []byte
	allowInsecure bool
	hasRun        bool
}

// GetURL returns the URL that the Request is for.
func (r *Request) GetURL() url.URL {
	if r.url == nil {
		return url.URL{}
	}
	return *r.url
}

// BuildRequestNoAuth creates a new Request with the intention of not authenticating.
func (c *bytemarkClient) BuildRequestNoAuth(method string, endpoint Endpoint, path string, parts ...string) (r *Request, err error) {
	url, err := c.BuildURL(endpoint, path, parts...)
	if err != nil {
		return
	}
	return &Request{
		client:        c,
		endpoint:      endpoint,
		url:           url,
		method:        method,
		allowInsecure: c.allowInsecure,
	}, nil
}

// BuildRequest builds a request that will be authenticated by the endpoint given.
func (c *bytemarkClient) BuildRequest(method string, endpoint Endpoint, path string, parts ...string) (r *Request, err error) {
	url, err := c.BuildURL(endpoint, path, parts...)
	if err != nil {
		return
	}
	return &Request{
		authenticate:  true,
		client:        c,
		endpoint:      endpoint,
		url:           url,
		method:        method,
		allowInsecure: c.allowInsecure,
	}, nil
}

// AllowInsecure tells the Request that it's ok if the endpoint isn't communicated with over HTTPS.
func (r *Request) AllowInsecure() {
	r.allowInsecure = true
}

// mkHTTPClient creates an http.Client for this request. If the staging endpoint is used, InsecureSkipVerify is used because I guess we don't have a good cert for that brain.
func (r *Request) mkHTTPClient() (c *http.Client) {
	c = new(http.Client)
	if r.url.Host == "staging.bigv.io" {
		c.Transport = &http.Transport{
			// disable gas lint for this line (gas looks for insecure TLS settings, among other things)
			/* #nosec */
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		}
	}
	return c
}

// mkHTTPRequest assembles an http.Request for this Request, adding Authorization headers as needed, setting the Content-Type correctly for whichever endpoint it's talking to.
func (r *Request) mkHTTPRequest(body io.Reader) (req *http.Request, err error) {
	req, err = http.NewRequest(r.method, r.url.String(), body)
	if err != nil {
		return nil, err
	}
	req.Close = true
	req.Header.Add("User-Agent", "bytemark-client-"+Version)

	if r.endpoint == SPPEndpoint {
		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	} else {
		req.Header.Add("Accept", "application/json")
		req.Header.Add("Content-Type", "application/json")
	}
	if r.authenticate {
		if r.client.GetSessionToken() == "" {
			return nil, NilAuthError{}
		}
		// if we could settle on a single standard
		// rather than two basically-identical ones that'd be cool
		if r.endpoint == BillingEndpoint {
			req.Header.Add("Authorization", "Token token="+r.client.GetSessionToken())
		} else {
			req.Header.Add("Authorization", "Bearer "+r.client.GetSessionToken())
		}
	}
	return
}

// MarshalAndRun marshals the 'in' object and passes that and 'out' to Run as the body and responseObject, respectively.
func (r *Request) MarshalAndRun(in interface{}, out interface{}) (statusCode int, responseBody []byte, err error) {
	var b bytes.Buffer
	err = json.NewEncoder(&b).Encode(in)
	if err != nil {
		return
	}

	return r.Run(&b, out)
}

// Run performs the request with the given body, and attempts to unmarshal a successful response into responseObject
func (r *Request) Run(body io.Reader, responseObject interface{}) (statusCode int, responseBody []byte, err error) {
	if r.hasRun {
		err = RequestAlreadyRunError{r}
		return
	}
	r.hasRun = true

	if !r.allowInsecure && r.url.Scheme == "http" {
		err = InsecureConnectionError{r}
		return
	}
	var rb []byte
	if body != nil {

		rb, err = ioutil.ReadAll(body)
		if err != nil {
			return 0, nil, err
		}
		log.Debugf(log.LvlHTTPData, "request body: '%s'\r\n", string(rb))
	}

	cli := r.mkHTTPClient()

	req, err := r.mkHTTPRequest(bytes.NewBuffer(rb))
	if err != nil {
		return
	}
	if len(rb) > 0 {
		req.Header.Add("Content-Length", fmt.Sprintf("%d", len(rb)))
	}
	res, err := cli.Do(req)
	if err != nil {
		return
	}

	statusCode = res.StatusCode

	log.Debugf(log.LvlOutline, "%s %s: %d\r\n", r.method, req.URL, res.StatusCode)

	responseBody, err = r.handleResponse(req, rb, res, responseObject)
	return
}

// handleResponse deals with the response coming back from the request - creating an error if required, unmarshalling responseObject if necessary
func (r *Request) handleResponse(req *http.Request, requestBody []byte, res *http.Response, responseObject interface{}) (body []byte, err error) {
	body, err = ioutil.ReadAll(res.Body)
	log.Debugf(log.LvlHTTPData, "response body: '%s'\r\n", body)
	if err != nil {
		return
	}

	baseErr := APIError{
		Method:      r.method,
		URL:         req.URL,
		StatusCode:  res.StatusCode,
		RequestBody: string(requestBody),
	}

	baseErr.ResponseBody = string(body)

	switch res.StatusCode {
	case 400:
		// because we need to reference fields specific to BadRequestError later
		err = newBadRequestError(baseErr, body)
	case 401:
		err = UnauthorizedError{baseErr}
	case 403:
		err = ForbiddenError{baseErr}
	case 404:
		err = NotFoundError{baseErr}
	case 500:
		err = InternalServerError{baseErr}
	case 503:
		err = ServiceUnavailableError{baseErr}
	default:
		if 200 <= res.StatusCode && res.StatusCode <= 299 {
			if responseObject != nil {
				jsonErr := json.Unmarshal(body, responseObject)
				if jsonErr != nil {
					return body, jsonErr
				}
			}
		} else {
			err = UnknownStatusCodeError{baseErr}
		}
	}
	return
}

// BuildURL pieces together a URL from parts, escaping as necessary..
func (c *bytemarkClient) BuildURL(endpoint Endpoint, format string, args ...string) (*url.URL, error) {
	arr := make([]interface{}, len(args))
	for i, str := range args {
		arr[i] = url.QueryEscape(str)
	}
	endpointURL := ""
	switch endpoint {
	case APIEndpoint:
		endpointURL = c.urls.API
	case BrainEndpoint:
		endpointURL = c.urls.Brain
	case BillingEndpoint:
		endpointURL = c.urls.Billing
	case SPPEndpoint:
		endpointURL = c.urls.SPP
	default:
		return nil, UnsupportedEndpointError(endpoint)
	}
	if !strings.HasPrefix(format, "/") {
		return nil, UnsupportedEndpointError(-1)
	}
	return url.Parse(endpointURL + fmt.Sprintf(format, arr...))
}
<commit_msg>Fix error message about insecure requests<commit_after>package lib

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"github.com/BytemarkHosting/bytemark-client/util/log"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// RequestAlreadyRunError is returned if the Run method was already called for this Request.
type RequestAlreadyRunError struct {
	Request *Request
}

// InsecureConnectionError is returned if the endpoint isn't https but AllowInsecure was not called.
type InsecureConnectionError struct {
	Request *Request
}

func (e RequestAlreadyRunError) Error() string {
	return "A Request was Run twice"
}

func (e InsecureConnectionError) Error() string {
	return "A Request to an insecure endpoint was attempted when AllowInsecureRequests had not been called."
}

// Request is the workhorse of the bytemark-client/lib - it builds up a request, then Run can be called to get its results.
type Request struct {
	authenticate  bool
	client        Client
	endpoint      Endpoint
	url           *url.URL
	method        string
	body          []byte
	allowInsecure bool
	hasRun        bool
}

// GetURL returns the URL that the Request is for.
func (r *Request) GetURL() url.URL {
	if r.url == nil {
		return url.URL{}
	}
	return *r.url
}

// BuildRequestNoAuth creates a new Request with the intention of not authenticating.
func (c *bytemarkClient) BuildRequestNoAuth(method string, endpoint Endpoint, path string, parts ...string) (r *Request, err error) {
	url, err := c.BuildURL(endpoint, path, parts...)
	if err != nil {
		return
	}
	return &Request{
		client:        c,
		endpoint:      endpoint,
		url:           url,
		method:        method,
		allowInsecure: c.allowInsecure,
	}, nil
}

// BuildRequest builds a request that will be authenticated by the endpoint given.
func (c *bytemarkClient) BuildRequest(method string, endpoint Endpoint, path string, parts ...string) (r *Request, err error) {
	url, err := c.BuildURL(endpoint, path, parts...)
	if err != nil {
		return
	}
	return &Request{
		authenticate:  true,
		client:        c,
		endpoint:      endpoint,
		url:           url,
		method:        method,
		allowInsecure: c.allowInsecure,
	}, nil
}

// AllowInsecure tells the Request that it's ok if the endpoint isn't communicated with over HTTPS.
func (r *Request) AllowInsecure() {
	r.allowInsecure = true
}

// mkHTTPClient creates an http.Client for this request. If the staging endpoint is used, InsecureSkipVerify is used because I guess we don't have a good cert for that brain.
func (r *Request) mkHTTPClient() (c *http.Client) {
	c = new(http.Client)
	if r.url.Host == "staging.bigv.io" {
		c.Transport = &http.Transport{
			// disable gas lint for this line (gas looks for insecure TLS settings, among other things)
			/* #nosec */
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		}
	}
	return c
}

// mkHTTPRequest assembles an http.Request for this Request, adding Authorization headers as needed, setting the Content-Type correctly for whichever endpoint it's talking to.
func (r *Request) mkHTTPRequest(body io.Reader) (req *http.Request, err error) {
	req, err = http.NewRequest(r.method, r.url.String(), body)
	if err != nil {
		return nil, err
	}
	req.Close = true
	req.Header.Add("User-Agent", "bytemark-client-"+Version)

	if r.endpoint == SPPEndpoint {
		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	} else {
		req.Header.Add("Accept", "application/json")
		req.Header.Add("Content-Type", "application/json")
	}
	if r.authenticate {
		if r.client.GetSessionToken() == "" {
			return nil, NilAuthError{}
		}
		// if we could settle on a single standard
		// rather than two basically-identical ones that'd be cool
		if r.endpoint == BillingEndpoint {
			req.Header.Add("Authorization", "Token token="+r.client.GetSessionToken())
		} else {
			req.Header.Add("Authorization", "Bearer "+r.client.GetSessionToken())
		}
	}
	return
}

// MarshalAndRun marshals the 'in' object and passes that and 'out' to Run as the body and responseObject, respectively.
func (r *Request) MarshalAndRun(in interface{}, out interface{}) (statusCode int, responseBody []byte, err error) {
	var b bytes.Buffer
	err = json.NewEncoder(&b).Encode(in)
	if err != nil {
		return
	}

	return r.Run(&b, out)
}

// Run performs the request with the given body, and attempts to unmarshal a successful response into responseObject
func (r *Request) Run(body io.Reader, responseObject interface{}) (statusCode int, responseBody []byte, err error) {
	if r.hasRun {
		err = RequestAlreadyRunError{r}
		return
	}
	r.hasRun = true

	if !r.allowInsecure && r.url.Scheme == "http" {
		err = InsecureConnectionError{r}
		return
	}
	var rb []byte
	if body != nil {

		rb, err = ioutil.ReadAll(body)
		if err != nil {
			return 0, nil, err
		}
		log.Debugf(log.LvlHTTPData, "request body: '%s'\r\n", string(rb))
	}

	cli := r.mkHTTPClient()

	req, err := r.mkHTTPRequest(bytes.NewBuffer(rb))
	if err != nil {
		return
	}
	if len(rb) > 0 {
		req.Header.Add("Content-Length", fmt.Sprintf("%d", len(rb)))
	}
	res, err := cli.Do(req)
	if err != nil {
		return
	}

	statusCode = res.StatusCode

	log.Debugf(log.LvlOutline, "%s %s: %d\r\n", r.method, req.URL, res.StatusCode)

	responseBody, err = r.handleResponse(req, rb, res, responseObject)
	return
}

// handleResponse deals with the response coming back from the request - creating an error if required, unmarshalling responseObject if necessary
func (r *Request) handleResponse(req *http.Request, requestBody []byte, res *http.Response, responseObject interface{}) (body []byte, err error) {
	body, err = ioutil.ReadAll(res.Body)
	log.Debugf(log.LvlHTTPData, "response body: '%s'\r\n", body)
	if err != nil {
		return
	}

	baseErr := APIError{
		Method:      r.method,
		URL:         req.URL,
		StatusCode:  res.StatusCode,
		RequestBody: string(requestBody),
	}

	baseErr.ResponseBody = string(body)

	switch res.StatusCode {
	case 400:
		// because we need to reference fields specific to BadRequestError later
		err = newBadRequestError(baseErr, body)
	case 401:
		err = UnauthorizedError{baseErr}
	case 403:
		err = ForbiddenError{baseErr}
	case 404:
		err = NotFoundError{baseErr}
	case 500:
		err = InternalServerError{baseErr}
	case 503:
		err = ServiceUnavailableError{baseErr}
	default:
		if 200 <= res.StatusCode && res.StatusCode <= 299 {
			if responseObject != nil {
				jsonErr := json.Unmarshal(body, responseObject)
				if jsonErr != nil {
					return body, jsonErr
				}
			}
		} else {
			err = UnknownStatusCodeError{baseErr}
		}
	}
	return
}

// BuildURL pieces together a URL from parts, escaping as necessary..
func (c *bytemarkClient) BuildURL(endpoint Endpoint, format string, args ...string) (*url.URL, error) {
	arr := make([]interface{}, len(args))
	for i, str := range args {
		arr[i] = url.QueryEscape(str)
	}
	endpointURL := ""
	switch endpoint {
	case APIEndpoint:
		endpointURL = c.urls.API
	case BrainEndpoint:
		endpointURL = c.urls.Brain
	case BillingEndpoint:
		endpointURL = c.urls.Billing
	case SPPEndpoint:
		endpointURL = c.urls.SPP
	default:
		return nil, UnsupportedEndpointError(endpoint)
	}
	if !strings.HasPrefix(format, "/") {
		return nil, UnsupportedEndpointError(-1)
	}
	return url.Parse(endpointURL + fmt.Sprintf(format, arr...))
}
<|endoftext|>


<commit_before>package cli

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/schollz/croc/v6/src/croc"
	"github.com/schollz/croc/v6/src/models"
	"github.com/schollz/croc/v6/src/tcp"
	"github.com/schollz/croc/v6/src/utils"
	log "github.com/schollz/logger"
	"github.com/urfave/cli"
)

var Version string

func Run() (err error) {
	// use all of the processors
	runtime.GOMAXPROCS(runtime.NumCPU())

	app := cli.NewApp()
	app.Name = "croc"
	if Version == "" {
		Version = "v6.0.12-fbbdd1b"
	}
	app.Version = Version
	app.Compiled = time.Now()
	app.Usage = "easily and securely transfer stuff from one computer to another"
	app.UsageText = "croc allows any two computers to directly and securely transfer files"
	// app.ArgsUsage = "[args and such]"
	app.Commands = []cli.Command{
		{
			Name:        "send",
			Usage:       "send a file",
			Description: "send a file over the relay",
			ArgsUsage:   "[filename]",
			Flags: []cli.Flag{
				cli.StringFlag{Name: "code, c", Usage: "codephrase used to connect to relay"},
				cli.BoolFlag{Name: "no-local", Usage: "disable local relay when sending"},
				cli.StringFlag{Name: "ports", Value: "9009,9010,9011,9012,9013", Usage: "ports of the local relay (optional)"},
			},
			HelpName: "croc send",
			Action: func(c *cli.Context) error {
				return send(c)
			},
		},
		{
			Name:        "relay",
			Description: "start relay",
			HelpName:    "croc relay",
			Action: func(c *cli.Context) error {
				return relay(c)
			},
			Flags: []cli.Flag{
				cli.StringFlag{Name: "ports", Value: "9009,9010,9011,9012,9013", Usage: "ports of the relay"},
			},
		},
	}
	app.Flags = []cli.Flag{
		cli.BoolFlag{Name: "remember", Usage: "save these settings to reuse next time"},
		cli.BoolFlag{Name: "debug", Usage: "increase verbosity (a lot)"},
		cli.BoolFlag{Name: "yes", Usage: "automatically agree to all prompts"},
		cli.BoolFlag{Name: "stdout", Usage: "redirect file to stdout"},
		cli.StringFlag{Name: "relay", Value: models.DEFAULT_RELAY, Usage: "address of the relay"},
		cli.StringFlag{Name: "out", Value: ".", Usage: "specify an output folder to receive the file"},
	}
	app.EnableBashCompletion = true
	app.HideHelp = false
	app.HideVersion = false
	app.BashComplete = func(c *cli.Context) {
		fmt.Fprintf(c.App.Writer, "send\nreceive\relay")
	}
	app.Action = func(c *cli.Context) error {
		// if trying to send but forgot send, let the user know
		if c.Args().First() != "" && utils.Exists(c.Args().First()) {
			_, fname := filepath.Split(c.Args().First())
			yn := utils.GetInput(fmt.Sprintf("Did you mean to send '%s'? (y/n) ", fname))
			if strings.ToLower(yn) == "y" {
				return send(c)
			}
		}
		return receive(c)
	}

	return app.Run(os.Args)
}

// func saveDefaultConfig(c *cli.Context) error {
// 	return croc.SaveDefaultConfig()
// }

func getConfigDir() (homedir string, err error) {
	homedir, err = os.UserHomeDir()
	if err != nil {
		log.Error(err)
		return
	}
	homedir = path.Join(homedir, ".config", "croc")
	if _, err := os.Stat(homedir); os.IsNotExist(err) {
		log.Debugf("creating home directory %s", homedir)
		err = os.MkdirAll(homedir, 0700)
	}
	return
}

func send(c *cli.Context) (err error) {
	if c.GlobalBool("debug") {
		log.SetLevel("debug")
		log.Debug("debug mode on")
	}
	configFile, err := getConfigDir()
	if err != nil {
		log.Error(err)
		return
	}
	configFile = path.Join(configFile, "send.json")

	crocOptions := croc.Options{
		SharedSecret: c.String("code"),
		IsSender:     true,
		Debug:        c.GlobalBool("debug"),
		NoPrompt:     c.GlobalBool("yes"),
		RelayAddress: c.GlobalString("relay"),
		Stdout:       c.GlobalBool("stdout"),
		DisableLocal: c.Bool("no-local"),
		RelayPorts:   strings.Split(c.String("ports"), ","),
	}
	b, errOpen := ioutil.ReadFile(configFile)
	if errOpen == nil && !c.GlobalBool("remember") {
		var rememberedOptions croc.Options
		err = json.Unmarshal(b, &rememberedOptions)
		if err != nil {
			log.Error(err)
			return
		}
		// update anything that isn't explicitly set
		if !c.GlobalIsSet("relay") {
			crocOptions.RelayAddress = rememberedOptions.RelayAddress
		}
		if !c.IsSet("no-local") {
			crocOptions.DisableLocal = rememberedOptions.DisableLocal
		}
		if !c.IsSet("ports") {
			crocOptions.RelayPorts = rememberedOptions.RelayPorts
		}
		if !c.IsSet("code") {
			crocOptions.SharedSecret = rememberedOptions.SharedSecret
		}
	}

	var fnames []string
	stat, _ := os.Stdin.Stat()
	if (stat.Mode() & os.ModeCharDevice) == 0 {
		f, err := ioutil.TempFile(".", "croc-stdin-")
		if err != nil {
			return err
		}
		_, err = io.Copy(f, os.Stdin)
		if err != nil {
			return err
		}
		err = f.Close()
		if err != nil {
			return err
		}
		fnames = []string{f.Name()}
		defer func() {
			err = os.Remove(fnames[0])
			if err != nil {
				log.Error(err)
			}
		}()
	} else {
		fnames = append([]string{c.Args().First()}, c.Args().Tail()...)
	}
	if len(fnames) == 0 {
		return errors.New("must specify file: croc send [filename]")
	}

	if len(crocOptions.SharedSecret) == 0 {
		// generate code phrase
		crocOptions.SharedSecret = utils.GetRandomName()
	}

	haveFolder := false
	paths := []string{}
	for _, fname := range fnames {
		stat, err := os.Stat(fname)
		if err != nil {
			return err
		}
		if stat.IsDir() {
			haveFolder = true
			err = filepath.Walk(fname,
				func(pathName string, info os.FileInfo, err error) error {
					if err != nil {
						return err
					}
					if !info.IsDir() {
						paths = append(paths, filepath.ToSlash(pathName))
					}
					return nil
				})
			if err != nil {
				return err
			}
		} else {
			paths = append(paths, filepath.ToSlash(fname))
		}
	}

	cr, err := croc.New(crocOptions)
	if err != nil {
		return
	}

	// save the config
	if c.GlobalBool("remember") {
		log.Debug("saving config file")
		var bConfig []byte
		// if the code wasn't set, don't save it
		if c.String("code") == "" {
			crocOptions.SharedSecret = ""
		}
		bConfig, err = json.MarshalIndent(crocOptions, "", "    ")
		if err != nil {
			log.Error(err)
			return
		}
		err = ioutil.WriteFile(configFile, bConfig, 0644)
		if err != nil {
			log.Error(err)
			return
		}
		log.Debugf("wrote %s", configFile)
	}

	err = cr.Send(croc.TransferOptions{
		PathToFiles:      paths,
		KeepPathInRemote: haveFolder,
	})

	return
}

func receive(c *cli.Context) (err error) {
	var sharedSecret string
	if c.String("code") != "" {
		sharedSecret = c.String("code")
	}
	if c.Args().First() != "" {
		sharedSecret = c.Args().First()
	}
	if sharedSecret == "" {
		sharedSecret = utils.GetInput("Enter receive code: ")
	}
	if c.GlobalString("out") != "" {
		os.Chdir(c.GlobalString("out"))
	}

	cr, err := croc.New(croc.Options{
		SharedSecret: sharedSecret,
		IsSender:     false,
		Debug:        c.GlobalBool("debug"),
		NoPrompt:     c.GlobalBool("yes"),
		RelayAddress: c.GlobalString("relay"),
		Stdout:       c.GlobalBool("stdout"),
	})
	if err != nil {
		return
	}
	err = cr.Receive()
	return
}

func relay(c *cli.Context) (err error) {
	debugString := "info"
	if c.GlobalBool("debug") {
		debugString = "debug"
	}
	ports := strings.Split(c.String("ports"), ",")
	tcpPorts := strings.Join(ports[1:], ",")
	for i, port := range ports {
		if i == 0 {
			continue
		}
		go func(portStr string) {
			err = tcp.Run(debugString, portStr)
			if err != nil {
				panic(err)
			}
		}(port)
	}
	return tcp.Run(debugString, ports[0], tcpPorts)
}

// func dirSize(path string) (int64, error) {
// 	var size int64
// 	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
// 		if !info.IsDir() {
// 			size += info.Size()
// 		}
// 		return err
// 	})
// 	return size, err
// }
<commit_msg>add receiver config<commit_after>package cli

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/schollz/croc/v6/src/croc"
	"github.com/schollz/croc/v6/src/models"
	"github.com/schollz/croc/v6/src/tcp"
	"github.com/schollz/croc/v6/src/utils"
	log "github.com/schollz/logger"
	"github.com/urfave/cli"
)

var Version string

func Run() (err error) {
	// use all of the processors
	runtime.GOMAXPROCS(runtime.NumCPU())

	app := cli.NewApp()
	app.Name = "croc"
	if Version == "" {
		Version = "v6.0.12-fbbdd1b"
	}
	app.Version = Version
	app.Compiled = time.Now()
	app.Usage = "easily and securely transfer stuff from one computer to another"
	app.UsageText = "croc allows any two computers to directly and securely transfer files"
	// app.ArgsUsage = "[args and such]"
	app.Commands = []cli.Command{
		{
			Name:        "send",
			Usage:       "send a file",
			Description: "send a file over the relay",
			ArgsUsage:   "[filename]",
			Flags: []cli.Flag{
				cli.StringFlag{Name: "code, c", Usage: "codephrase used to connect to relay"},
				cli.BoolFlag{Name: "no-local", Usage: "disable local relay when sending"},
				cli.StringFlag{Name: "ports", Value: "9009,9010,9011,9012,9013", Usage: "ports of the local relay (optional)"},
			},
			HelpName: "croc send",
			Action: func(c *cli.Context) error {
				return send(c)
			},
		},
		{
			Name:        "relay",
			Description: "start relay",
			HelpName:    "croc relay",
			Action: func(c *cli.Context) error {
				return relay(c)
			},
			Flags: []cli.Flag{
				cli.StringFlag{Name: "ports", Value: "9009,9010,9011,9012,9013", Usage: "ports of the relay"},
			},
		},
	}
	app.Flags = []cli.Flag{
		cli.BoolFlag{Name: "remember", Usage: "save these settings to reuse next time"},
		cli.BoolFlag{Name: "debug", Usage: "increase verbosity (a lot)"},
		cli.BoolFlag{Name: "yes", Usage: "automatically agree to all prompts"},
		cli.BoolFlag{Name: "stdout", Usage: "redirect file to stdout"},
		cli.StringFlag{Name: "relay", Value: models.DEFAULT_RELAY, Usage: "address of the relay"},
		cli.StringFlag{Name: "out", Value: ".", Usage: "specify an output folder to receive the file"},
	}
	app.EnableBashCompletion = true
	app.HideHelp = false
	app.HideVersion = false
	app.BashComplete = func(c *cli.Context) {
		fmt.Fprintf(c.App.Writer, "send\nreceive\relay")
	}
	app.Action = func(c *cli.Context) error {
		// if trying to send but forgot send, let the user know
		if c.Args().First() != "" && utils.Exists(c.Args().First()) {
			_, fname := filepath.Split(c.Args().First())
			yn := utils.GetInput(fmt.Sprintf("Did you mean to send '%s'? (y/n) ", fname))
			if strings.ToLower(yn) == "y" {
				return send(c)
			}
		}
		return receive(c)
	}

	return app.Run(os.Args)
}

// func saveDefaultConfig(c *cli.Context) error {
// 	return croc.SaveDefaultConfig()
// }

func getConfigDir() (homedir string, err error) {
	homedir, err = os.UserHomeDir()
	if err != nil {
		log.Error(err)
		return
	}
	homedir = path.Join(homedir, ".config", "croc")
	if _, err := os.Stat(homedir); os.IsNotExist(err) {
		log.Debugf("creating home directory %s", homedir)
		err = os.MkdirAll(homedir, 0700)
	}
	return
}

func send(c *cli.Context) (err error) {
	if c.GlobalBool("debug") {
		log.SetLevel("debug")
		log.Debug("debug mode on")
	}
	configFile, err := getConfigDir()
	if err != nil {
		log.Error(err)
		return
	}
	configFile = path.Join(configFile, "send.json")

	crocOptions := croc.Options{
		SharedSecret: c.String("code"),
		IsSender:     true,
		Debug:        c.GlobalBool("debug"),
		NoPrompt:     c.GlobalBool("yes"),
		RelayAddress: c.GlobalString("relay"),
		Stdout:       c.GlobalBool("stdout"),
		DisableLocal: c.Bool("no-local"),
		RelayPorts:   strings.Split(c.String("ports"), ","),
	}
	b, errOpen := ioutil.ReadFile(configFile)
	if errOpen == nil && !c.GlobalBool("remember") {
		var rememberedOptions croc.Options
		err = json.Unmarshal(b, &rememberedOptions)
		if err != nil {
			log.Error(err)
			return
		}
		// update anything that isn't explicitly set
		if !c.GlobalIsSet("relay") {
			crocOptions.RelayAddress = rememberedOptions.RelayAddress
		}
		if !c.IsSet("no-local") {
			crocOptions.DisableLocal = rememberedOptions.DisableLocal
		}
		if !c.IsSet("ports") {
			crocOptions.RelayPorts = rememberedOptions.RelayPorts
		}
		if !c.IsSet("code") {
			crocOptions.SharedSecret = rememberedOptions.SharedSecret
		}
	}

	var fnames []string
	stat, _ := os.Stdin.Stat()
	if (stat.Mode() & os.ModeCharDevice) == 0 {
		f, err := ioutil.TempFile(".", "croc-stdin-")
		if err != nil {
			return err
		}
		_, err = io.Copy(f, os.Stdin)
		if err != nil {
			return err
		}
		err = f.Close()
		if err != nil {
			return err
		}
		fnames = []string{f.Name()}
		defer func() {
			err = os.Remove(fnames[0])
			if err != nil {
				log.Error(err)
			}
		}()
	} else {
		fnames = append([]string{c.Args().First()}, c.Args().Tail()...)
	}
	if len(fnames) == 0 {
		return errors.New("must specify file: croc send [filename]")
	}

	if len(crocOptions.SharedSecret) == 0 {
		// generate code phrase
		crocOptions.SharedSecret = utils.GetRandomName()
	}

	haveFolder := false
	paths := []string{}
	for _, fname := range fnames {
		stat, err := os.Stat(fname)
		if err != nil {
			return err
		}
		if stat.IsDir() {
			haveFolder = true
			err = filepath.Walk(fname,
				func(pathName string, info os.FileInfo, err error) error {
					if err != nil {
						return err
					}
					if !info.IsDir() {
						paths = append(paths, filepath.ToSlash(pathName))
					}
					return nil
				})
			if err != nil {
				return err
			}
		} else {
			paths = append(paths, filepath.ToSlash(fname))
		}
	}

	cr, err := croc.New(crocOptions)
	if err != nil {
		return
	}

	// save the config
	if c.GlobalBool("remember") {
		log.Debug("saving config file")
		var bConfig []byte
		// if the code wasn't set, don't save it
		if c.String("code") == "" {
			crocOptions.SharedSecret = ""
		}
		bConfig, err = json.MarshalIndent(crocOptions, "", "    ")
		if err != nil {
			log.Error(err)
			return
		}
		err = ioutil.WriteFile(configFile, bConfig, 0644)
		if err != nil {
			log.Error(err)
			return
		}
		log.Debugf("wrote %s", configFile)
	}

	err = cr.Send(croc.TransferOptions{
		PathToFiles:      paths,
		KeepPathInRemote: haveFolder,
	})

	return
}

func receive(c *cli.Context) (err error) {
	crocOptions := croc.Options{
		SharedSecret: c.String("code"),
		IsSender:     false,
		Debug:        c.GlobalBool("debug"),
		NoPrompt:     c.GlobalBool("yes"),
		RelayAddress: c.GlobalString("relay"),
		Stdout:       c.GlobalBool("stdout"),
	}
	if c.Args().First() != "" {
		crocOptions.SharedSecret = c.Args().First()
	}

	// load options here
	if c.GlobalBool("debug") {
		log.SetLevel("debug")
		log.Debug("debug mode on")
	}
	configFile, err := getConfigDir()
	if err != nil {
		log.Error(err)
		return
	}
	configFile = path.Join(configFile, "receive.json")
	b, errOpen := ioutil.ReadFile(configFile)
	if errOpen == nil && !c.GlobalBool("remember") {
		var rememberedOptions croc.Options
		err = json.Unmarshal(b, &rememberedOptions)
		if err != nil {
			log.Error(err)
			return
		}
		// update anything that isn't explicitly set
		if !c.GlobalIsSet("relay") {
			crocOptions.RelayAddress = rememberedOptions.RelayAddress
		}
		if crocOptions.SharedSecret == "" {
			crocOptions.SharedSecret = rememberedOptions.SharedSecret
		}
	}

	if crocOptions.SharedSecret == "" {
		crocOptions.SharedSecret = utils.GetInput("Enter receive code: ")
	}
	if c.GlobalString("out") != "" {
		os.Chdir(c.GlobalString("out"))
	}

	cr, err := croc.New(crocOptions)

	if err != nil {
		return
	}
	err = cr.Receive()
	return
}

func relay(c *cli.Context) (err error) {
	debugString := "info"
	if c.GlobalBool("debug") {
		debugString = "debug"
	}
	ports := strings.Split(c.String("ports"), ",")
	tcpPorts := strings.Join(ports[1:], ",")
	for i, port := range ports {
		if i == 0 {
			continue
		}
		go func(portStr string) {
			err = tcp.Run(debugString, portStr)
			if err != nil {
				panic(err)
			}
		}(port)
	}
	return tcp.Run(debugString, ports[0], tcpPorts)
}

// func dirSize(path string) (int64, error) {
// 	var size int64
// 	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
// 		if !info.IsDir() {
// 			size += info.Size()
// 		}
// 		return err
// 	})
// 	return size, err
// }
<|endoftext|>


<commit_before>package git

import (
	"reflect"
	"testing"

	"github.com/ironman-project/ironman/template/repository"
)

func TestNew(t *testing.T) {
	type args struct {
		baseRepository *repository.BaseRepository
	}
	tests := []struct {
		name string
		args args
		want repository.Repository
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := New(tt.args.baseRepository); !reflect.DeepEqual(got, tt.want) {
				t.Errorf("New() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestRepository_Install(t *testing.T) {
	type fields struct {
		BaseRepository *repository.BaseRepository
	}
	type args struct {
		location string
	}
	tests := []struct {
		name    string
		fields  fields
		args    args
		wantErr bool
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			r := &Repository{
				BaseRepository: tt.fields.BaseRepository,
			}
			if err := r.Install(tt.args.location); (err != nil) != tt.wantErr {
				t.Errorf("Repository.Install() error = %v, wantErr %v", err, tt.wantErr)
			}
		})
	}
}

func TestRepository_Update(t *testing.T) {
	type fields struct {
		BaseRepository *repository.BaseRepository
	}
	type args struct {
		name string
	}
	tests := []struct {
		name    string
		fields  fields
		args    args
		wantErr bool
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			r := &Repository{
				BaseRepository: tt.fields.BaseRepository,
			}
			if err := r.Update(tt.args.name); (err != nil) != tt.wantErr {
				t.Errorf("Repository.Update() error = %v, wantErr %v", err, tt.wantErr)
			}
		})
	}
}
<commit_msg>#9 add basic tests for git repository install<commit_after>package git

import (
	"os"
	"testing"

	"github.com/ironman-project/ironman/template/repository"
)

func newTestGitRepository() repository.Repository {
	baseRepository := repository.NewBaseRepository("testing")
	return New(baseRepository)
}

func TestRepository_Install(t *testing.T) {

	type args struct {
		location string
	}
	tests := []struct {
		name               string
		args               args
		expectedTemplateID string
		wantErr            bool
	}{
		{
			"Install template",
			args{"https://github.com/ottogiron/wizard-hello-world.git"},
			"wizard-hello-world",
			false,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			r := newTestGitRepository()
			defer func() {
				_ = r.Uninstall(tt.expectedTemplateID)
			}()
			if err := r.Install(tt.args.location); (err != nil) != tt.wantErr {
				t.Errorf("Repository.Install() error = %v, wantErr %v", err, tt.wantErr)
				return
			}

			expectedTemplatePath := r.TemplatePath(tt.expectedTemplateID)
			if _, err := os.Stat(expectedTemplatePath); os.IsNotExist(err) {
				t.Errorf("Repository.Install() template was not installed want path %v", expectedTemplatePath)
			}
		})
	}
}

func TestRepository_Update(t *testing.T) {
	type fields struct {
		BaseRepository *repository.BaseRepository
	}
	type args struct {
		name string
	}
	tests := []struct {
		name    string
		fields  fields
		args    args
		wantErr bool
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			r := &Repository{
				BaseRepository: tt.fields.BaseRepository,
			}
			if err := r.Update(tt.args.name); (err != nil) != tt.wantErr {
				t.Errorf("Repository.Update() error = %v, wantErr %v", err, tt.wantErr)
			}
		})
	}
}
<|endoftext|>


<commit_before>package lib

// Version is the github-nippou version
const Version = "4.1.10"
<commit_msg>:up: Bump version to 4.1.11<commit_after>package lib

// Version is the github-nippou version
const Version = "4.1.11"
<|endoftext|>


<commit_before>// +build e2e

/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package conformance

import (
	"context"
	"fmt"
	"net/http"
	"strings"
	"testing"

	pkgTest "github.com/knative/pkg/test"
	"github.com/knative/pkg/test/logging"
	"github.com/knative/pkg/test/spoof"
	"github.com/knative/serving/test"
	"golang.org/x/sync/errgroup"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	// Mysteriously required to support GCP auth (required by k8s libs). Apparently just importing it is enough. @_@ side effects @_@. https://github.com/kubernetes/client-go/issues/242
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
)

const (
	concurrentRequests = 100
	// We expect to see 100% of requests succeed for traffic sent directly to revisions.
	// This might be a bad assumption.
	minDirectPercentage = 1
	// We expect to see at least 25% of either response since we're routing 50/50.
	// This might be a bad assumption.
	minSplitPercentage = 0.25

	expectedBlue  = "What a spaceport!"
	expectedGreen = "Re-energize yourself with a slice of pepperoni!"
)

// Probe until we get a successful response. This ensures the domain is
// routable before we send it a bunch of traffic.
func probeDomain(logger *logging.BaseLogger, clients *test.Clients, domain string) error {
	client, err := pkgTest.NewSpoofingClient(clients.KubeClient, logger, domain, test.ServingFlags.ResolvableDomain)
	if err != nil {
		return err
	}
	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s", domain), nil)
	if err != nil {
		return err
	}
	// TODO(tcnghia): Replace this probing with Status check when we have them.
	_, err = client.Poll(req, pkgTest.Retrying(pkgTest.MatchesAny, http.StatusNotFound, http.StatusServiceUnavailable))
	return err
}

// sendRequests sends "num" requests to "domain", returning a string for each spoof.Response.Body.
func sendRequests(client spoof.Interface, domain string, num int) ([]string, error) {
	responses := make([]string, num)

	// Launch "num" requests, recording the responses we get in "responses".
	g, _ := errgroup.WithContext(context.Background())
	for i := 0; i < num; i++ {
		// We don't index into "responses" inside the goroutine to avoid a race, see #1545.
		result := &responses[i]
		g.Go(func() error {
			req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s", domain), nil)
			if err != nil {
				return err
			}

			resp, err := client.Do(req)
			if err != nil {
				return err
			}

			*result = string(resp.Body)
			return nil
		})
	}
	return responses, g.Wait()
}

// checkResponses verifies that each "expectedResponse" is present in "actualResponses" at least "min" times.
func checkResponses(logger *logging.BaseLogger, num int, min int, domain string, expectedResponses []string, actualResponses []string) error {
	// counts maps the expected response body to the number of matching requests we saw.
	counts := make(map[string]int)
	// badCounts maps the unexpected response body to the number of matching requests we saw.
	badCounts := make(map[string]int)

	// counts := eval(
	//   SELECT body, count(*) AS total
	//   FROM $actualResponses
	//   WHERE body IN $expectedResponses
	//   GROUP BY body
	// )
	for _, ar := range actualResponses {
		expected := false
		for _, er := range expectedResponses {
			if strings.Contains(string(ar), er) {
				counts[er]++
				expected = true
			}
		}
		if !expected {
			badCounts[ar]++
		}
	}

	// Verify that we saw each entry in "expectedResponses" at least "min" times.
	// check(SELECT body FROM $counts WHERE total < $min)
	totalMatches := 0
	for _, er := range expectedResponses {
		count := counts[er]
		if count < min {
			return fmt.Errorf("domain %s failed: want min %d, got %d for response %q", domain, min, count, er)
		} else {
			logger.Infof("wanted at least %d, got %d requests for domain %s", min, count, domain)
		}
		totalMatches += count
	}
	// Verify that the total expected responses match the number of requests made.
	for badResponse, count := range badCounts {
		logger.Infof("saw unexpected response %q %d times", badResponse, count)
	}
	if totalMatches < num {
		return fmt.Errorf("saw expected responses %d times, wanted %d", totalMatches, num)
	}
	// If we made it here, the implementation conforms. Congratulations!
	return nil
}

// checkDistribution sends "num" requests to "domain", then validates that
// we see each body in "expectedResponses" at least "min" times.
func checkDistribution(logger *logging.BaseLogger, clients *test.Clients, domain string, num, min int, expectedResponses []string) error {
	client, err := pkgTest.NewSpoofingClient(clients.KubeClient, logger, domain, test.ServingFlags.ResolvableDomain)
	if err != nil {
		return err
	}

	logger.Infof("Performing %d concurrent requests to %s", num, domain)
	actualResponses, err := sendRequests(client, domain, num)
	if err != nil {
		return err
	}

	return checkResponses(logger, num, min, domain, expectedResponses, actualResponses)
}

// TestBlueGreenRoute verifies that a route configured with a 50/50 traffic split
// between two revisions will (approximately) route traffic evenly between them.
// Also, traffic that targets revisions *directly* will be routed to the correct
// revision 100% of the time.
func TestBlueGreenRoute(t *testing.T) {
	clients := setup(t)

	// add test case specific name to its own logger
	logger := logging.GetContextLogger("TestBlueGreenRoute")

	var imagePaths []string
	imagePaths = append(imagePaths, test.ImagePath(image1))
	imagePaths = append(imagePaths, test.ImagePath(image2))

	var names, blue, green test.ResourceNames
	names.Config = test.AppendRandomString("prod", logger)
	names.Route = test.AppendRandomString("pizzaplanet", logger)

	test.CleanupOnInterrupt(func() { tearDown(clients, names) }, logger)
	defer tearDown(clients, names)

	logger.Infof("Creating a Configuration")
	if err := test.CreateConfiguration(logger, clients, names, imagePaths[0]); err != nil {
		t.Fatalf("Failed to create Configuration: %v", err)
	}

	var err error

	logger.Infof("The Configuration will be updated with the name of the Revision once it is created")
	blue.Revision, err = getNextRevisionName(clients, names)
	if err != nil {
		t.Fatalf("Configuration %s was not updated with the new revision: %v", names.Config, err)
	}

	logger.Infof("Updating the Configuration to use a different image")
	err = updateConfigWithImage(clients, names, imagePaths)
	if err != nil {
		t.Fatalf("Patch update for Configuration %s with new image %s failed: %v", names.Config, imagePaths[1], err)
	}

	// getNextRevisionName waits for names.Revision to change, so we set it to the blue revision and wait for the (new) green revision.
	names.Revision = blue.Revision

	logger.Infof("Since the Configuration was updated a new Revision will be created and the Configuration will be updated")
	green.Revision, err = getNextRevisionName(clients, names)
	if err != nil {
		t.Fatalf("Configuration %s was not updated with the Revision for image %s: %v", names.Config, image2, err)
	}

	// We should only need to wait until the Revision is routable,
	// i.e. Ready or Inactive.  At that point, activator could start
	// queuing requests until the Revision wakes up.  However, due to
	// #882 we are currently lumping the inactive splits and that
	// would result in 100% requests reaching Blue or Green.
	//
	// TODO: After we implement #1583 and honor the split percentage
	// for inactive cases, change this wait to allow for inactive
	// revisions as well.
	logger.Infof("Waiting for revision %q to be ready", blue.Revision)
	if err := test.WaitForRevisionState(clients.ServingClient, blue.Revision, test.IsRevisionReady, "RevisionIsReady"); err != nil {
		t.Fatalf("The Revision %q still can't serve traffic: %v", blue.Revision, err)
	}
	logger.Infof("Waiting for revision %q to be ready", green.Revision)
	if err := test.WaitForRevisionState(clients.ServingClient, green.Revision, test.IsRevisionReady, "RevisionIsReady"); err != nil {
		t.Fatalf("The Revision %q still can't serve traffic: %v", green.Revision, err)
	}

	// Set names for traffic targets to make them directly routable.
	blue.TrafficTarget = "blue"
	green.TrafficTarget = "green"

	logger.Infof("Creating a Route")
	if err := test.CreateBlueGreenRoute(logger, clients, names, blue, green); err != nil {
		t.Fatalf("Failed to create Route: %v", err)
	}

	logger.Infof("When the Route reports as Ready, everything should be ready.")
	if err := test.WaitForRouteState(clients.ServingClient, names.Route, test.IsRouteReady, "RouteIsReady"); err != nil {
		t.Fatalf("The Route %s was not marked as Ready to serve traffic: %v", names.Route, err)
	}

	route, err := clients.ServingClient.Routes.Get(names.Route, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Error fetching Route %s: %v", names.Route, err)
	}

	// TODO(tcnghia): Add a RouteStatus method to create this string.
	blueDomain := fmt.Sprintf("%s.%s", blue.TrafficTarget, route.Status.Domain)
	greenDomain := fmt.Sprintf("%s.%s", green.TrafficTarget, route.Status.Domain)
	tealDomain := route.Status.Domain

	// Istio network programming takes some time to be effective.  Currently Istio
	// does not expose a Status, so we rely on probes to know when they are effective.
	// It doesn't matter which domain we probe, we just need to choose one.
	logger.Infof("Probing domain %s", tealDomain)
	if err := probeDomain(logger, clients, tealDomain); err != nil {
		t.Fatalf("Error probing domain %s: %v", tealDomain, err)
	}

	// Send concurrentRequests to blueDomain, greenDomain, and tealDomain.
	g, _ := errgroup.WithContext(context.Background())
	g.Go(func() error {
		min := int(concurrentRequests * minSplitPercentage)
		return checkDistribution(logger, clients, tealDomain, concurrentRequests, min, []string{expectedBlue, expectedGreen})
	})
	g.Go(func() error {
		min := int(concurrentRequests * minDirectPercentage)
		return checkDistribution(logger, clients, blueDomain, concurrentRequests, min, []string{expectedBlue})
	})
	g.Go(func() error {
		min := int(concurrentRequests * minDirectPercentage)
		return checkDistribution(logger, clients, greenDomain, concurrentRequests, min, []string{expectedGreen})
	})
	if err := g.Wait(); err != nil {
		t.Fatalf("Error sending requests: %v", err)
	}
}
<commit_msg>Reduce blue/green number of concurrent requests. (#2008)<commit_after>// +build e2e

/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package conformance

import (
	"context"
	"fmt"
	"math"
	"net/http"
	"strings"
	"testing"

	pkgTest "github.com/knative/pkg/test"
	"github.com/knative/pkg/test/logging"
	"github.com/knative/pkg/test/spoof"
	"github.com/knative/serving/test"
	"golang.org/x/sync/errgroup"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	// Mysteriously required to support GCP auth (required by k8s libs). Apparently just importing it is enough. @_@ side effects @_@. https://github.com/kubernetes/client-go/issues/242
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
)

const (
	concurrentRequests = 50
	// We expect to see 100% of requests succeed for traffic sent directly to revisions.
	// This might be a bad assumption.
	minDirectPercentage = 1
	// We expect to see at least 25% of either response since we're routing 50/50.
	// This might be a bad assumption.
	minSplitPercentage = 0.25

	expectedBlue  = "What a spaceport!"
	expectedGreen = "Re-energize yourself with a slice of pepperoni!"
)

// Probe until we get a successful response. This ensures the domain is
// routable before we send it a bunch of traffic.
func probeDomain(logger *logging.BaseLogger, clients *test.Clients, domain string) error {
	client, err := pkgTest.NewSpoofingClient(clients.KubeClient, logger, domain, test.ServingFlags.ResolvableDomain)
	if err != nil {
		return err
	}
	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s", domain), nil)
	if err != nil {
		return err
	}
	// TODO(tcnghia): Replace this probing with Status check when we have them.
	_, err = client.Poll(req, pkgTest.Retrying(pkgTest.MatchesAny, http.StatusNotFound, http.StatusServiceUnavailable))
	return err
}

// sendRequests sends "num" requests to "domain", returning a string for each spoof.Response.Body.
func sendRequests(client spoof.Interface, domain string, num int) ([]string, error) {
	responses := make([]string, num)

	// Launch "num" requests, recording the responses we get in "responses".
	g, _ := errgroup.WithContext(context.Background())
	for i := 0; i < num; i++ {
		// We don't index into "responses" inside the goroutine to avoid a race, see #1545.
		result := &responses[i]
		g.Go(func() error {
			req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s", domain), nil)
			if err != nil {
				return err
			}

			resp, err := client.Do(req)
			if err != nil {
				return err
			}

			*result = string(resp.Body)
			return nil
		})
	}
	return responses, g.Wait()
}

// checkResponses verifies that each "expectedResponse" is present in "actualResponses" at least "min" times.
func checkResponses(logger *logging.BaseLogger, num int, min int, domain string, expectedResponses []string, actualResponses []string) error {
	// counts maps the expected response body to the number of matching requests we saw.
	counts := make(map[string]int)
	// badCounts maps the unexpected response body to the number of matching requests we saw.
	badCounts := make(map[string]int)

	// counts := eval(
	//   SELECT body, count(*) AS total
	//   FROM $actualResponses
	//   WHERE body IN $expectedResponses
	//   GROUP BY body
	// )
	for _, ar := range actualResponses {
		expected := false
		for _, er := range expectedResponses {
			if strings.Contains(string(ar), er) {
				counts[er]++
				expected = true
			}
		}
		if !expected {
			badCounts[ar]++
		}
	}

	// Verify that we saw each entry in "expectedResponses" at least "min" times.
	// check(SELECT body FROM $counts WHERE total < $min)
	totalMatches := 0
	for _, er := range expectedResponses {
		count := counts[er]
		if count < min {
			return fmt.Errorf("domain %s failed: want min %d, got %d for response %q", domain, min, count, er)
		} else {
			logger.Infof("wanted at least %d, got %d requests for domain %s", min, count, domain)
		}
		totalMatches += count
	}
	// Verify that the total expected responses match the number of requests made.
	for badResponse, count := range badCounts {
		logger.Infof("saw unexpected response %q %d times", badResponse, count)
	}
	if totalMatches < num {
		return fmt.Errorf("saw expected responses %d times, wanted %d", totalMatches, num)
	}
	// If we made it here, the implementation conforms. Congratulations!
	return nil
}

// checkDistribution sends "num" requests to "domain", then validates that
// we see each body in "expectedResponses" at least "min" times.
func checkDistribution(logger *logging.BaseLogger, clients *test.Clients, domain string, num, min int, expectedResponses []string) error {
	client, err := pkgTest.NewSpoofingClient(clients.KubeClient, logger, domain, test.ServingFlags.ResolvableDomain)
	if err != nil {
		return err
	}

	logger.Infof("Performing %d concurrent requests to %s", num, domain)
	actualResponses, err := sendRequests(client, domain, num)
	if err != nil {
		return err
	}

	return checkResponses(logger, num, min, domain, expectedResponses, actualResponses)
}

// TestBlueGreenRoute verifies that a route configured with a 50/50 traffic split
// between two revisions will (approximately) route traffic evenly between them.
// Also, traffic that targets revisions *directly* will be routed to the correct
// revision 100% of the time.
func TestBlueGreenRoute(t *testing.T) {
	clients := setup(t)

	// add test case specific name to its own logger
	logger := logging.GetContextLogger("TestBlueGreenRoute")

	var imagePaths []string
	imagePaths = append(imagePaths, test.ImagePath(image1))
	imagePaths = append(imagePaths, test.ImagePath(image2))

	var names, blue, green test.ResourceNames
	names.Config = test.AppendRandomString("prod", logger)
	names.Route = test.AppendRandomString("pizzaplanet", logger)

	test.CleanupOnInterrupt(func() { tearDown(clients, names) }, logger)
	defer tearDown(clients, names)

	logger.Infof("Creating a Configuration")
	if err := test.CreateConfiguration(logger, clients, names, imagePaths[0]); err != nil {
		t.Fatalf("Failed to create Configuration: %v", err)
	}

	var err error

	logger.Infof("The Configuration will be updated with the name of the Revision once it is created")
	blue.Revision, err = getNextRevisionName(clients, names)
	if err != nil {
		t.Fatalf("Configuration %s was not updated with the new revision: %v", names.Config, err)
	}

	logger.Infof("Updating the Configuration to use a different image")
	err = updateConfigWithImage(clients, names, imagePaths)
	if err != nil {
		t.Fatalf("Patch update for Configuration %s with new image %s failed: %v", names.Config, imagePaths[1], err)
	}

	// getNextRevisionName waits for names.Revision to change, so we set it to the blue revision and wait for the (new) green revision.
	names.Revision = blue.Revision

	logger.Infof("Since the Configuration was updated a new Revision will be created and the Configuration will be updated")
	green.Revision, err = getNextRevisionName(clients, names)
	if err != nil {
		t.Fatalf("Configuration %s was not updated with the Revision for image %s: %v", names.Config, image2, err)
	}

	// We should only need to wait until the Revision is routable,
	// i.e. Ready or Inactive.  At that point, activator could start
	// queuing requests until the Revision wakes up.  However, due to
	// #882 we are currently lumping the inactive splits and that
	// would result in 100% requests reaching Blue or Green.
	//
	// TODO: After we implement #1583 and honor the split percentage
	// for inactive cases, change this wait to allow for inactive
	// revisions as well.
	logger.Infof("Waiting for revision %q to be ready", blue.Revision)
	if err := test.WaitForRevisionState(clients.ServingClient, blue.Revision, test.IsRevisionReady, "RevisionIsReady"); err != nil {
		t.Fatalf("The Revision %q still can't serve traffic: %v", blue.Revision, err)
	}
	logger.Infof("Waiting for revision %q to be ready", green.Revision)
	if err := test.WaitForRevisionState(clients.ServingClient, green.Revision, test.IsRevisionReady, "RevisionIsReady"); err != nil {
		t.Fatalf("The Revision %q still can't serve traffic: %v", green.Revision, err)
	}

	// Set names for traffic targets to make them directly routable.
	blue.TrafficTarget = "blue"
	green.TrafficTarget = "green"

	logger.Infof("Creating a Route")
	if err := test.CreateBlueGreenRoute(logger, clients, names, blue, green); err != nil {
		t.Fatalf("Failed to create Route: %v", err)
	}

	logger.Infof("When the Route reports as Ready, everything should be ready.")
	if err := test.WaitForRouteState(clients.ServingClient, names.Route, test.IsRouteReady, "RouteIsReady"); err != nil {
		t.Fatalf("The Route %s was not marked as Ready to serve traffic: %v", names.Route, err)
	}

	route, err := clients.ServingClient.Routes.Get(names.Route, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Error fetching Route %s: %v", names.Route, err)
	}

	// TODO(tcnghia): Add a RouteStatus method to create this string.
	blueDomain := fmt.Sprintf("%s.%s", blue.TrafficTarget, route.Status.Domain)
	greenDomain := fmt.Sprintf("%s.%s", green.TrafficTarget, route.Status.Domain)
	tealDomain := route.Status.Domain

	// Istio network programming takes some time to be effective.  Currently Istio
	// does not expose a Status, so we rely on probes to know when they are effective.
	// It doesn't matter which domain we probe, we just need to choose one.
	logger.Infof("Probing domain %s", tealDomain)
	if err := probeDomain(logger, clients, tealDomain); err != nil {
		t.Fatalf("Error probing domain %s: %v", tealDomain, err)
	}

	// Send concurrentRequests to blueDomain, greenDomain, and tealDomain.
	g, _ := errgroup.WithContext(context.Background())
	g.Go(func() error {
		min := int(math.Floor(concurrentRequests * minSplitPercentage))
		return checkDistribution(logger, clients, tealDomain, concurrentRequests, min, []string{expectedBlue, expectedGreen})
	})
	g.Go(func() error {
		min := int(math.Floor(concurrentRequests * minDirectPercentage))
		return checkDistribution(logger, clients, blueDomain, concurrentRequests, min, []string{expectedBlue})
	})
	g.Go(func() error {
		min := int(math.Floor(concurrentRequests * minDirectPercentage))
		return checkDistribution(logger, clients, greenDomain, concurrentRequests, min, []string{expectedGreen})
	})
	if err := g.Wait(); err != nil {
		t.Fatalf("Error sending requests: %v", err)
	}
}
<|endoftext|>