<commit_before>package job

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"strings"

	log "github.com/Sirupsen/logrus"
)

const (
	BeginDelimiter = "----BEGIN PANAMAX DATA----"
	EndDelimiter   = "----END PANAMAX DATA----"
	EOT            = byte('\u0003')
)

var (
	accessor         JobAccessor
	containerFactory ContainerFactory
)

func init() {
	accessor = &redisJobAccessor{}
	containerFactory = &dockerContainerFactory{}
}

type Job struct {
	ID             string    `json:"id,omitempty"`
	Name           string    `json:"name,omitempty"`
	Steps          []JobStep `json:"steps,omitempty"`
	StepsCompleted string    `json:"stepsCompleted,omitempty"`
}

type JobStep struct {
	Name        string   `json:"name,omitempty"`
	Source      string   `json:"source,omitempty"`
	Environment []EnvVar `json:"environment,omitempty"`
}

type EnvVar struct {
	Variable string `json:"variable"`
	Value    string `json:"value"`
}

type JobLog struct {
	Index int      `json:"index,omitempty"`
	Lines []string `json:"lines"`
}

func ListAll() ([]Job, error) {
	return accessor.All()
}

func GetByID(jobID string) (*Job, error) {
	return accessor.Get(jobID)
}

func (job *Job) Create() error {
	return accessor.Create(job)
}

func (job *Job) Delete() error {
	return accessor.Delete(job.ID)
}

func (job *Job) GetLog(index int) (*JobLog, error) {
	return accessor.GetJobLog(job.ID, index)
}

func (job *Job) Execute() error {
	var capture io.Reader

	for i := range job.Steps {
		capture, _ = job.executeStep(i, capture)
		accessor.CompleteStep(job.ID)
	}
	return nil
}

func (job *Job) executeStep(stepIndex int, stdIn io.Reader) (io.Reader, error) {
	stdOut := &bytes.Buffer{}
	stdErr := &bytes.Buffer{}
	step := job.Steps[stepIndex]
	container := containerFactory.NewContainer(step.Source, stringifyEnvironment(step.Environment))

	if err := container.Create(); err != nil {
		return nil, err
	}
	log.Debugf("Container %s created", container)

	go func() {
		container.Attach(stdIn, stdOut, stdErr)
		stdOut.Write([]byte{EOT, '\n'})
	}()

	if err := container.Start(); err != nil {
		return nil, err
	}
	log.Debugf("Container %s started", container)

	output, err := job.captureOutput(stdOut)
	if err != nil {
		return nil, err
	}
	log.Debugf("Container %s stopped", container)

	if err := container.Remove(); err != nil {
		return nil, err
	}
	log.Debugf("Container %s removed", container)

	return output, nil
}

func (job *Job) captureOutput(r io.Reader) (io.Reader, error) {
	reader := bufio.NewReader(r)
	buffer := &bytes.Buffer{}
	capture := false

	for {
		line, _ := reader.ReadBytes('\n')

		if len(line) > 0 {
			if line[0] == EOT {
				break
			}
			s := strings.TrimSpace(string(line))
			log.Debugf(s)
			accessor.AppendLogLine(job.ID, s)

			if s == EndDelimiter {
				capture = false
			}

			if capture {
				buffer.WriteString(s + "\n")
			}

			if s == BeginDelimiter {
				capture = true
			}
		}
	}

	return buffer, nil
}

func stringifyEnvironment(env []EnvVar) []string {
	envStrings := []string{}

	for _, v := range env {
		s := fmt.Sprintf("%s=%s", v.Variable, v.Value)
		envStrings = append(envStrings, s)
	}

	return envStrings
}
<commit_msg>Error handling in job.Execute<commit_after>package job

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"strings"

	log "github.com/Sirupsen/logrus"
)

const (
	BeginDelimiter = "----BEGIN PANAMAX DATA----"
	EndDelimiter   = "----END PANAMAX DATA----"
	EOT            = byte('\u0003')
)

var (
	accessor         JobAccessor
	containerFactory ContainerFactory
)

func init() {
	accessor = &redisJobAccessor{}
	containerFactory = &dockerContainerFactory{}
}

type Job struct {
	ID             string    `json:"id,omitempty"`
	Name           string    `json:"name,omitempty"`
	Steps          []JobStep `json:"steps,omitempty"`
	StepsCompleted string    `json:"stepsCompleted,omitempty"`
}

type JobStep struct {
	Name        string   `json:"name,omitempty"`
	Source      string   `json:"source,omitempty"`
	Environment []EnvVar `json:"environment,omitempty"`
}

type EnvVar struct {
	Variable string `json:"variable"`
	Value    string `json:"value"`
}

type JobLog struct {
	Index int      `json:"index,omitempty"`
	Lines []string `json:"lines"`
}

func ListAll() ([]Job, error) {
	return accessor.All()
}

func GetByID(jobID string) (*Job, error) {
	return accessor.Get(jobID)
}

func (job *Job) Create() error {
	return accessor.Create(job)
}

func (job *Job) Delete() error {
	return accessor.Delete(job.ID)
}

func (job *Job) GetLog(index int) (*JobLog, error) {
	return accessor.GetJobLog(job.ID, index)
}

func (job *Job) Execute() error {
	var capture io.Reader
	var err error

	for i := range job.Steps {
		capture, err = job.executeStep(i, capture)

		if err != nil {
			break
		}

		accessor.CompleteStep(job.ID)
	}

	return err
}

func (job *Job) executeStep(stepIndex int, stdIn io.Reader) (io.Reader, error) {
	stdOut := &bytes.Buffer{}
	stdErr := &bytes.Buffer{}
	step := job.Steps[stepIndex]
	container := containerFactory.NewContainer(step.Source, stringifyEnvironment(step.Environment))

	if err := container.Create(); err != nil {
		return nil, err
	}
	log.Debugf("Container %s created", container)

	go func() {
		container.Attach(stdIn, stdOut, stdErr)
		stdOut.Write([]byte{EOT, '\n'})
	}()

	if err := container.Start(); err != nil {
		return nil, err
	}
	log.Debugf("Container %s started", container)

	output, err := job.captureOutput(stdOut)
	if err != nil {
		return nil, err
	}
	log.Debugf("Container %s stopped", container)

	if err := container.Remove(); err != nil {
		return nil, err
	}
	log.Debugf("Container %s removed", container)

	return output, nil
}

func (job *Job) captureOutput(r io.Reader) (io.Reader, error) {
	reader := bufio.NewReader(r)
	buffer := &bytes.Buffer{}
	capture := false

	for {
		line, _ := reader.ReadBytes('\n')

		if len(line) > 0 {
			if line[0] == EOT {
				break
			}
			s := strings.TrimSpace(string(line))
			log.Debugf(s)
			accessor.AppendLogLine(job.ID, s)

			if s == EndDelimiter {
				capture = false
			}

			if capture {
				buffer.WriteString(s + "\n")
			}

			if s == BeginDelimiter {
				capture = true
			}
		}
	}

	return buffer, nil
}

func stringifyEnvironment(env []EnvVar) []string {
	envStrings := []string{}

	for _, v := range env {
		s := fmt.Sprintf("%s=%s", v.Variable, v.Value)
		envStrings = append(envStrings, s)
	}

	return envStrings
}
<|endoftext|>


<commit_before>package define

import (
	"github.com/sacloud/usacloud/output"
	"github.com/sacloud/usacloud/schema"
)

func ObjectStorageResource() *schema.Resource {

	commands := map[string]*schema.Command{
		"list": {
			Type:                   schema.CommandCustom,
			Aliases:                []string{"l", "ls"},
			Params:                 objectStorageListParam(),
			TableType:              output.TableSimple,
			TableColumnDefines:     objectStorageListColumns(),
			UseCustomCommand:       true,
			UseCustomArgCompletion: true,
			ArgsUsage:              "<remote path>",
			SkipAuth:               true,
			NeedlessConfirm:        true,
		},
		"put": {
			Type:                   schema.CommandCustom,
			Params:                 objectStoragePutParam(),
			UseCustomCommand:       true,
			UseCustomArgCompletion: true,
			ArgsUsage:              "<local file/directory> <remote path>",
			SkipAuth:               true,
			NoOutput:               true,
		},
		"get": {
			Type:                   schema.CommandCustom,
			Params:                 objectStorageGetParam(),
			UseCustomCommand:       true,
			UseCustomArgCompletion: true,
			ArgsUsage:              "<remote path> <local file/directory>",
			SkipAuth:               true,
			NoOutput:               true,
			NeedlessConfirm:        true,
		},
		"delete": {
			Type:                   schema.CommandCustom,
			Aliases:                []string{"rm", "del"},
			Params:                 objectStorageDelParam(),
			UseCustomCommand:       true,
			UseCustomArgCompletion: true,
			ArgsUsage:              "<remote path>",
			SkipAuth:               true,
			NoOutput:               true,
		},
	}

	return &schema.Resource{
		Commands:         commands,
		Aliases:          []string{"ojs"},
		ResourceCategory: CategoryOther,
	}
}

func objectStorageListColumns() []output.ColumnDef {
	return []output.ColumnDef{
		{Name: "Key"},
		{Name: "Size"},
		{Name: "ETag"},
	}
}

func objectStorageDetailIncludes() []string {
	return []string{}
}

func objectStorageDetailExcludes() []string {
	return []string{}
}

func objectStorageListParam() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"access-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"AWS_ACCESS_KEY_ID", "SACLOUD_OJS_ACCESS_KEY_ID"},
			Required:    true,
		},
		"secret-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"AWS_SECRET_ACCESS_KEY", "SACLOUD_OJS_SECRET_ACCESS_KEY"},
			Required:    true,
		},
		"bucket": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set bucket",
			EnvVars:     []string{"SACLOUD_OJS_BUCKET_NAME"},
		},
	}
}

func objectStoragePutParam() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"access-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"AWS_ACCESS_KEY_ID", "SACLOUD_OJS_ACCESS_KEY_ID"},
			Required:    true,
		},
		"secret-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"AWS_SECRET_ACCESS_KEY", "SACLOUD_OJS_SECRET_ACCESS_KEY"},
			Required:    true,
		},
		"bucket": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set bucket",
			EnvVars:     []string{"SACLOUD_OJS_BUCKET_NAME"},
		},
		"content-type": {
			Type:         schema.TypeString,
			HandlerType:  schema.HandlerNoop,
			Description:  "set content-type",
			DefaultValue: "application/octet-stream",
		},
		"recursive": {
			Type:        schema.TypeBool,
			HandlerType: schema.HandlerNoop,
			Aliases:     []string{"r"},
			Description: "put objects recursive",
		},
	}
}

func objectStorageGetParam() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"access-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"AWS_ACCESS_KEY_ID", "SACLOUD_OJS_ACCESS_KEY_ID"},
			Required:    true,
		},
		"secret-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"AWS_SECRET_ACCESS_KEY", "SACLOUD_OJS_SECRET_ACCESS_KEY"},
			Required:    true,
		},
		"bucket": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set bucket",
			EnvVars:     []string{"SACLOUD_OJS_BUCKET_NAME"},
		},
		"recursive": {
			Type:        schema.TypeBool,
			HandlerType: schema.HandlerNoop,
			Aliases:     []string{"r"},
			Description: "get objects recursive",
		},
	}
}

func objectStorageDelParam() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"access-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"AWS_ACCESS_KEY_ID", "SACLOUD_OJS_ACCESS_KEY_ID"},
			Required:    true,
		},
		"secret-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"AWS_SECRET_ACCESS_KEY", "SACLOUD_OJS_SECRET_ACCESS_KEY"},
			Required:    true,
		},
		"bucket": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set bucket",
			EnvVars:     []string{"SACLOUD_OJS_BUCKET_NAME"},
		},
		"recursive": {
			Type:        schema.TypeBool,
			HandlerType: schema.HandlerNoop,
			Aliases:     []string{"r"},
			Description: "delete objects recursive",
		},
	}
}
<commit_msg>Fix EnvVars handling on ObjectStorage<commit_after>package define

import (
	"github.com/sacloud/usacloud/output"
	"github.com/sacloud/usacloud/schema"
)

func ObjectStorageResource() *schema.Resource {

	commands := map[string]*schema.Command{
		"list": {
			Type:                   schema.CommandCustom,
			Aliases:                []string{"l", "ls"},
			Params:                 objectStorageListParam(),
			TableType:              output.TableSimple,
			TableColumnDefines:     objectStorageListColumns(),
			UseCustomCommand:       true,
			UseCustomArgCompletion: true,
			ArgsUsage:              "<remote path>",
			SkipAuth:               true,
			NeedlessConfirm:        true,
		},
		"put": {
			Type:                   schema.CommandCustom,
			Params:                 objectStoragePutParam(),
			UseCustomCommand:       true,
			UseCustomArgCompletion: true,
			ArgsUsage:              "<local file/directory> <remote path>",
			SkipAuth:               true,
			NoOutput:               true,
		},
		"get": {
			Type:                   schema.CommandCustom,
			Params:                 objectStorageGetParam(),
			UseCustomCommand:       true,
			UseCustomArgCompletion: true,
			ArgsUsage:              "<remote path> <local file/directory>",
			SkipAuth:               true,
			NoOutput:               true,
			NeedlessConfirm:        true,
		},
		"delete": {
			Type:                   schema.CommandCustom,
			Aliases:                []string{"rm", "del"},
			Params:                 objectStorageDelParam(),
			UseCustomCommand:       true,
			UseCustomArgCompletion: true,
			ArgsUsage:              "<remote path>",
			SkipAuth:               true,
			NoOutput:               true,
		},
	}

	return &schema.Resource{
		Commands:         commands,
		Aliases:          []string{"ojs"},
		ResourceCategory: CategoryOther,
	}
}

func objectStorageListColumns() []output.ColumnDef {
	return []output.ColumnDef{
		{Name: "Key"},
		{Name: "Size"},
		{Name: "ETag"},
	}
}

func objectStorageDetailIncludes() []string {
	return []string{}
}

func objectStorageDetailExcludes() []string {
	return []string{}
}

func objectStorageListParam() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"access-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"SACLOUD_OJS_ACCESS_KEY_ID", "AWS_ACCESS_KEY_ID"},
			Required:    true,
		},
		"secret-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"SACLOUD_OJS_SECRET_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY"},
			Required:    true,
		},
		"bucket": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set bucket",
			EnvVars:     []string{"SACLOUD_OJS_BUCKET_NAME"},
		},
	}
}

func objectStoragePutParam() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"access-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"SACLOUD_OJS_ACCESS_KEY_ID", "AWS_ACCESS_KEY_ID"},
			Required:    true,
		},
		"secret-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"SACLOUD_OJS_SECRET_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY"},
			Required:    true,
		},
		"bucket": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set bucket",
			EnvVars:     []string{"SACLOUD_OJS_BUCKET_NAME"},
		},
		"content-type": {
			Type:         schema.TypeString,
			HandlerType:  schema.HandlerNoop,
			Description:  "set content-type",
			DefaultValue: "application/octet-stream",
		},
		"recursive": {
			Type:        schema.TypeBool,
			HandlerType: schema.HandlerNoop,
			Aliases:     []string{"r"},
			Description: "put objects recursive",
		},
	}
}

func objectStorageGetParam() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"access-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"SACLOUD_OJS_ACCESS_KEY_ID", "AWS_ACCESS_KEY_ID"},
			Required:    true,
		},
		"secret-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"SACLOUD_OJS_SECRET_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY"},
			Required:    true,
		},
		"bucket": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set bucket",
			EnvVars:     []string{"SACLOUD_OJS_BUCKET_NAME"},
		},
		"recursive": {
			Type:        schema.TypeBool,
			HandlerType: schema.HandlerNoop,
			Aliases:     []string{"r"},
			Description: "get objects recursive",
		},
	}
}

func objectStorageDelParam() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"access-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"SACLOUD_OJS_ACCESS_KEY_ID", "AWS_ACCESS_KEY_ID"},
			Required:    true,
		},
		"secret-key": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set access-key",
			EnvVars:     []string{"SACLOUD_OJS_SECRET_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY"},
			Required:    true,
		},
		"bucket": {
			Type:        schema.TypeString,
			HandlerType: schema.HandlerNoop,
			Description: "set bucket",
			EnvVars:     []string{"SACLOUD_OJS_BUCKET_NAME"},
		},
		"recursive": {
			Type:        schema.TypeBool,
			HandlerType: schema.HandlerNoop,
			Aliases:     []string{"r"},
			Description: "delete objects recursive",
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strconv"

	hyperloglog "github.com/influxdata/influxdb/pkg/estimator/hll"
	"github.com/seiflotfy/hlltc"
)

type fakeHash64 uint64

func (f fakeHash64) Sum64() uint64 { return uint64(f) }

func estimateError(got, exp uint64) float64 {
	var delta uint64
	if got > exp {
		delta = got - exp
	} else {
		delta = exp - got
	}
	return float64(delta) / float64(exp)
}

func main() {
	hlltc, _ := hlltc.New(14)
	hllpp, _ := hyperloglog.NewPlus(14)

	step := 10
	unique := map[string]bool{}

	for i := 1; len(unique) <= 10000000; i++ {
		str := "stream-" + strconv.Itoa(i)
		hlltc.Insert([]byte(str))
		hllpp.Add([]byte(str))
		unique[str] = true

		if len(unique)%step == 0 || len(unique) == 10000000 {
			step *= 5
			exact := uint64(len(unique))
			res := uint64(hlltc.Estimate())
			ratio := 100 * estimateError(res, exact)
			res2 := uint64(hllpp.Count())
			ratio2 := 100 * estimateError(res2, exact)
			fmt.Printf("Exact %d, got:\n\t axiom hlltc %d (%.4f%% off)\n\tinflux hllpp %d (%.4f%% off)\n", exact, res, ratio, res2, ratio2)
		}
	}

	data1, _ := hllpp.MarshalBinary()
	data2, _ := hlltc.MarshalBinary()
	fmt.Println("AxiomHQ HLLTC total size", len(data2))
	fmt.Println("InfluxData HLLPP total size", len(data1))

}
<commit_msg>fix demo<commit_after>package main

import (
	"fmt"
	"strconv"

	"github.com/axiomhq/hyperloglog"
	"github.com/influxdata/influxdb/pkg/estimator/hll"
)

type fakeHash64 uint64

func (f fakeHash64) Sum64() uint64 { return uint64(f) }

func estimateError(got, exp uint64) float64 {
	var delta uint64
	if got > exp {
		delta = got - exp
	} else {
		delta = exp - got
	}
	return float64(delta) / float64(exp)
}

func main() {
	hlltc, _ := hlltc.New(14)
	hllpp, _ := hll.NewPlus(14)

	step := 10
	unique := map[string]bool{}

	for i := 1; len(unique) <= 10000000; i++ {
		str := "stream-" + strconv.Itoa(i)
		hlltc.Insert([]byte(str))
		hllpp.Add([]byte(str))
		unique[str] = true

		if len(unique)%step == 0 || len(unique) == 10000000 {
			step *= 5
			exact := uint64(len(unique))
			res := uint64(hlltc.Estimate())
			ratio := 100 * estimateError(res, exact)
			res2 := uint64(hllpp.Count())
			ratio2 := 100 * estimateError(res2, exact)
			fmt.Printf("Exact %d, got:\n\t axiom hlltc %d (%.4f%% off)\n\tinflux hllpp %d (%.4f%% off)\n", exact, res, ratio, res2, ratio2)
		}
	}

	data1, _ := hllpp.MarshalBinary()
	data2, _ := hlltc.MarshalBinary()
	fmt.Println("AxiomHQ HLLTC total size", len(data2))
	fmt.Println("InfluxData HLLPP total size", len(data1))

}
<|endoftext|>


<commit_before>package main

type Config struct {
	Foo string
	Bar int
}
<commit_msg>Social/FollowingFeed: added config variable into socialapi/config<commit_after>package config

type Config struct {
	Foo string
	Bar int
}

var EventExchangeName = "EventExchange"
<|endoftext|>


<commit_before>package main

import (
	"reflect"
	"testing"
)

func TestDeckShouldHaveUniqueCards(t *testing.T) {
	deck := InitializeDeck()
	result := CheckDups(deck)
	if !result {
		t.Error("There is a duplicate card.")
	}
	return
}

func TestDeckShouldBeShuffled(t *testing.T) {
	deck := InitializeDeck()
	if reflect.DeepEqual(deck[:13], []Card{
		{1, "Clubs", "A"},
		{2, "Clubs", "2"},
		{3, "Clubs", "3"},
		{4, "Clubs", "4"},
		{5, "Clubs", "5"},
		{6, "Clubs", "6"},
		{7, "Clubs", "7"},
		{8, "Clubs", "8"},
		{9, "Clubs", "9"},
		{10, "Clubs", "10"},
		{11, "Clubs", "J"},
		{12, "Clubs", "Q"},
		{13, "Clubs", "K"},
	}) {
		t.Error("The deck is not shuffled correctly")
	}

	shuffledDeck := deck.Shuffle()
	if reflect.DeepEqual(shuffledDeck[:13], []Card{
		{1, "Clubs", "A"},
		{2, "Clubs", "2"},
		{3, "Clubs", "3"},
		{4, "Clubs", "4"},
		{5, "Clubs", "5"},
		{6, "Clubs", "6"},
		{7, "Clubs", "7"},
		{8, "Clubs", "8"},
		{9, "Clubs", "9"},
		{10, "Clubs", "10"},
		{11, "Clubs", "J"},
		{12, "Clubs", "Q"},
		{13, "Clubs", "K"},
	}) {
		t.Error("The deck is not shuffled correctly")
	}
	return
}

// Checks for duplicate objects in an array.
func CheckDups(deck []Card) bool {
	dups := map[Card]bool{}
	for _, card := range deck {
		if dups[card] == true {
			return false
		}
		dups[card] = true
	}
	return true
}
<commit_msg>Added tests for Deck<commit_after>package main

import (
	"reflect"
	"testing"
)

func TestDeckShouldHaveUniqueCards(t *testing.T) {
	deck := InitializeDeck()
	result := CheckDups(deck)
	if result {
		t.Error("There is a duplicate card.")
	}
	return
}

func TestDeckShouldBeShuffled(t *testing.T) {
	deck := InitializeDeck()
	if reflect.DeepEqual(deck[:13], []Card{
		{1, "Clubs", "A"},
		{2, "Clubs", "2"},
		{3, "Clubs", "3"},
		{4, "Clubs", "4"},
		{5, "Clubs", "5"},
		{6, "Clubs", "6"},
		{7, "Clubs", "7"},
		{8, "Clubs", "8"},
		{9, "Clubs", "9"},
		{10, "Clubs", "10"},
		{11, "Clubs", "J"},
		{12, "Clubs", "Q"},
		{13, "Clubs", "K"},
	}) {
		t.Error("The deck is not shuffled correctly")
	}

	shuffledDeck := deck.Shuffle()
	if reflect.DeepEqual(shuffledDeck[:13], []Card{
		{1, "Clubs", "A"},
		{2, "Clubs", "2"},
		{3, "Clubs", "3"},
		{4, "Clubs", "4"},
		{5, "Clubs", "5"},
		{6, "Clubs", "6"},
		{7, "Clubs", "7"},
		{8, "Clubs", "8"},
		{9, "Clubs", "9"},
		{10, "Clubs", "10"},
		{11, "Clubs", "J"},
		{12, "Clubs", "Q"},
		{13, "Clubs", "K"},
	}) {
		t.Error("The deck is not shuffled correctly")
	}
	return
}

// Checks for duplicate objects in an array.
func CheckDups(arr []Card) bool {
	dups := map[Card]bool{}
	for _, card := range arr {
		if dups[card] == true {
			return true
		}
		dups[card] = true
	}
	return false
}

func TestDrawCards(t *testing.T) {
	deck := InitializeDeck()
	testHand := &Hand{}
	deck.DrawCard(testHand)
	if len(*testHand) == 0 {
		t.Error("Failed to draw a card.")
	}
	deck.DrawCard(testHand)
	if CheckDups(*testHand) {
		t.Error("There are duplicates in the hand.")
	}
	return
}

// func TestDeckDealsTenCardsToPlayer(t *testing.T) {
// 	deck := InitializeDeck()
// 	p1 := &Player{"Michael", []Card{}}
// 	p2 := &Player{"AI", []Card{}}
// 	deck.Deal(p1, p2)
// 	if len(p1.Hand) != 10 {
// 		t.Error("Player one did not draw 10 cards!")
// 	}
// 	if len(p2.Hand) != 10 {
// 		t.Error("Player two did not draw 10 card!")
// 	}
// 	if CheckDups(p1.Hand) || CheckDups(p2.Hand) {
// 		t.Error("There are duplicate cards in the hands!")
// 	}
// 	return
// }
<|endoftext|>


<commit_before>//+build demo

package main

import (
	"image/color"
	"log"

	"engo.io/ecs"
	"engo.io/engo"
	"engo.io/engo/common"
)

type GameWorld struct{}

type Tile struct {
	ecs.BasicEntity
	common.RenderComponent
	common.SpaceComponent
}

func (game *GameWorld) Preload() {
	// A tmx file can be generated from the Tiled Map Editor.
	// The engo tmx loader only accepts tmx files that are base64 encoded and compressed with zlib.
	// When you add tilesets to the Tiled Editor, the location where you added them from is where the engo loader will look for them
	// Tileset from : http://opengameart.org

	if err := engo.Files.Load("example.tmx"); err != nil {
		panic(err)
	}
}

func (game *GameWorld) Setup(u engo.Updater) {
	w, _ := u.(*ecs.World)

	common.SetBackground(color.White)

	w.AddSystem(&common.RenderSystem{})

	resource, err := engo.Files.Resource("example.tmx")
	if err != nil {
		panic(err)
	}
	tmxResource := resource.(common.TMXResource)
	levelData := tmxResource.Level

	// Create render and space components for each of the tiles in all layers
	tileComponents := make([]*Tile, 0)

	for _, tileLayer := range levelData.TileLayers {
		for _, tileElement := range tileLayer.Tiles {
			if tileElement.Image != nil {

				tile := &Tile{BasicEntity: ecs.NewBasic()}
				tile.RenderComponent = common.RenderComponent{
					Drawable: tileElement,
					Scale:    engo.Point{1, 1},
				}
				tile.SpaceComponent = common.SpaceComponent{
					Position: tileElement.Point,
					Width:    0,
					Height:   0,
				}

				tileComponents = append(tileComponents, tile)
			}
		}
	}

	// Do the same for all image layers
	for _, imageLayer := range levelData.ImageLayers {
		for _, imageElement := range imageLayer.Images {
			if imageElement.Image != nil {
				tile := &Tile{BasicEntity: ecs.NewBasic()}
				tile.RenderComponent = common.RenderComponent{
					Drawable: imageElement,
					Scale:    engo.Point{1, 1},
				}
				tile.SpaceComponent = common.SpaceComponent{
					Position: imageElement.Point,
					Width:    0,
					Height:   0,
				}

				tileComponents = append(tileComponents, tile)
			}
		}
	}

	// Add each of the tiles entities and its components to the render system
	for _, system := range w.Systems() {
		switch sys := system.(type) {
		case *common.RenderSystem:
			for _, v := range tileComponents {
				sys.Add(&v.BasicEntity, &v.RenderComponent, &v.SpaceComponent)
			}

		}
	}

	// Access Object Layers
	for _, objectLayer := range levelData.ObjectLayers {
		log.Println("This object layer is called " + objectLayer.Name)
		// Do something with every regular Object
		for _, object := range objectLayer.Objects {
			log.Println("This object is called " + object.Name)
		}
	}

}

func (game *GameWorld) Type() string { return "GameWorld" }

func main() {
	opts := engo.RunOptions{
		Title:  "TileMap Demo",
		Width:  800,
		Height: 800,
	}
	engo.Run(opts, &GameWorld{})
}
<commit_msg>added z-index for tilelayers in tilemap demo<commit_after>//+build demo

package main

import (
	"image/color"
	"log"

	"engo.io/ecs"
	"engo.io/engo"
	"engo.io/engo/common"
)

type GameWorld struct{}

type Tile struct {
	ecs.BasicEntity
	common.RenderComponent
	common.SpaceComponent
}

func (game *GameWorld) Preload() {
	// A tmx file can be generated from the Tiled Map Editor.
	// The engo tmx loader only accepts tmx files that are base64 encoded and compressed with zlib.
	// When you add tilesets to the Tiled Editor, the location where you added them from is where the engo loader will look for them
	// Tileset from : http://opengameart.org

	if err := engo.Files.Load("example.tmx"); err != nil {
		panic(err)
	}
}

func (game *GameWorld) Setup(u engo.Updater) {
	w, _ := u.(*ecs.World)

	common.SetBackground(color.White)

	w.AddSystem(&common.RenderSystem{})

	resource, err := engo.Files.Resource("example.tmx")
	if err != nil {
		panic(err)
	}
	tmxResource := resource.(common.TMXResource)
	levelData := tmxResource.Level

	// Create render and space components for each of the tiles in all layers
	tileComponents := make([]*Tile, 0)

	for idx, tileLayer := range levelData.TileLayers {
		for _, tileElement := range tileLayer.Tiles {
			if tileElement.Image != nil {

				tile := &Tile{BasicEntity: ecs.NewBasic()}
				tile.RenderComponent = common.RenderComponent{
					Drawable: tileElement,
					Scale:    engo.Point{1, 1},
				}
				tile.RenderComponent.SetZIndex(float32(idx))
				tile.SpaceComponent = common.SpaceComponent{
					Position: tileElement.Point,
					Width:    0,
					Height:   0,
				}

				tileComponents = append(tileComponents, tile)
			}
		}
	}

	// Do the same for all image layers
	for _, imageLayer := range levelData.ImageLayers {
		for _, imageElement := range imageLayer.Images {
			if imageElement.Image != nil {
				tile := &Tile{BasicEntity: ecs.NewBasic()}
				tile.RenderComponent = common.RenderComponent{
					Drawable: imageElement,
					Scale:    engo.Point{1, 1},
				}
				tile.SpaceComponent = common.SpaceComponent{
					Position: imageElement.Point,
					Width:    0,
					Height:   0,
				}

				tileComponents = append(tileComponents, tile)
			}
		}
	}

	// Add each of the tiles entities and its components to the render system
	for _, system := range w.Systems() {
		switch sys := system.(type) {
		case *common.RenderSystem:
			for _, v := range tileComponents {
				sys.Add(&v.BasicEntity, &v.RenderComponent, &v.SpaceComponent)
			}

		}
	}

	// Access Object Layers
	for _, objectLayer := range levelData.ObjectLayers {
		log.Println("This object layer is called " + objectLayer.Name)
		// Do something with every regular Object
		for _, object := range objectLayer.Objects {
			log.Println("This object is called " + object.Name)
		}
	}

}

func (game *GameWorld) Type() string { return "GameWorld" }

func main() {
	opts := engo.RunOptions{
		Title:  "TileMap Demo",
		Width:  800,
		Height: 800,
	}
	engo.Run(opts, &GameWorld{})
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright 2016-2017 Authors of Cilium

package link

import (
	"fmt"
	"time"

	"github.com/vishvananda/netlink"

	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/logging"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/mac"
)

var linkCache ifNameCache

func init() {
	go func() {
		log := logging.DefaultLogger.WithField(logfields.LogSubsys, "link")
		for {
			if err := linkCache.syncCache(); err != nil {
				log.WithError(err).Error("failed to obtain network links. stopping cache sync")
				return
			}
			time.Sleep(15 * time.Second)
		}
	}()
}

// DeleteByName deletes the interface with the name ifName.
func DeleteByName(ifName string) error {
	iface, err := netlink.LinkByName(ifName)
	if err != nil {
		return fmt.Errorf("failed to lookup %q: %v", ifName, err)
	}

	if err = netlink.LinkDel(iface); err != nil {
		return fmt.Errorf("failed to delete %q: %v", ifName, err)
	}

	return nil
}

// Rename renames a network link
func Rename(curName, newName string) error {
	link, err := netlink.LinkByName(curName)
	if err != nil {
		return err
	}

	return netlink.LinkSetName(link, newName)
}

func GetHardwareAddr(ifName string) (mac.MAC, error) {
	iface, err := netlink.LinkByName(ifName)
	if err != nil {
		return nil, err
	}
	return mac.MAC(iface.Attrs().HardwareAddr), nil
}

func GetIfIndex(ifName string) (uint32, error) {
	iface, err := netlink.LinkByName(ifName)
	if err != nil {
		return 0, err
	}
	return uint32(iface.Attrs().Index), nil
}

type ifNameCache struct {
	mu          lock.RWMutex
	indexToName map[int]string
}

func (c *ifNameCache) syncCache() error {
	links, err := netlink.LinkList()
	if err != nil {
		return err
	}

	indexToName := make(map[int]string, len(links))
	for _, link := range links {
		indexToName[link.Attrs().Index] = link.Attrs().Name
	}

	c.mu.Lock()
	c.indexToName = indexToName
	c.mu.Unlock()
	return nil
}

func (c *ifNameCache) lookupName(ifIndex int) (string, bool) {
	c.mu.RLock()
	defer c.mu.RUnlock()

	name, ok := c.indexToName[ifIndex]
	return name, ok
}

// GetIfNameCached returns the name of an interface (if it exists) by looking
// it up in a regularly updated cache
func GetIfNameCached(ifIndex int) (string, bool) {
	return linkCache.lookupName(ifIndex)
}
<commit_msg>datapath/link: Use controller to track link sync<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright 2016-2017 Authors of Cilium

package link

import (
	"context"
	"fmt"
	"sync"
	"time"

	"github.com/vishvananda/netlink"

	"github.com/cilium/cilium/pkg/controller"
	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/mac"
)

var (
	// linkCache is the singleton instance of the LinkCache, only needed to
	// ensure that the single controller used to update the LinkCache is
	// triggered exactly once and the same instance is handed to all users.
	linkCache LinkCache
	once      sync.Once
)

func init() {
	NewLinkCache()
}

// DeleteByName deletes the interface with the name ifName.
func DeleteByName(ifName string) error {
	iface, err := netlink.LinkByName(ifName)
	if err != nil {
		return fmt.Errorf("failed to lookup %q: %v", ifName, err)
	}

	if err = netlink.LinkDel(iface); err != nil {
		return fmt.Errorf("failed to delete %q: %v", ifName, err)
	}

	return nil
}

// Rename renames a network link
func Rename(curName, newName string) error {
	link, err := netlink.LinkByName(curName)
	if err != nil {
		return err
	}

	return netlink.LinkSetName(link, newName)
}

func GetHardwareAddr(ifName string) (mac.MAC, error) {
	iface, err := netlink.LinkByName(ifName)
	if err != nil {
		return nil, err
	}
	return mac.MAC(iface.Attrs().HardwareAddr), nil
}

func GetIfIndex(ifName string) (uint32, error) {
	iface, err := netlink.LinkByName(ifName)
	if err != nil {
		return 0, err
	}
	return uint32(iface.Attrs().Index), nil
}

type LinkCache struct {
	mu          lock.RWMutex
	indexToName map[int]string
}

// NewLinkCache begins monitoring local interfaces for changes in order to
// track local link information.
func NewLinkCache() *LinkCache {
	once.Do(func() {
		linkCache = LinkCache{}
		controller.NewManager().UpdateController("link-cache",
			controller.ControllerParams{
				RunInterval: 15 * time.Second,
				DoFunc: func(ctx context.Context) error {
					return linkCache.syncCache()
				},
			},
		)
	})

	return &linkCache
}

func (c *LinkCache) syncCache() error {
	links, err := netlink.LinkList()
	if err != nil {
		return err
	}

	indexToName := make(map[int]string, len(links))
	for _, link := range links {
		indexToName[link.Attrs().Index] = link.Attrs().Name
	}

	c.mu.Lock()
	c.indexToName = indexToName
	c.mu.Unlock()
	return nil
}

func (c *LinkCache) lookupName(ifIndex int) (string, bool) {
	c.mu.RLock()
	defer c.mu.RUnlock()

	name, ok := c.indexToName[ifIndex]
	return name, ok
}

// GetIfNameCached returns the name of an interface (if it exists) by looking
// it up in a regularly updated cache
func GetIfNameCached(ifIndex int) (string, bool) {
	return linkCache.lookupName(ifIndex)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package store

import (
	"fmt"

	apiv1 "k8s.io/api/core/v1"
	"k8s.io/client-go/tools/cache"
)

// IngressLister makes a Store that lists Ingress.
type IngressLister struct {
	cache.Store
}

// SecretsLister makes a Store that lists Secrets.
type SecretLister struct {
	cache.Store
}

// GetByName searches for a secret in the local secrets Store
func (sl *SecretLister) GetByName(name string) (*apiv1.Secret, error) {
	s, exists, err := sl.GetByKey(name)
	if err != nil {
		return nil, err
	}
	if !exists {
		return nil, fmt.Errorf("secret %v was not found", name)
	}
	return s.(*apiv1.Secret), nil
}

// ConfigMapLister makes a Store that lists Configmaps.
type ConfigMapLister struct {
	cache.Store
}

// GetByName searches for a configmap in the local configmaps Store
func (cml *ConfigMapLister) GetByName(name string) (*apiv1.ConfigMap, error) {
	s, exists, err := cml.GetByKey(name)
	if err != nil {
		return nil, err
	}
	if !exists {
		return nil, fmt.Errorf("configmap %v was not found", name)
	}
	return s.(*apiv1.ConfigMap), nil
}

// ServiceLister makes a Store that lists Services.
type ServiceLister struct {
	cache.Store
}

// GetByName searches for a service in the local secrets Store
func (sl *ServiceLister) GetByName(name string) (*apiv1.Service, error) {
	s, exists, err := sl.GetByKey(name)
	if err != nil {
		return nil, err
	}
	if !exists {
		return nil, fmt.Errorf("service %v was not found", name)
	}
	return s.(*apiv1.Service), nil
}

// NodeLister makes a Store that lists Nodes.
type NodeLister struct {
	cache.Store
}

// EndpointLister makes a Store that lists Endpoints.
type EndpointLister struct {
	cache.Store
}

// GetServiceEndpoints returns the endpoints of a service, matched on service name.
func (s *EndpointLister) GetServiceEndpoints(svc *apiv1.Service) (ep apiv1.Endpoints, err error) {
	for _, m := range s.Store.List() {
		ep = *m.(*apiv1.Endpoints)
		if svc.Name == ep.Name && svc.Namespace == ep.Namespace {
			return ep, nil
		}
	}
	err = fmt.Errorf("could not find endpoints for service: %v", svc.Name)
	return
}
<commit_msg><commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package store

import (
	"fmt"

	apiv1 "k8s.io/api/core/v1"
	"k8s.io/client-go/tools/cache"
)

// IngressLister makes a Store that lists Ingress.
type IngressLister struct {
	cache.Store
}

// SecretLister makes a Store that lists Secrets.
type SecretLister struct {
	cache.Store
}

// GetByName searches for a secret in the local secrets Store
func (sl *SecretLister) GetByName(name string) (*apiv1.Secret, error) {
	s, exists, err := sl.GetByKey(name)
	if err != nil {
		return nil, err
	}
	if !exists {
		return nil, fmt.Errorf("secret %v was not found", name)
	}
	return s.(*apiv1.Secret), nil
}

// ConfigMapLister makes a Store that lists Configmaps.
type ConfigMapLister struct {
	cache.Store
}

// GetByName searches for a configmap in the local configmaps Store
func (cml *ConfigMapLister) GetByName(name string) (*apiv1.ConfigMap, error) {
	s, exists, err := cml.GetByKey(name)
	if err != nil {
		return nil, err
	}
	if !exists {
		return nil, fmt.Errorf("configmap %v was not found", name)
	}
	return s.(*apiv1.ConfigMap), nil
}

// ServiceLister makes a Store that lists Services.
type ServiceLister struct {
	cache.Store
}

// GetByName searches for a service in the local secrets Store
func (sl *ServiceLister) GetByName(name string) (*apiv1.Service, error) {
	s, exists, err := sl.GetByKey(name)
	if err != nil {
		return nil, err
	}
	if !exists {
		return nil, fmt.Errorf("service %v was not found", name)
	}
	return s.(*apiv1.Service), nil
}

// EndpointLister makes a Store that lists Endpoints.
type EndpointLister struct {
	cache.Store
}

// GetServiceEndpoints returns the endpoints of a service, matched on service name.
func (s *EndpointLister) GetServiceEndpoints(svc *apiv1.Service) (*apiv1.Endpoints, error) {
	for _, m := range s.Store.List() {
		ep := m.(*apiv1.Endpoints)
		if svc.Name == ep.Name && svc.Namespace == ep.Namespace {
			return ep, nil
		}
	}
	return nil, fmt.Errorf("could not find endpoints for service: %v", svc.Name)
}
<|endoftext|>


<commit_before>package merger

// ListMerger solves `threshold`-occurrence problem:
// For given inverted lists find the set of strings ids, that appears at least
// `threshold` times.
type ListMerger interface {
	// Merge returns list of candidates, that appears at least `threshold` times.
	Merge(rid Rid, threshold int) ([]MergeCandidate, error)
}

// Rid represents inverted lists for ListMerger
type Rid []ListIterator

// Len is the number of elements in the collection.
func (p Rid) Len() int { return len(p) }

// Less reports whether the element with
// index i should sort before the element with index j.
func (p Rid) Less(i, j int) bool { return p[i].Len() < p[j].Len() }

// Swap swaps the elements with indexes i and j.
func (p Rid) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

// MergeCandidate is result of merging Rid
type MergeCandidate uint64

func NewMergeCandidate(position uint32, overlap int) MergeCandidate {
	return MergeCandidate(uint64(position) << 32 | uint64(overlap))
}

func (m MergeCandidate) Position() uint32 {
	return uint32(m >> 32)
}

func (m MergeCandidate) Overlap() int {
	return int(uint32(m))
}

func (m *MergeCandidate) Increment() {
	*m = NewMergeCandidate(m.Position(), m.Overlap() + 1)
}

<commit_msg>Update docs for MergeCandidate type<commit_after>package merger

// ListMerger solves `threshold`-occurrence problem:
// For given inverted lists find the set of strings ids, that appears at least
// `threshold` times.
type ListMerger interface {
	// Merge returns list of candidates, that appears at least `threshold` times.
	Merge(rid Rid, threshold int) ([]MergeCandidate, error)
}

// Rid represents inverted lists for ListMerger
type Rid []ListIterator

// Len is the number of elements in the collection.
func (p Rid) Len() int { return len(p) }

// Less reports whether the element with
// index i should sort before the element with index j.
func (p Rid) Less(i, j int) bool { return p[i].Len() < p[j].Len() }

// Swap swaps the elements with indexes i and j.
func (p Rid) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

// MergeCandidate is result of merging Rid
type MergeCandidate uint64

// NewMergeCandidate creates a new instance of MergeCandidate
func NewMergeCandidate(position uint32, overlap int) MergeCandidate {
	return MergeCandidate(uint64(position) << 32 | uint64(overlap))
}

// Position returns the given position of the candidate
func (m MergeCandidate) Position() uint32 {
	return uint32(m >> 32)
}

// Overlap returns the current overlap count of the candidate
func (m MergeCandidate) Overlap() int {
	return int(uint32(m))
}

// Increment increments the overlap value of the candidate
func (m *MergeCandidate) Increment() {
	*m = NewMergeCandidate(m.Position(), m.Overlap() + 1)
}

<|endoftext|>


<commit_before>package transport

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"time"
)

func NewListener(addr string, info TLSInfo) (net.Listener, error) {
	l, err := net.Listen("tcp", addr)
	if err != nil {
		return nil, err
	}

	if !info.Empty() {
		cfg, err := info.ServerConfig()
		if err != nil {
			return nil, err
		}

		l = tls.NewListener(l, cfg)
	}

	return l, nil
}

func NewTransport(info TLSInfo) (*http.Transport, error) {
	t := &http.Transport{
		// timeouts taken from http.DefaultTransport
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
	}

	if !info.Empty() {
		tlsCfg, err := info.ClientConfig()
		if err != nil {
			return nil, err
		}
		t.TLSClientConfig = tlsCfg
	}

	return t, nil
}

type TLSInfo struct {
	CertFile string
	KeyFile  string
	CAFile   string

	// parseFunc exists to simplify testing. Typically, parseFunc
	// should be left nil. In that case, tls.X509KeyPair will be used.
	parseFunc func([]byte, []byte) (tls.Certificate, error)
}

func (info TLSInfo) Empty() bool {
	return info.CertFile == "" && info.KeyFile == ""
}

func (info TLSInfo) baseConfig() (*tls.Config, error) {
	if info.KeyFile == "" || info.CertFile == "" {
		return nil, fmt.Errorf("KeyFile and CertFile must both be present[key: %v, cert: %v]", info.KeyFile, info.CertFile)
	}

	cert, err := ioutil.ReadFile(info.CertFile)
	if err != nil {
		return nil, err
	}

	key, err := ioutil.ReadFile(info.KeyFile)
	if err != nil {
		return nil, err
	}

	parseFunc := info.parseFunc
	if parseFunc == nil {
		parseFunc = tls.X509KeyPair
	}

	tlsCert, err := parseFunc(cert, key)
	if err != nil {
		return nil, err
	}

	var cfg tls.Config
	cfg.Certificates = []tls.Certificate{tlsCert}
	return &cfg, nil
}

// ServerConfig generates a tls.Config object for use by an HTTP server
func (info TLSInfo) ServerConfig() (*tls.Config, error) {
	cfg, err := info.baseConfig()
	if err != nil {
		return nil, err
	}

	if info.CAFile != "" {
		cfg.ClientAuth = tls.RequireAndVerifyClientCert
		cp, err := newCertPool(info.CAFile)
		if err != nil {
			return nil, err
		}
		cfg.ClientCAs = cp
	} else {
		cfg.ClientAuth = tls.NoClientCert
	}

	return cfg, nil
}

// ClientConfig generates a tls.Config object for use by an HTTP client
func (info TLSInfo) ClientConfig() (*tls.Config, error) {
	cfg, err := info.baseConfig()
	if err != nil {
		return nil, err
	}

	if info.CAFile != "" {
		cp, err := newCertPool(info.CAFile)
		if err != nil {
			return nil, err
		}

		cfg.RootCAs = cp
	}

	return cfg, nil
}

// newCertPool creates x509 certPool with provided CA file
func newCertPool(CAFile string) (*x509.CertPool, error) {
	certPool := x509.NewCertPool()
	pemByte, err := ioutil.ReadFile(CAFile)
	if err != nil {
		return nil, err
	}

	for {
		var block *pem.Block
		block, pemByte = pem.Decode(pemByte)
		if block == nil {
			return certPool, nil
		}
		cert, err := x509.ParseCertificate(block.Bytes)
		if err != nil {
			return nil, err
		}
		certPool.AddCert(cert)
	}
}
<commit_msg>pkg: set minimum TLS version to 1.0 (disable SSL3)<commit_after>package transport

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"time"
)

func NewListener(addr string, info TLSInfo) (net.Listener, error) {
	l, err := net.Listen("tcp", addr)
	if err != nil {
		return nil, err
	}

	if !info.Empty() {
		cfg, err := info.ServerConfig()
		if err != nil {
			return nil, err
		}

		l = tls.NewListener(l, cfg)
	}

	return l, nil
}

func NewTransport(info TLSInfo) (*http.Transport, error) {
	t := &http.Transport{
		// timeouts taken from http.DefaultTransport
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
	}

	if !info.Empty() {
		tlsCfg, err := info.ClientConfig()
		if err != nil {
			return nil, err
		}
		t.TLSClientConfig = tlsCfg
	}

	return t, nil
}

type TLSInfo struct {
	CertFile string
	KeyFile  string
	CAFile   string

	// parseFunc exists to simplify testing. Typically, parseFunc
	// should be left nil. In that case, tls.X509KeyPair will be used.
	parseFunc func([]byte, []byte) (tls.Certificate, error)
}

func (info TLSInfo) Empty() bool {
	return info.CertFile == "" && info.KeyFile == ""
}

func (info TLSInfo) baseConfig() (*tls.Config, error) {
	if info.KeyFile == "" || info.CertFile == "" {
		return nil, fmt.Errorf("KeyFile and CertFile must both be present[key: %v, cert: %v]", info.KeyFile, info.CertFile)
	}

	cert, err := ioutil.ReadFile(info.CertFile)
	if err != nil {
		return nil, err
	}

	key, err := ioutil.ReadFile(info.KeyFile)
	if err != nil {
		return nil, err
	}

	parseFunc := info.parseFunc
	if parseFunc == nil {
		parseFunc = tls.X509KeyPair
	}

	tlsCert, err := parseFunc(cert, key)
	if err != nil {
		return nil, err
	}

	cfg := &tls.Config{
		Certificates: []tls.Certificate{tlsCert},
		MinVersion:   tls.VersionTLS10,
	}
	return cfg, nil
}

// ServerConfig generates a tls.Config object for use by an HTTP server
func (info TLSInfo) ServerConfig() (*tls.Config, error) {
	cfg, err := info.baseConfig()
	if err != nil {
		return nil, err
	}

	if info.CAFile != "" {
		cfg.ClientAuth = tls.RequireAndVerifyClientCert
		cp, err := newCertPool(info.CAFile)
		if err != nil {
			return nil, err
		}
		cfg.ClientCAs = cp
	} else {
		cfg.ClientAuth = tls.NoClientCert
	}

	return cfg, nil
}

// ClientConfig generates a tls.Config object for use by an HTTP client
func (info TLSInfo) ClientConfig() (*tls.Config, error) {
	cfg, err := info.baseConfig()
	if err != nil {
		return nil, err
	}

	if info.CAFile != "" {
		cp, err := newCertPool(info.CAFile)
		if err != nil {
			return nil, err
		}

		cfg.RootCAs = cp
	}

	return cfg, nil
}

// newCertPool creates x509 certPool with provided CA file
func newCertPool(CAFile string) (*x509.CertPool, error) {
	certPool := x509.NewCertPool()
	pemByte, err := ioutil.ReadFile(CAFile)
	if err != nil {
		return nil, err
	}

	for {
		var block *pem.Block
		block, pemByte = pem.Decode(pemByte)
		if block == nil {
			return certPool, nil
		}
		cert, err := x509.ParseCertificate(block.Bytes)
		if err != nil {
			return nil, err
		}
		certPool.AddCert(cert)
	}
}
<|endoftext|>


<commit_before>package true_git

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/flant/logboek"
	"github.com/flant/werf/pkg/lock"
)

type WithWorkTreeOptions struct {
	HasSubmodules bool
}

func WithWorkTree(gitDir, workTreeCacheDir string, commit string, opts WithWorkTreeOptions, f func(workTreeDir string) error) error {
	return withWorkTreeCacheLock(workTreeCacheDir, func() error {
		var err error

		gitDir, err = filepath.Abs(gitDir)
		if err != nil {
			return fmt.Errorf("bad git dir %s: %s", gitDir, err)
		}

		workTreeCacheDir, err = filepath.Abs(workTreeCacheDir)
		if err != nil {
			return fmt.Errorf("bad work tree cache dir %s: %s", workTreeCacheDir, err)
		}

		if opts.HasSubmodules {
			err := checkSubmoduleConstraint()
			if err != nil {
				return err
			}
		}

		workTreeDir, err := prepareWorkTree(gitDir, workTreeCacheDir, commit, opts.HasSubmodules)
		if err != nil {
			return fmt.Errorf("cannot prepare worktree: %s", err)
		}

		return f(workTreeDir)
	})
}

func withWorkTreeCacheLock(workTreeCacheDir string, f func() error) error {
	lockName := fmt.Sprintf("git_work_tree_cache %s", workTreeCacheDir)
	return lock.WithLock(lockName, lock.LockOptions{Timeout: 600 * time.Second}, f)
}

func prepareWorkTree(repoDir, workTreeCacheDir string, commit string, withSubmodules bool) (string, error) {
	if err := os.MkdirAll(workTreeCacheDir, os.ModePerm); err != nil {
		return "", fmt.Errorf("unable to create dir %s: %s", workTreeCacheDir, err)
	}

	gitDirPath := filepath.Join(workTreeCacheDir, "git_dir")
	if _, err := os.Stat(gitDirPath); os.IsNotExist(err) {
		if err := ioutil.WriteFile(gitDirPath, []byte(repoDir+"\n"), 0644); err != nil {
			return "", fmt.Errorf("error writing %s: %s", gitDirPath, err)
		}
	} else if err != nil {
		return "", fmt.Errorf("unable to access %s: %s", gitDirPath, err)
	}

	currentCommitPath := filepath.Join(workTreeCacheDir, "current_commit")
	worktreePath := filepath.Join(workTreeCacheDir, "worktree")
	currentCommit := ""

	currentCommitExists := true
	if _, err := os.Stat(currentCommitPath); os.IsNotExist(err) {
		currentCommitExists = false
	} else if err != nil {
		return "", fmt.Errorf("unable to access %s: %s", currentCommitPath, err)
	}
	if currentCommitExists {
		if data, err := ioutil.ReadFile(currentCommitPath); err == nil {
			currentCommit = strings.TrimSpace(string(data))

			if currentCommit == commit {
				return worktreePath, nil
			}
		} else {
			return "", fmt.Errorf("error reading %s: %s", currentCommitPath, err)
		}

		if err := os.RemoveAll(currentCommitPath); err != nil {
			return "", fmt.Errorf("unable to remove %s: %s", currentCommitPath, err)
		}
	}

	// Switch worktree state to the desired commit.
	// If worktree already exists — it will be used as a cache.
	workTreeDir := filepath.Join(workTreeCacheDir, "worktree")
	logProcessMsg := fmt.Sprintf("Switch work tree %s to commit %s", workTreeDir, commit)
	if err := logboek.LogProcess(logProcessMsg, logboek.LogProcessOptions{}, func() error {
		logboek.LogInfoF("Work tree dir: %s\n", workTreeDir)
		if currentCommit != "" {
			logboek.LogInfoF("Current commit: %s\n", currentCommit)
		}

		if err := switchWorkTree(repoDir, workTreeDir, commit, withSubmodules); err != nil {
			logboek.LogInfoF("WARNING Work tree switch have failed: %s\n")
			logboek.LogInfoF("WARNING Will empty work tree cache and retry work tree files preparation\n")

			if err := os.RemoveAll(workTreeDir); err != nil {
				return fmt.Errorf("Unable to remove %s to retry work tree switch: %s", workTreeDir, err)
			}
			return switchWorkTree(repoDir, workTreeDir, commit, withSubmodules)
		}

		return nil
	}); err != nil {
		return "", fmt.Errorf("unable to switch work tree %s to commit %s: %s", workTreeDir, commit, err)
	}

	if err := ioutil.WriteFile(currentCommitPath, []byte(commit+"\n"), 0644); err != nil {
		return "", fmt.Errorf("error writing %s: %s", currentCommitPath, err)
	}

	return workTreeDir, nil
}

func debugWorktreeSwitch() bool {
	return os.Getenv("WERF_TRUE_GIT_DEBUG_WORKTREE_SWITCH") == "1"
}

func switchWorkTree(repoDir, workTreeDir string, commit string, withSubmodules bool) error {
	var err error

	err = os.MkdirAll(workTreeDir, os.ModePerm)
	if err != nil {
		return fmt.Errorf("unable to create work tree dir %s: %s", workTreeDir, err)
	}

	var cmd *exec.Cmd
	var output *bytes.Buffer

	cmd = exec.Command(
		"git", "-c", "core.autocrlf=false", "--git-dir", repoDir, "--work-tree", workTreeDir,
		"reset", "--hard", commit,
	)
	output = setCommandRecordingLiveOutput(cmd)
	if debugWorktreeSwitch() {
		fmt.Printf("[DEBUG WORKTREE SWITCH] %s\n", strings.Join(append([]string{cmd.Path}, cmd.Args[1:]...), " "))
	}
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("git reset failed: %s\n%s", err, output.String())
	}

	cmd = exec.Command(
		"git", "--git-dir", repoDir, "--work-tree", workTreeDir,
		"clean", "-d", "-f", "-f", "-x",
	)
	output = setCommandRecordingLiveOutput(cmd)
	if debugWorktreeSwitch() {
		fmt.Printf("[DEBUG WORKTREE SWITCH] %s\n", strings.Join(append([]string{cmd.Path}, cmd.Args[1:]...), " "))
	}
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("git clean failed: %s\n%s", err, output.String())
	}

	if withSubmodules {
		var err error

		err = syncSubmodules(repoDir, workTreeDir)
		if err != nil {
			return fmt.Errorf("cannot sync submodules: %s", err)
		}

		err = updateSubmodules(repoDir, workTreeDir)
		if err != nil {
			return fmt.Errorf("cannot update submodules: %s", err)
		}

		cmd = exec.Command(
			"git", "--git-dir", repoDir, "--work-tree", workTreeDir,
			"submodule", "foreach", "--recursive",
			"git", "-c", "core.autocrlf=false", "reset", "--hard",
		)
		cmd.Dir = workTreeDir // required for `git submodule` to work
		output = setCommandRecordingLiveOutput(cmd)
		if debugWorktreeSwitch() {
			fmt.Printf("[DEBUG WORKTREE SWITCH] %s\n", strings.Join(append([]string{cmd.Path}, cmd.Args[1:]...), " "))
		}
		err = cmd.Run()
		if err != nil {
			return fmt.Errorf("git submodules reset failed: %s\n%s", err, output.String())
		}

		cmd = exec.Command(
			"git", "--git-dir", repoDir, "--work-tree", workTreeDir,
			"submodule", "foreach", "--recursive",
			"git", "clean", "-d", "-f", "-f", "-x",
		)
		cmd.Dir = workTreeDir // required for `git submodule` to work
		output = setCommandRecordingLiveOutput(cmd)
		if debugWorktreeSwitch() {
			fmt.Printf("[DEBUG WORKTREE SWITCH] %s\n", strings.Join(append([]string{cmd.Path}, cmd.Args[1:]...), " "))
		}
		err = cmd.Run()
		if err != nil {
			return fmt.Errorf("git submodules clean failed: %s\n%s", err, output.String())
		}
	}

	return nil
}
<commit_msg>[git] Handle invalid work tree cache when git repo of the work tree gone (2)<commit_after>package true_git

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/flant/logboek"
	"github.com/flant/werf/pkg/lock"
)

type WithWorkTreeOptions struct {
	HasSubmodules bool
}

func WithWorkTree(gitDir, workTreeCacheDir string, commit string, opts WithWorkTreeOptions, f func(workTreeDir string) error) error {
	return withWorkTreeCacheLock(workTreeCacheDir, func() error {
		var err error

		gitDir, err = filepath.Abs(gitDir)
		if err != nil {
			return fmt.Errorf("bad git dir %s: %s", gitDir, err)
		}

		workTreeCacheDir, err = filepath.Abs(workTreeCacheDir)
		if err != nil {
			return fmt.Errorf("bad work tree cache dir %s: %s", workTreeCacheDir, err)
		}

		if opts.HasSubmodules {
			err := checkSubmoduleConstraint()
			if err != nil {
				return err
			}
		}

		workTreeDir, err := prepareWorkTree(gitDir, workTreeCacheDir, commit, opts.HasSubmodules)
		if err != nil {
			return fmt.Errorf("cannot prepare worktree: %s", err)
		}

		return f(workTreeDir)
	})
}

func withWorkTreeCacheLock(workTreeCacheDir string, f func() error) error {
	lockName := fmt.Sprintf("git_work_tree_cache %s", workTreeCacheDir)
	return lock.WithLock(lockName, lock.LockOptions{Timeout: 600 * time.Second}, f)
}

func checkIsWorkTreeValid(repoDir, workTreeDir, repoToCacheLinkFilePath string) (bool, error) {
	if _, err := os.Stat(repoToCacheLinkFilePath); err == nil {
		if data, err := ioutil.ReadFile(repoToCacheLinkFilePath); err != nil {
			return false, fmt.Errorf("error reading %s: %s", repoToCacheLinkFilePath, err)
		} else if strings.TrimSpace(string(data)) == workTreeDir {
			return true, nil
		}
	} else if !os.IsNotExist(err) {
		return false, fmt.Errorf("error accessing %s: %s", repoToCacheLinkFilePath, err)
	}

	return false, nil
}

func prepareWorkTree(repoDir, workTreeCacheDir string, commit string, withSubmodules bool) (string, error) {
	if err := os.MkdirAll(workTreeCacheDir, os.ModePerm); err != nil {
		return "", fmt.Errorf("unable to create dir %s: %s", workTreeCacheDir, err)
	}

	gitDirPath := filepath.Join(workTreeCacheDir, "git_dir")
	if _, err := os.Stat(gitDirPath); os.IsNotExist(err) {
		if err := ioutil.WriteFile(gitDirPath, []byte(repoDir+"\n"), 0644); err != nil {
			return "", fmt.Errorf("error writing %s: %s", gitDirPath, err)
		}
	} else if err != nil {
		return "", fmt.Errorf("unable to access %s: %s", gitDirPath, err)
	}

	currentCommitPath := filepath.Join(workTreeCacheDir, "current_commit")
	workTreeDir := filepath.Join(workTreeCacheDir, "worktree")
	repoToCacheLinkFilePath := filepath.Join(repoDir, "werf_work_tree_cache_dir")
	currentCommit := ""

	isWorkTreeDirExist := false
	isWorkTreeValid := true
	if _, err := os.Stat(workTreeDir); err == nil {
		isWorkTreeDirExist = true
		if isValid, err := checkIsWorkTreeValid(repoDir, workTreeDir, repoToCacheLinkFilePath); err != nil {
			return "", fmt.Errorf("unable to check work tree %s validity with repo %s: %s", workTreeDir, repoDir, err)
		} else {
			isWorkTreeValid = isValid
		}
	} else if !os.IsNotExist(err) {
		return "", fmt.Errorf("error accessing %s: %s", workTreeDir, err)
	}

	currentCommitExists := true
	if _, err := os.Stat(currentCommitPath); os.IsNotExist(err) {
		currentCommitExists = false
	} else if err != nil {
		return "", fmt.Errorf("unable to access %s: %s", currentCommitPath, err)
	}
	if currentCommitExists {
		if data, err := ioutil.ReadFile(currentCommitPath); err == nil {
			currentCommit = strings.TrimSpace(string(data))

			if currentCommit == commit && isWorkTreeDirExist && isWorkTreeValid {
				return workTreeDir, nil
			}
		} else {
			return "", fmt.Errorf("error reading %s: %s", currentCommitPath, err)
		}

		if err := os.RemoveAll(currentCommitPath); err != nil {
			return "", fmt.Errorf("unable to remove %s: %s", currentCommitPath, err)
		}
	}

	if isWorkTreeDirExist && !isWorkTreeValid {
		logboek.LogInfoF("Removing invalidated work tree dir %s of repo %s\n", workTreeDir, repoDir)
		if err := os.RemoveAll(workTreeDir); err != nil {
			return "", fmt.Errorf("unable to remove invalidated work tree dir %s: %s", workTreeDir, err)
		}
	}

	// Switch worktree state to the desired commit.
	// If worktree already exists — it will be used as a cache.
	logProcessMsg := fmt.Sprintf("Switch work tree %s to commit %s", workTreeDir, commit)
	if err := logboek.LogProcess(logProcessMsg, logboek.LogProcessOptions{}, func() error {
		logboek.LogInfoF("Work tree dir: %s\n", workTreeDir)
		if currentCommit != "" {
			logboek.LogInfoF("Current commit: %s\n", currentCommit)
		}
		return switchWorkTree(repoDir, workTreeDir, commit, withSubmodules)
	}); err != nil {
		return "", fmt.Errorf("unable to switch work tree %s to commit %s: %s", workTreeDir, commit, err)
	}

	if err := ioutil.WriteFile(repoToCacheLinkFilePath, []byte(workTreeDir+"\n"), 0644); err != nil {
		return "", fmt.Errorf("unable to write %s: %s", repoToCacheLinkFilePath, err)
	}

	if err := ioutil.WriteFile(currentCommitPath, []byte(commit+"\n"), 0644); err != nil {
		return "", fmt.Errorf("error writing %s: %s", currentCommitPath, err)
	}

	return workTreeDir, nil
}

func debugWorktreeSwitch() bool {
	return os.Getenv("WERF_TRUE_GIT_DEBUG_WORKTREE_SWITCH") == "1"
}

func switchWorkTree(repoDir, workTreeDir string, commit string, withSubmodules bool) error {
	var err error

	err = os.MkdirAll(workTreeDir, os.ModePerm)
	if err != nil {
		return fmt.Errorf("unable to create work tree dir %s: %s", workTreeDir, err)
	}

	var cmd *exec.Cmd
	var output *bytes.Buffer

	cmd = exec.Command(
		"git", "-c", "core.autocrlf=false", "--git-dir", repoDir, "--work-tree", workTreeDir,
		"reset", "--hard", commit,
	)
	output = setCommandRecordingLiveOutput(cmd)
	if debugWorktreeSwitch() {
		fmt.Printf("[DEBUG WORKTREE SWITCH] %s\n", strings.Join(append([]string{cmd.Path}, cmd.Args[1:]...), " "))
	}
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("git reset failed: %s\n%s", err, output.String())
	}

	cmd = exec.Command(
		"git", "--git-dir", repoDir, "--work-tree", workTreeDir,
		"clean", "-d", "-f", "-f", "-x",
	)
	output = setCommandRecordingLiveOutput(cmd)
	if debugWorktreeSwitch() {
		fmt.Printf("[DEBUG WORKTREE SWITCH] %s\n", strings.Join(append([]string{cmd.Path}, cmd.Args[1:]...), " "))
	}
	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("git clean failed: %s\n%s", err, output.String())
	}

	if withSubmodules {
		var err error

		err = syncSubmodules(repoDir, workTreeDir)
		if err != nil {
			return fmt.Errorf("cannot sync submodules: %s", err)
		}

		err = updateSubmodules(repoDir, workTreeDir)
		if err != nil {
			return fmt.Errorf("cannot update submodules: %s", err)
		}

		cmd = exec.Command(
			"git", "--git-dir", repoDir, "--work-tree", workTreeDir,
			"submodule", "foreach", "--recursive",
			"git", "-c", "core.autocrlf=false", "reset", "--hard",
		)
		cmd.Dir = workTreeDir // required for `git submodule` to work
		output = setCommandRecordingLiveOutput(cmd)
		if debugWorktreeSwitch() {
			fmt.Printf("[DEBUG WORKTREE SWITCH] %s\n", strings.Join(append([]string{cmd.Path}, cmd.Args[1:]...), " "))
		}
		err = cmd.Run()
		if err != nil {
			return fmt.Errorf("git submodules reset failed: %s\n%s", err, output.String())
		}

		cmd = exec.Command(
			"git", "--git-dir", repoDir, "--work-tree", workTreeDir,
			"submodule", "foreach", "--recursive",
			"git", "clean", "-d", "-f", "-f", "-x",
		)
		cmd.Dir = workTreeDir // required for `git submodule` to work
		output = setCommandRecordingLiveOutput(cmd)
		if debugWorktreeSwitch() {
			fmt.Printf("[DEBUG WORKTREE SWITCH] %s\n", strings.Join(append([]string{cmd.Path}, cmd.Args[1:]...), " "))
		}
		err = cmd.Run()
		if err != nil {
			return fmt.Errorf("git submodules clean failed: %s\n%s", err, output.String())
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package middleware

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"strings"
	"time"

	"github.com/HouzuoGuo/laitos/datastruct"
	"github.com/HouzuoGuo/laitos/inet"
	"github.com/HouzuoGuo/laitos/lalog"
	"github.com/HouzuoGuo/laitos/misc"
	"github.com/aws/aws-xray-sdk-go/xray"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	// PrometheusHandlerTypeLabel is the name of data label given to prometheus observers, the label data shall be the symbol name of the HTTP handler's type.
	PrometheusHandlerTypeLabel = "handler_type"
	// PrometheusHandlerLocationLabel is the name of data label given to prometheus observers, the label data shall be the URL location at which HTTP handler is installed.
	PrometheusHandlerLocationLabel = "url_location"
	// PrometheusHandlerHostLabel is the name of data label given to prometheus observers, the label data shall be the intended host (example.com:443) requested by the client.
	PrometheusHandlerHostLabel = "host"
	// MaxLatestRequests is the maximum number of latest HTTP requests to be
	// kept in-memory for on-demand inspection.
	MaxLatestRequests = 200
)

var (
	// LatestRequests is a ring buffer of the latest HTTP requests processed by
	// HTTP daemons. HandleLatestRequestsInspector will be retrieving these
	// captured requests and present them to user for inspection.
	LatestRequests *datastruct.RingBuffer

	// EnableLatestRequestsRecording determines whether the default set of
	// middleware installed to HTTP daemons will be capturing the latest HTTP
	// requests into a ring buffer for inspection. HandleLatestRequestsInspector
	// will be turning it on and off.
	EnableLatestRequestsRecording bool
)

func init() {
	LatestRequests = datastruct.NewRingBuffer(MaxLatestRequests)
}

/*
GetRealClientIP returns the IP of HTTP client that initiated the HTTP request.
Usually, the return value is identical to IP portion of RemoteAddr, but if there is a proxy server in between,
such as a load balancer or LAN proxy, the return value will be the client IP address read from header
"X-Real-Ip" (preferred) or "X-Forwarded-For".
*/
func GetRealClientIP(r *http.Request) string {
	if colon := strings.LastIndexByte(r.RemoteAddr, ':'); colon > 1 {
		ip := r.RemoteAddr[:strings.LastIndexByte(r.RemoteAddr, ':')]
		if strings.HasPrefix(ip, "127.") {
			if realIP := r.Header["X-Real-Ip"]; len(realIP) > 0 {
				ip = realIP[0]
			} else if forwardedFor := r.Header["X-Forwarded-For"]; len(forwardedFor) > 0 {
				// X-Forwarded-For value looks like "1.1.1.1[, 2.2.2.2, 3.3.3.3 ...]" where the first IP is the client IP
				split := strings.Split(forwardedFor[0], ",")
				if len(split) > 0 {
					ip = split[0]
				}
			}
		}
		return ip
	}
	return ""
}

// RecordInternalStats decorates the HTTP handler function by recording the request handing duration in internal stats.
func RecordInternalStats(stats *misc.Stats, next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		// Record the duration of request handling in stats
		beginTime := time.Now()
		defer func() {
			stats.Trigger(float64(time.Now().UnixNano() - beginTime.UnixNano()))
		}()
		next(w, r)
	}
}

// WithAWSXray decorates the HTTP handler function using AWS x-ray library for distributed tracing.
func WithAWSXray(next http.HandlerFunc) http.HandlerFunc {
	if misc.EnableAWSIntegration && inet.IsAWS() {
		// Integrate the decorated handler with AWS x-ray. The crucial x-ray daemon program seems to be only capable of running on AWS compute resources.
		return xray.Handler(xray.NewDynamicSegmentNamer("LaitosHTTPD", "*"), next).ServeHTTP
	}
	return next
}

// RateLimit decorates the HTTP handler function by applying a rate limit to the client, identified by its IP.
// If the client has made too many requests, it will respond to the client with HTTP status too-many-requests, without invoking the next handler function.
func RateLimit(rateLimit *misc.RateLimit, next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		remoteIP := GetRealClientIP(r)
		if !rateLimit.Add(remoteIP, true) {
			http.Error(w, "", http.StatusTooManyRequests)
			return
		}
		next(w, r)
	}
}

// EmergencyLockdown decorates the HTTP handler function by determining whether the program-wide emergency lock-down is in-effect.
// If the lock-down is in effect, the HTTP client will get an empty (albeit successful) response, without invoking the next handler function.
func EmergencyLockdown(next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if misc.EmergencyLockDown {
			/*
				An error response usually should carry status 5xx in this case, but the intention of
				emergency stop is to disable the program rather than crashing it and relaunching it.
				If an external trigger such as load balancer health check knocks on HTTP endpoint and restarts
				the program after consecutive HTTP failures, it would defeat the intention of emergency stop.
				Hence the status code here is OK.
			*/
			_, _ = w.Write([]byte(misc.ErrEmergencyLockDown.Error()))
			return
		}
		next(w, r)
	}
}

// RecordPrometheusStats decorates the HTTP handler function by recording its execution stats with prometheus.
// The recorded stats will be exposed on an HTTP endpoint dedicated to reading prometheus metrics.
func RecordPrometheusStats(
	handlerTypeLabel, handlerLocationLabel string,
	durationHistogram, timeToFirstByteHistogram, responseSizeHistogram *prometheus.HistogramVec,
	next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !misc.EnablePrometheusIntegration {
			next(w, r)
			return
		}
		beginTime := time.Now()
		responseRecorder := &HTTPResponseRecorder{
			ResponseWriter: w,
			statusCode:     http.StatusOK, // the default status code written by any response writer is always 200 OK
		}
		// Record stats from the hijacked connection only if it is supported by the HTTP protocol.
		// Notably, HTTP/2 handles multiple request-responses simultaneously and hence it cannot support this operation.
		var interceptRecorder *HTTPInterceptRecorder
		if interceptor, ok := w.(http.Hijacker); ok {
			interceptRecorder = &HTTPInterceptRecorder{Hijacker: interceptor}
			responseRecorder.Hijacker = interceptRecorder
		}
		next(responseRecorder, r)
		promLabels := prometheus.Labels{
			PrometheusHandlerTypeLabel:     handlerTypeLabel,
			PrometheusHandlerLocationLabel: handlerLocationLabel,
			PrometheusHandlerHostLabel:     r.Host,
		}
		durationObs := durationHistogram.With(promLabels)
		durationObs.Observe(time.Since(beginTime).Seconds())
		if interceptRecorder != nil && interceptRecorder.ConnRecorder != nil && !interceptRecorder.ConnRecorder.timestampAtWriteCall.IsZero() {
			timeToFirstByteObs := timeToFirstByteHistogram.With(promLabels)
			timeToFirstByteObs.Observe(time.Since(interceptRecorder.ConnRecorder.timestampAtWriteCall).Seconds())
			responseSizeObs := responseSizeHistogram.With(promLabels)
			responseSizeObs.Observe(float64(interceptRecorder.ConnRecorder.totalWritten))
		} else if !responseRecorder.timestampAtWriteCall.IsZero() {
			timeToFirstByteObs := timeToFirstByteHistogram.With(promLabels)
			timeToFirstByteObs.Observe(time.Since(responseRecorder.timestampAtWriteCall).Seconds())
			responseSizeObs := responseSizeHistogram.With(promLabels)
			responseSizeObs.Observe(float64(responseRecorder.totalWritten))
		}
	}
}

// RestrictMaxRequestSize decorates the HTTP handler function by restricting how much of the request body can be read by the next handler function.
// This helps to prevent a malfunctioning HTTP client coupled with a faulty handler to use excessive amount of system memory.
func RestrictMaxRequestSize(maxRequestBodyBytes int, next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxRequestBodyBytes))
		next(w, r)
	}
}

// LogRequestStats decorates the HTTP handler function by logging several the request parameters - enough to identify the handler and request origin,
// as well as execution stats such as time-to-first-byte.
func LogRequestStats(logger lalog.Logger, next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		beginTime := time.Now()
		responseRecorder := &HTTPResponseRecorder{
			ResponseWriter: w,
			statusCode:     http.StatusOK, // the default status code written by any response writer is always 200 OK
		}
		// Record stats from the hijacked connection only if it is supported by the HTTP protocol.
		// Notably, HTTP/2 handles multiple request-responses simultaneously and hence it cannot support this operation.
		var interceptRecorder *HTTPInterceptRecorder
		if interceptor, ok := w.(http.Hijacker); ok {
			interceptRecorder = &HTTPInterceptRecorder{Hijacker: interceptor}
			responseRecorder.Hijacker = interceptRecorder
		}
		next(responseRecorder, r)
		processingDuration := time.Since(beginTime)
		var timeToFirstByte time.Duration
		var totalWritten int
		if interceptRecorder != nil && interceptRecorder.ConnRecorder != nil && !interceptRecorder.ConnRecorder.timestampAtWriteCall.IsZero() {
			timeToFirstByte = time.Since(interceptRecorder.ConnRecorder.timestampAtWriteCall)
			totalWritten = interceptRecorder.ConnRecorder.totalWritten
		} else if !responseRecorder.timestampAtWriteCall.IsZero() {
			timeToFirstByte = time.Since(responseRecorder.timestampAtWriteCall)
			totalWritten = responseRecorder.totalWritten
		}
		if timeToFirstByte == 0 {
			logger.Info("decoratedHandler", GetRealClientIP(r), nil, "request: %s \"%s\" %s, Host: %s, user-agent: %s, referer: %s, responded with code %d in %d bytes and %dus",
				r.Method, r.URL.EscapedPath(), r.Proto, r.Host, r.Header.Get("User-Agent"), r.Header.Get("Referer"), responseRecorder.statusCode, totalWritten, processingDuration.Microseconds())
		} else {
			logger.Info("decoratedHandler", GetRealClientIP(r), nil, "request: %s \"%s\" %s, Host: %s, user-agent: %s, referer: %s, responded with code %d in %d bytes and %dus (time to 1st byte %dus)",
				r.Method, r.URL.EscapedPath(), r.Proto, r.Host, r.Header.Get("User-Agent"), r.Header.Get("Referer"), responseRecorder.statusCode, totalWritten, processingDuration.Microseconds(), timeToFirstByte.Microseconds())
		}
	}
}

type bytesReaderCloser struct {
	*bytes.Reader
	io.Closer
}

func (buf bytesReaderCloser) Close() error {
	return nil
}

// RecordLatestRequests records the request body for on-demand inspection.
func RecordLatestRequests(logger lalog.Logger, next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Body != nil && EnableLatestRequestsRecording {
			// Read the entire request into memory.
			requestBody, err := ioutil.ReadAll(r.Body)
			_ = r.Body.Close()
			if err != nil {
				logger.Warning("RecordLatestRequests", GetRealClientIP(r), err, "failed to read request body")
			}
			// Present the copy of request body to DumpRequest.
			r.Body = &bytesReaderCloser{Reader: bytes.NewReader(requestBody)}
			dump, err := httputil.DumpRequest(r, true)
			if err == nil {
				LatestRequests.Push(fmt.Sprintf("From: %s\n%s", GetRealClientIP(r), string(dump)))
			} else {
				logger.Warning("RecordLatestRequests", GetRealClientIP(r), err, "failed to dump request")
			}
			// Present the copy of request body to the next middleware/handler.
			r.Body = &bytesReaderCloser{Reader: bytes.NewReader(requestBody)}

		}
		next(w, r)
	}
}
<commit_msg>httpd: display transfer rate in the middleware<commit_after>package middleware

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"strings"
	"time"

	"github.com/HouzuoGuo/laitos/datastruct"
	"github.com/HouzuoGuo/laitos/inet"
	"github.com/HouzuoGuo/laitos/lalog"
	"github.com/HouzuoGuo/laitos/misc"
	"github.com/aws/aws-xray-sdk-go/xray"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	// PrometheusHandlerTypeLabel is the name of data label given to prometheus observers, the label data shall be the symbol name of the HTTP handler's type.
	PrometheusHandlerTypeLabel = "handler_type"
	// PrometheusHandlerLocationLabel is the name of data label given to prometheus observers, the label data shall be the URL location at which HTTP handler is installed.
	PrometheusHandlerLocationLabel = "url_location"
	// PrometheusHandlerHostLabel is the name of data label given to prometheus observers, the label data shall be the intended host (example.com:443) requested by the client.
	PrometheusHandlerHostLabel = "host"
	// MaxLatestRequests is the maximum number of latest HTTP requests to be
	// kept in-memory for on-demand inspection.
	MaxLatestRequests = 200
)

var (
	// LatestRequests is a ring buffer of the latest HTTP requests processed by
	// HTTP daemons. HandleLatestRequestsInspector will be retrieving these
	// captured requests and present them to user for inspection.
	LatestRequests *datastruct.RingBuffer

	// EnableLatestRequestsRecording determines whether the default set of
	// middleware installed to HTTP daemons will be capturing the latest HTTP
	// requests into a ring buffer for inspection. HandleLatestRequestsInspector
	// will be turning it on and off.
	EnableLatestRequestsRecording bool
)

func init() {
	LatestRequests = datastruct.NewRingBuffer(MaxLatestRequests)
}

/*
GetRealClientIP returns the IP of HTTP client that initiated the HTTP request.
Usually, the return value is identical to IP portion of RemoteAddr, but if there is a proxy server in between,
such as a load balancer or LAN proxy, the return value will be the client IP address read from header
"X-Real-Ip" (preferred) or "X-Forwarded-For".
*/
func GetRealClientIP(r *http.Request) string {
	if colon := strings.LastIndexByte(r.RemoteAddr, ':'); colon > 1 {
		ip := r.RemoteAddr[:strings.LastIndexByte(r.RemoteAddr, ':')]
		if strings.HasPrefix(ip, "127.") {
			if realIP := r.Header["X-Real-Ip"]; len(realIP) > 0 {
				ip = realIP[0]
			} else if forwardedFor := r.Header["X-Forwarded-For"]; len(forwardedFor) > 0 {
				// X-Forwarded-For value looks like "1.1.1.1[, 2.2.2.2, 3.3.3.3 ...]" where the first IP is the client IP
				split := strings.Split(forwardedFor[0], ",")
				if len(split) > 0 {
					ip = split[0]
				}
			}
		}
		return ip
	}
	return ""
}

// RecordInternalStats decorates the HTTP handler function by recording the request handing duration in internal stats.
func RecordInternalStats(stats *misc.Stats, next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		// Record the duration of request handling in stats
		beginTime := time.Now()
		defer func() {
			stats.Trigger(float64(time.Now().UnixNano() - beginTime.UnixNano()))
		}()
		next(w, r)
	}
}

// WithAWSXray decorates the HTTP handler function using AWS x-ray library for distributed tracing.
func WithAWSXray(next http.HandlerFunc) http.HandlerFunc {
	if misc.EnableAWSIntegration && inet.IsAWS() {
		// Integrate the decorated handler with AWS x-ray. The crucial x-ray daemon program seems to be only capable of running on AWS compute resources.
		return xray.Handler(xray.NewDynamicSegmentNamer("LaitosHTTPD", "*"), next).ServeHTTP
	}
	return next
}

// RateLimit decorates the HTTP handler function by applying a rate limit to the client, identified by its IP.
// If the client has made too many requests, it will respond to the client with HTTP status too-many-requests, without invoking the next handler function.
func RateLimit(rateLimit *misc.RateLimit, next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		remoteIP := GetRealClientIP(r)
		if !rateLimit.Add(remoteIP, true) {
			http.Error(w, "", http.StatusTooManyRequests)
			return
		}
		next(w, r)
	}
}

// EmergencyLockdown decorates the HTTP handler function by determining whether the program-wide emergency lock-down is in-effect.
// If the lock-down is in effect, the HTTP client will get an empty (albeit successful) response, without invoking the next handler function.
func EmergencyLockdown(next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if misc.EmergencyLockDown {
			/*
				An error response usually should carry status 5xx in this case, but the intention of
				emergency stop is to disable the program rather than crashing it and relaunching it.
				If an external trigger such as load balancer health check knocks on HTTP endpoint and restarts
				the program after consecutive HTTP failures, it would defeat the intention of emergency stop.
				Hence the status code here is OK.
			*/
			_, _ = w.Write([]byte(misc.ErrEmergencyLockDown.Error()))
			return
		}
		next(w, r)
	}
}

// RecordPrometheusStats decorates the HTTP handler function by recording its execution stats with prometheus.
// The recorded stats will be exposed on an HTTP endpoint dedicated to reading prometheus metrics.
func RecordPrometheusStats(
	handlerTypeLabel, handlerLocationLabel string,
	durationHistogram, timeToFirstByteHistogram, responseSizeHistogram *prometheus.HistogramVec,
	next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !misc.EnablePrometheusIntegration {
			next(w, r)
			return
		}
		beginTime := time.Now()
		responseRecorder := &HTTPResponseRecorder{
			ResponseWriter: w,
			statusCode:     http.StatusOK, // the default status code written by any response writer is always 200 OK
		}
		// Record stats from the hijacked connection only if it is supported by the HTTP protocol.
		// Notably, HTTP/2 handles multiple request-responses simultaneously and hence it cannot support this operation.
		var interceptRecorder *HTTPInterceptRecorder
		if interceptor, ok := w.(http.Hijacker); ok {
			interceptRecorder = &HTTPInterceptRecorder{Hijacker: interceptor}
			responseRecorder.Hijacker = interceptRecorder
		}
		next(responseRecorder, r)
		promLabels := prometheus.Labels{
			PrometheusHandlerTypeLabel:     handlerTypeLabel,
			PrometheusHandlerLocationLabel: handlerLocationLabel,
			PrometheusHandlerHostLabel:     r.Host,
		}
		durationObs := durationHistogram.With(promLabels)
		durationObs.Observe(time.Since(beginTime).Seconds())
		if interceptRecorder != nil && interceptRecorder.ConnRecorder != nil && !interceptRecorder.ConnRecorder.timestampAtWriteCall.IsZero() {
			timeToFirstByteObs := timeToFirstByteHistogram.With(promLabels)
			timeToFirstByteObs.Observe(time.Since(interceptRecorder.ConnRecorder.timestampAtWriteCall).Seconds())
			responseSizeObs := responseSizeHistogram.With(promLabels)
			responseSizeObs.Observe(float64(interceptRecorder.ConnRecorder.totalWritten))
		} else if !responseRecorder.timestampAtWriteCall.IsZero() {
			timeToFirstByteObs := timeToFirstByteHistogram.With(promLabels)
			timeToFirstByteObs.Observe(time.Since(responseRecorder.timestampAtWriteCall).Seconds())
			responseSizeObs := responseSizeHistogram.With(promLabels)
			responseSizeObs.Observe(float64(responseRecorder.totalWritten))
		}
	}
}

// RestrictMaxRequestSize decorates the HTTP handler function by restricting how much of the request body can be read by the next handler function.
// This helps to prevent a malfunctioning HTTP client coupled with a faulty handler to use excessive amount of system memory.
func RestrictMaxRequestSize(maxRequestBodyBytes int, next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxRequestBodyBytes))
		next(w, r)
	}
}

// LogRequestStats decorates the HTTP handler function by logging several the request parameters - enough to identify the handler and request origin,
// as well as execution stats such as time-to-first-byte.
func LogRequestStats(logger lalog.Logger, next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		beginTime := time.Now()
		responseRecorder := &HTTPResponseRecorder{
			ResponseWriter: w,
			statusCode:     http.StatusOK, // the default status code written by any response writer is always 200 OK
		}
		// Record stats from the hijacked connection only if it is supported by the HTTP protocol.
		// Notably, HTTP/2 handles multiple request-responses simultaneously and hence it cannot support this operation.
		var interceptRecorder *HTTPInterceptRecorder
		if interceptor, ok := w.(http.Hijacker); ok {
			interceptRecorder = &HTTPInterceptRecorder{Hijacker: interceptor}
			responseRecorder.Hijacker = interceptRecorder
		}
		next(responseRecorder, r)
		processingDuration := time.Since(beginTime)
		var timeToFirstByte time.Duration
		var totalWritten int
		if interceptRecorder != nil && interceptRecorder.ConnRecorder != nil && !interceptRecorder.ConnRecorder.timestampAtWriteCall.IsZero() {
			timeToFirstByte = time.Since(interceptRecorder.ConnRecorder.timestampAtWriteCall)
			totalWritten = interceptRecorder.ConnRecorder.totalWritten
		} else if !responseRecorder.timestampAtWriteCall.IsZero() {
			timeToFirstByte = time.Since(responseRecorder.timestampAtWriteCall)
			totalWritten = responseRecorder.totalWritten
		}
		kiloBytesPerSec := float64(totalWritten/1000) / (processingDuration.Seconds())
		if timeToFirstByte == 0 {
			logger.Info("decoratedHandler", GetRealClientIP(r), nil, "request: %s \"%s\" %s, Host: %s, user-agent: %s, referer: %s, responded with code %d in %d bytes and %dus (%.2f KB/s)",
				r.Method, r.URL.EscapedPath(), r.Proto, r.Host, r.Header.Get("User-Agent"), r.Header.Get("Referer"), responseRecorder.statusCode, totalWritten, processingDuration.Microseconds(), kiloBytesPerSec)
		} else {
			logger.Info("decoratedHandler", GetRealClientIP(r), nil, "request: %s \"%s\" %s, Host: %s, user-agent: %s, referer: %s, responded with code %d in %d bytes and %dus (%.2f KB/s, time to 1st byte %dus)",
				r.Method, r.URL.EscapedPath(), r.Proto, r.Host, r.Header.Get("User-Agent"), r.Header.Get("Referer"), responseRecorder.statusCode, totalWritten, processingDuration.Microseconds(), kiloBytesPerSec, timeToFirstByte.Microseconds())
		}
	}
}

type bytesReaderCloser struct {
	*bytes.Reader
	io.Closer
}

func (buf bytesReaderCloser) Close() error {
	return nil
}

// RecordLatestRequests records the request body for on-demand inspection.
func RecordLatestRequests(logger lalog.Logger, next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Body != nil && EnableLatestRequestsRecording {
			// Read the entire request into memory.
			requestBody, err := ioutil.ReadAll(r.Body)
			_ = r.Body.Close()
			if err != nil {
				logger.Warning("RecordLatestRequests", GetRealClientIP(r), err, "failed to read request body")
			}
			// Present the copy of request body to DumpRequest.
			r.Body = &bytesReaderCloser{Reader: bytes.NewReader(requestBody)}
			dump, err := httputil.DumpRequest(r, true)
			if err == nil {
				LatestRequests.Push(fmt.Sprintf("From: %s\n%s", GetRealClientIP(r), string(dump)))
			} else {
				logger.Warning("RecordLatestRequests", GetRealClientIP(r), err, "failed to dump request")
			}
			// Present the copy of request body to the next middleware/handler.
			r.Body = &bytesReaderCloser{Reader: bytes.NewReader(requestBody)}

		}
		next(w, r)
	}
}
<|endoftext|>