<commit_before>package scoop_protocol

// KinesisWriterConfig is used to configure a KinesisWriter
import (
	"errors"
	"fmt"
	"time"
)

// AnnotatedKinesisConfig is a Kinesis configuration annotated with meta information.
type AnnotatedKinesisConfig struct {
	AWSAccount       int64
	Team             string
	Version          int
	Contact          string
	Usage            string
	ConsumingLibrary string
	SpadeConfig      KinesisWriterConfig
	LastEditedAt     time.Time
	LastChangedBy    string
	Dropped          bool
	DroppedReason    string
}

// KinesisWriterConfig describes a Kinesis Writer that the processor uses to export data to a Kinesis Stream/Firehose
type KinesisWriterConfig struct {
	StreamName             string
	StreamRole             string
	StreamType             string // StreamType should be either "stream" or "firehose"
	Compress               bool   // true if compress data with flate, false to output json
	FirehoseRedshiftStream bool   // true if JSON destined for Firehose->Redshift streaming
	EventNameTargetField   string // Field name to write the event's name to (useful for uncompressed streams)
	BufferSize             int
	MaxAttemptsPerRecord   int
	RetryDelay             string

	Events map[string]*struct {
		Filter     string
		FilterFunc func(map[string]string) bool `json:"-"`
		Fields     []string
	}

	Globber GlobberConfig
	Batcher BatcherConfig
}

// Validate returns an error if the Kinesis Writer config is not valid, or nil if it is.
// It also sets the FilterFunc on Events with Filters.
func (c *KinesisWriterConfig) Validate() error {
	if c.StreamType == "" || c.StreamName == "" {
		return fmt.Errorf("Mandatory fields stream type and stream name aren't populated")
	}

	err := c.Globber.Validate()
	if err != nil {
		return fmt.Errorf("globber config invalid: %s", err)
	}

	err = c.Batcher.Validate()
	if err != nil {
		return fmt.Errorf("batcher config invalid: %s", err)
	}

	for _, e := range c.Events {
		if e.Filter != "" {
			e.FilterFunc = filterFuncs[e.Filter]
			if e.FilterFunc == nil {
				return fmt.Errorf("batcher config invalid: %s", err)
			}
		}
	}

	if c.FirehoseRedshiftStream && (c.StreamType != "firehose" || c.Compress) {
		return fmt.Errorf("Redshift streaming only valid with non-compressed firehose")
	}

	_, err = time.ParseDuration(c.RetryDelay)
	return err
}

var filterFuncs = map[string]func(map[string]string) bool{
	"isVod": func(fields map[string]string) bool {
		return fields["vod_id"] != "" && fields["vod_type"] != "clip"
	},
	"isUserIDSet": func(fields map[string]string) bool {
		return fields["user_id"] != ""
	},
	"isChannelIDSet": func(fields map[string]string) bool {
		return fields["channel_id"] != ""
	},
}

// BatcherConfig is used to configure a batcher instance
type BatcherConfig struct {
	// MaxSize is the max combined size of the batch
	MaxSize int

	// MaxEntries is the max number of entries that can be batched together
	// if batches does not have an entry limit, set MaxEntries as -1
	MaxEntries int

	// MaxAge is the max age of the oldest entry in the glob
	MaxAge string

	// BufferLength is the length of the channel where newly
	// submitted entries are stored, decreasing the size of this
	// buffer can cause stalls, and increasing the size can increase
	// shutdown time
	BufferLength int
}

// Validate returns an error if the batcher config is invalid, nil otherwise.
func (c *BatcherConfig) Validate() error {
	maxAge, err := time.ParseDuration(c.MaxAge)
	if err != nil {
		return err
	}

	if maxAge <= 0 {
		return errors.New("MaxAge must be a positive value")
	}

	if c.MaxSize <= 0 {
		return errors.New("MaxSize must be a positive value")
	}

	if c.MaxEntries <= 0 && c.MaxEntries != -1 {
		return errors.New("MaxEntries must be a positive value or -1")
	}

	if c.BufferLength == 0 {
		return errors.New("BufferLength must be a positive value")
	}

	return nil
}

// GlobberConfig is used to configure a globber instance
type GlobberConfig struct {
	// MaxSize is the max size per glob before compression
	MaxSize int

	// MaxAge is the max age of the oldest entry in the glob
	MaxAge string

	// BufferLength is the length of the channel where newly
	// submitted entries are stored, decreasing the size of this
	// buffer can cause stalls, and increasing the size can increase
	// shutdown time
	BufferLength int
}

// Validate returns an error if the config is invalid, nil otherwise.
func (c *GlobberConfig) Validate() error {
	maxAge, err := time.ParseDuration(c.MaxAge)
	if err != nil {
		return err
	}

	if maxAge <= 0 {
		return errors.New("MaxAge must be a positive value")
	}

	if c.MaxSize <= 0 {
		return errors.New("MaxSize must be a positive value")
	}

	if c.BufferLength == 0 {
		return errors.New("BufferLength must be a positive value")
	}

	return nil
}
<commit_msg>Add ExcludeEmptyFields to KinesisWriterConfig<commit_after>package scoop_protocol

// KinesisWriterConfig is used to configure a KinesisWriter
import (
	"errors"
	"fmt"
	"time"
)

// AnnotatedKinesisConfig is a Kinesis configuration annotated with meta information.
type AnnotatedKinesisConfig struct {
	AWSAccount       int64
	Team             string
	Version          int
	Contact          string
	Usage            string
	ConsumingLibrary string
	SpadeConfig      KinesisWriterConfig
	LastEditedAt     time.Time
	LastChangedBy    string
	Dropped          bool
	DroppedReason    string
}

// KinesisWriterConfig describes a Kinesis Writer that the processor uses to export data to a Kinesis Stream/Firehose
type KinesisWriterConfig struct {
	StreamName             string
	StreamRole             string
	StreamType             string // StreamType should be either "stream" or "firehose"
	Compress               bool   // true if compress data with flate, false to output json
	FirehoseRedshiftStream bool   // true if JSON destined for Firehose->Redshift streaming
	EventNameTargetField   string // Field name to write the event's name to (useful for uncompressed streams)
	ExcludeEmptyFields     bool   // true if empty fields should be excluded from the JSON
	BufferSize             int
	MaxAttemptsPerRecord   int
	RetryDelay             string

	Events map[string]*struct {
		Filter     string
		FilterFunc func(map[string]string) bool `json:"-"`
		Fields     []string
	}

	Globber GlobberConfig
	Batcher BatcherConfig
}

// Validate returns an error if the Kinesis Writer config is not valid, or nil if it is.
// It also sets the FilterFunc on Events with Filters.
func (c *KinesisWriterConfig) Validate() error {
	if c.StreamType == "" || c.StreamName == "" {
		return fmt.Errorf("Mandatory fields stream type and stream name aren't populated")
	}

	err := c.Globber.Validate()
	if err != nil {
		return fmt.Errorf("globber config invalid: %s", err)
	}

	err = c.Batcher.Validate()
	if err != nil {
		return fmt.Errorf("batcher config invalid: %s", err)
	}

	for _, e := range c.Events {
		if e.Filter != "" {
			e.FilterFunc = filterFuncs[e.Filter]
			if e.FilterFunc == nil {
				return fmt.Errorf("batcher config invalid: %s", err)
			}
		}
	}

	if c.FirehoseRedshiftStream && (c.StreamType != "firehose" || c.Compress) {
		return fmt.Errorf("Redshift streaming only valid with non-compressed firehose")
	}

	_, err = time.ParseDuration(c.RetryDelay)
	return err
}

var filterFuncs = map[string]func(map[string]string) bool{
	"isVod": func(fields map[string]string) bool {
		return fields["vod_id"] != "" && fields["vod_type"] != "clip"
	},
	"isUserIDSet": func(fields map[string]string) bool {
		return fields["user_id"] != ""
	},
	"isChannelIDSet": func(fields map[string]string) bool {
		return fields["channel_id"] != ""
	},
}

// BatcherConfig is used to configure a batcher instance
type BatcherConfig struct {
	// MaxSize is the max combined size of the batch
	MaxSize int

	// MaxEntries is the max number of entries that can be batched together
	// if batches does not have an entry limit, set MaxEntries as -1
	MaxEntries int

	// MaxAge is the max age of the oldest entry in the glob
	MaxAge string

	// BufferLength is the length of the channel where newly
	// submitted entries are stored, decreasing the size of this
	// buffer can cause stalls, and increasing the size can increase
	// shutdown time
	BufferLength int
}

// Validate returns an error if the batcher config is invalid, nil otherwise.
func (c *BatcherConfig) Validate() error {
	maxAge, err := time.ParseDuration(c.MaxAge)
	if err != nil {
		return err
	}

	if maxAge <= 0 {
		return errors.New("MaxAge must be a positive value")
	}

	if c.MaxSize <= 0 {
		return errors.New("MaxSize must be a positive value")
	}

	if c.MaxEntries <= 0 && c.MaxEntries != -1 {
		return errors.New("MaxEntries must be a positive value or -1")
	}

	if c.BufferLength == 0 {
		return errors.New("BufferLength must be a positive value")
	}

	return nil
}

// GlobberConfig is used to configure a globber instance
type GlobberConfig struct {
	// MaxSize is the max size per glob before compression
	MaxSize int

	// MaxAge is the max age of the oldest entry in the glob
	MaxAge string

	// BufferLength is the length of the channel where newly
	// submitted entries are stored, decreasing the size of this
	// buffer can cause stalls, and increasing the size can increase
	// shutdown time
	BufferLength int
}

// Validate returns an error if the config is invalid, nil otherwise.
func (c *GlobberConfig) Validate() error {
	maxAge, err := time.ParseDuration(c.MaxAge)
	if err != nil {
		return err
	}

	if maxAge <= 0 {
		return errors.New("MaxAge must be a positive value")
	}

	if c.MaxSize <= 0 {
		return errors.New("MaxSize must be a positive value")
	}

	if c.BufferLength == 0 {
		return errors.New("BufferLength must be a positive value")
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os/exec"
	"testing"
)

// pulling an image from the central registry should work
func TestPullImageFromCentralRegistry(t *testing.T) {
	pullCmd := exec.Command(dockerBinary, "pull", "busybox:latest")
	out, exitCode, err := runCommandWithOutput(pullCmd)
	errorOut(err, t, fmt.Sprintf("%s %s", out, err))

	if err != nil || exitCode != 0 {
		t.Fatal("pulling the busybox image from the registry has failed")
	}
	logDone("pull - pull busybox")
}

// pulling a non-existing image from the central registry should return a non-zero exit code
func TestPullNonExistingImage(t *testing.T) {
	pullCmd := exec.Command(dockerBinary, "pull", "fooblahblah1234")
	_, exitCode, err := runCommandWithOutput(pullCmd)

	if err == nil || exitCode == 0 {
		t.Fatal("expected non-zero exit status when pulling non-existing image")
	}
	logDone("pull - pull fooblahblah1234 (non-existing image)")
}
<commit_msg>integcli: pull scratch for pull test<commit_after>package main

import (
	"fmt"
	"os/exec"
	"testing"
)

// pulling an image from the central registry should work
func TestPullImageFromCentralRegistry(t *testing.T) {
	pullCmd := exec.Command(dockerBinary, "pull", "scratch")
	out, exitCode, err := runCommandWithOutput(pullCmd)
	errorOut(err, t, fmt.Sprintf("%s %s", out, err))

	if err != nil || exitCode != 0 {
		t.Fatal("pulling the busybox image from the registry has failed")
	}
	logDone("pull - pull busybox")
}

// pulling a non-existing image from the central registry should return a non-zero exit code
func TestPullNonExistingImage(t *testing.T) {
	pullCmd := exec.Command(dockerBinary, "pull", "fooblahblah1234")
	_, exitCode, err := runCommandWithOutput(pullCmd)

	if err == nil || exitCode == 0 {
		t.Fatal("expected non-zero exit status when pulling non-existing image")
	}
	logDone("pull - pull fooblahblah1234 (non-existing image)")
}
<|endoftext|>


<commit_before>package queueinformer

import (
	"fmt"

	opClient "github.com/coreos-inc/operator-client/pkg/client"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
)

// An Operator is a collection of QueueInformers
// OpClient is used to establish the connection to kubernetes
type Operator struct {
	queueInformers []*QueueInformer
	OpClient       opClient.Interface
}

// NewOperator creates a new Operator configured to manage the cluster defined in kubeconfig.
func NewOperator(kubeconfig string, queueInformers ...*QueueInformer) (*Operator, error) {
	opClient := opClient.NewClient(kubeconfig)
	if queueInformers == nil {
		queueInformers = []*QueueInformer{}
	}
	operator := &Operator{
		OpClient:       opClient,
		queueInformers: queueInformers,
	}
	return operator, nil
}

// RegisterQueueInformer adds a QueueInformer to this operator
func (o *Operator) RegisterQueueInformer(queueInformer *QueueInformer) {
	if o.queueInformers == nil {
		o.queueInformers = []*QueueInformer{}
	}
	o.queueInformers = append(o.queueInformers, queueInformer)
}

// Run starts the operator's control loops
func (o *Operator) Run(stopc <-chan struct{}) error {
	for _, queueInformer := range o.queueInformers {
		defer queueInformer.queue.ShutDown()
	}

	errChan := make(chan error)
	go func() {
		v, err := o.OpClient.KubernetesInterface().Discovery().ServerVersion()
		if err != nil {
			errChan <- errors.Wrap(err, "communicating with server failed")
			return
		}
		log.Infof("connection established. cluster-version: %v", v)
		errChan <- nil
	}()

	select {
	case err := <-errChan:
		if err != nil {
			return err
		}
		log.Info("Operator ready")
	case <-stopc:
		return nil
	}

	for _, queueInformer := range o.queueInformers {
		go o.worker(queueInformer)
		go queueInformer.informer.Run(stopc)
	}

	<-stopc
	return nil
}

// worker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the syncHandler is never invoked concurrently with the same key.
func (o *Operator) worker(loop *QueueInformer) {
	for o.processNextWorkItem(loop) {
	}
}

func (o *Operator) processNextWorkItem(loop *QueueInformer) bool {
	queue := loop.queue
	key, quit := queue.Get()
	log.Infof("processing %s", key)

	if quit {
		return false
	}
	defer queue.Done(key)

	if err := o.sync(loop, key.(string)); err != nil {
		utilruntime.HandleError(errors.Wrap(err, fmt.Sprintf("Sync %q failed", key)))
		queue.AddRateLimited(key)
		return true
	}
	queue.Forget(key)
	return true
}

func (o *Operator) sync(loop *QueueInformer, key string) error {
	log.Infof("getting %s from queue", key)
	obj, exists, err := loop.informer.GetIndexer().GetByKey(key)
	if err != nil {
		return err
	}

	if !exists {
		// For now, we ignore the case where an object used to exist but no longer does
		log.Infof("couldn't get %s from queue", key)
		return nil
	}
	return loop.syncHandler(obj)
}
<commit_msg>wait for caches to sync before processing queue<commit_after>package queueinformer

import (
	"fmt"

	opClient "github.com/coreos-inc/operator-client/pkg/client"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/client-go/tools/cache"
)

// An Operator is a collection of QueueInformers
// OpClient is used to establish the connection to kubernetes
type Operator struct {
	queueInformers []*QueueInformer
	OpClient       opClient.Interface
}

// NewOperator creates a new Operator configured to manage the cluster defined in kubeconfig.
func NewOperator(kubeconfig string, queueInformers ...*QueueInformer) (*Operator, error) {
	opClient := opClient.NewClient(kubeconfig)
	if queueInformers == nil {
		queueInformers = []*QueueInformer{}
	}
	operator := &Operator{
		OpClient:       opClient,
		queueInformers: queueInformers,
	}
	return operator, nil
}

// RegisterQueueInformer adds a QueueInformer to this operator
func (o *Operator) RegisterQueueInformer(queueInformer *QueueInformer) {
	if o.queueInformers == nil {
		o.queueInformers = []*QueueInformer{}
	}
	o.queueInformers = append(o.queueInformers, queueInformer)
}

// Run starts the operator's control loops
func (o *Operator) Run(stopc <-chan struct{}) error {
	for _, queueInformer := range o.queueInformers {
		defer queueInformer.queue.ShutDown()
	}

	errChan := make(chan error)
	go func() {
		v, err := o.OpClient.KubernetesInterface().Discovery().ServerVersion()
		if err != nil {
			errChan <- errors.Wrap(err, "communicating with server failed")
			return
		}
		log.Infof("connection established. cluster-version: %v", v)
		errChan <- nil
	}()

	var hasSyncedCheckFns []cache.InformerSynced
	for _, queueInformer := range o.queueInformers {
		hasSyncedCheckFns = append(hasSyncedCheckFns, queueInformer.informer.HasSynced)
	}

	select {
	case err := <-errChan:
		if err != nil {
			return err
		}
		log.Info("Operator ready")
	case <-stopc:
		return nil
	}

	log.Info("starting informers...")
	for _, queueInformer := range o.queueInformers {
		go queueInformer.informer.Run(stopc)
	}

	log.Info("waiting for caches to sync...")
	if ok := cache.WaitForCacheSync(stopc, hasSyncedCheckFns...); !ok {
		return fmt.Errorf("failed to wait for caches to sync")
	}

	log.Info("starting workers...")
	for _, queueInformer := range o.queueInformers {
		go o.worker(queueInformer)
	}
	<-stopc
	return nil
}

// worker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the syncHandler is never invoked concurrently with the same key.
func (o *Operator) worker(loop *QueueInformer) {
	for o.processNextWorkItem(loop) {
	}
}

func (o *Operator) processNextWorkItem(loop *QueueInformer) bool {
	queue := loop.queue
	key, quit := queue.Get()
	log.Infof("processing %s", key)

	if quit {
		return false
	}
	defer queue.Done(key)

	if err := o.sync(loop, key.(string)); err != nil {
		utilruntime.HandleError(errors.Wrap(err, fmt.Sprintf("Sync %q failed", key)))
		queue.AddRateLimited(key)
		return true
	}
	queue.Forget(key)
	return true
}

func (o *Operator) sync(loop *QueueInformer, key string) error {
	log.Infof("getting %s from queue", key)
	obj, exists, err := loop.informer.GetIndexer().GetByKey(key)
	if err != nil {
		return err
	}

	if !exists {
		// For now, we ignore the case where an object used to exist but no longer does
		log.Infof("couldn't get %s from queue", key)
		return nil
	}
	return loop.syncHandler(obj)
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package regtest

import (
	"context"
	"path"
	"testing"

	"golang.org/x/tools/internal/lsp/fake"
)

const internalDefinition = `
-- go.mod --
module mod

go 1.12
-- main.go --
package main

import "fmt"

func main() {
	fmt.Println(message)
}
-- const.go --
package main

const message = "Hello World."
`

func TestGoToInternalDefinition(t *testing.T) {
	t.Parallel()
	runner.Run(t, internalDefinition, func(ctx context.Context, t *testing.T, env *Env) {
		env.OpenFile("main.go")
		name, pos := env.GoToDefinition("main.go", fake.Pos{Line: 5, Column: 13})
		if want := "const.go"; name != want {
			t.Errorf("GoToDefinition: got file %q, want %q", name, want)
		}
		if want := (fake.Pos{Line: 2, Column: 6}); pos != want {
			t.Errorf("GoToDefinition: got position %v, want %v", pos, want)
		}
	})
}

const stdlibDefinition = `
-- go.mod --
module mod

go 1.12
-- main.go --
package main

import (
	"fmt"
	"time"
)

func main() {
	fmt.Println(time.Now())
}`

func TestGoToStdlibDefinition(t *testing.T) {
	t.Parallel()
	runner.Run(t, stdlibDefinition, func(ctx context.Context, t *testing.T, env *Env) {
		env.OpenFile("main.go")
		name, pos := env.GoToDefinition("main.go", fake.Pos{Line: 8, Column: 19})
		if got, want := path.Base(name), "time.go"; got != want {
			t.Errorf("GoToDefinition: got file %q, want %q", name, want)
		}

		// Test that we can jump to definition from outside our workspace.
		// See golang.org/issues/37045.
		newName, newPos := env.GoToDefinition(name, pos)
		if newName != name {
			t.Errorf("GoToDefinition is not idempotent: got %q, want %q", newName, name)
		}
		if newPos != pos {
			t.Errorf("GoToDefinition is not idempotent: got %v, want %v", newPos, pos)
		}
	})
}
<commit_msg>internal/lsp/regtest: skip flaky TestGoToStdlibDefinition<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package regtest

import (
	"context"
	"path"
	"testing"

	"golang.org/x/tools/internal/lsp/fake"
)

const internalDefinition = `
-- go.mod --
module mod

go 1.12
-- main.go --
package main

import "fmt"

func main() {
	fmt.Println(message)
}
-- const.go --
package main

const message = "Hello World."
`

func TestGoToInternalDefinition(t *testing.T) {
	t.Parallel()
	runner.Run(t, internalDefinition, func(ctx context.Context, t *testing.T, env *Env) {
		env.OpenFile("main.go")
		name, pos := env.GoToDefinition("main.go", fake.Pos{Line: 5, Column: 13})
		if want := "const.go"; name != want {
			t.Errorf("GoToDefinition: got file %q, want %q", name, want)
		}
		if want := (fake.Pos{Line: 2, Column: 6}); pos != want {
			t.Errorf("GoToDefinition: got position %v, want %v", pos, want)
		}
	})
}

const stdlibDefinition = `
-- go.mod --
module mod

go 1.12
-- main.go --
package main

import (
	"fmt"
	"time"
)

func main() {
	fmt.Println(time.Now())
}`

func TestGoToStdlibDefinition(t *testing.T) {
	t.Skip("skipping due to golang.org/issues/37318")
	t.Parallel()
	runner.Run(t, stdlibDefinition, func(ctx context.Context, t *testing.T, env *Env) {
		env.OpenFile("main.go")
		name, pos := env.GoToDefinition("main.go", fake.Pos{Line: 8, Column: 19})
		if got, want := path.Base(name), "time.go"; got != want {
			t.Errorf("GoToDefinition: got file %q, want %q", name, want)
		}

		// Test that we can jump to definition from outside our workspace.
		// See golang.org/issues/37045.
		newName, newPos := env.GoToDefinition(name, pos)
		if newName != name {
			t.Errorf("GoToDefinition is not idempotent: got %q, want %q", newName, name)
		}
		if newPos != pos {
			t.Errorf("GoToDefinition is not idempotent: got %v, want %v", newPos, pos)
		}
	})
}
<|endoftext|>


<commit_before>// Copyright 2017 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package segment

import (
	"bytes"
	"fmt"
	"regexp"
	"regexp/syntax"
	"strings"

	"github.com/issue9/mux/params"
)

// 将路由语法转换成正则表达式语法，比如：
//  {id:\\d+}/author => (?P<id>\\d+)
var repl = strings.NewReplacer(string(nameStart), "(?P<",
	string(regexpSeparator), ">",
	string(nameEnd), ")")

type reg struct {
	name       string
	value      string
	endpoint   bool
	expr       *regexp.Regexp
	syntaxExpr *syntax.Regexp
}

func newReg(str string) (Segment, error) {
	index := strings.IndexByte(str, regexpSeparator)

	r := repl.Replace(str)
	expr, err := regexp.Compile(r)
	if err != nil {
		return nil, err
	}

	syntaxExpr, err := syntax.Parse(r, syntax.Perl)
	if err != nil {
		return nil, err
	}

	return &reg{
		value:      str,
		name:       str[1:index],
		expr:       expr,
		syntaxExpr: syntaxExpr,
		endpoint:   IsEndpoint(str),
	}, nil
}

func (r *reg) Type() Type {
	return TypeRegexp
}

func (r *reg) Value() string {
	return r.value
}

func (r *reg) Endpoint() bool {
	return r.endpoint
}

func (r *reg) Match(path string, params params.Params) (bool, string) {
	locs := r.expr.FindStringSubmatchIndex(path)
	if locs == nil || locs[0] != 0 { // 不匹配
		return false, path
	}

	params[r.name] = path[:locs[3]]
	return true, path[locs[1]:]
}

func (r *reg) DeleteParams(params params.Params) {
	delete(params, r.name)
}

func (r *reg) URL(buf *bytes.Buffer, params map[string]string) error {
	param, found := params[r.name]
	if !found {
		return fmt.Errorf("缺少参数 %s", r.name)
	}

	index := strings.IndexByte(r.value, nameEnd)
	url := strings.Replace(r.value, r.value[:index+1], param, 1)

	_, err := buf.WriteString(url)
	return err
}
<commit_msg>[internal/tree/segment] 去掉不再使用的变量<commit_after>// Copyright 2017 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package segment

import (
	"bytes"
	"fmt"
	"regexp"
	"strings"

	"github.com/issue9/mux/params"
)

// 将路由语法转换成正则表达式语法，比如：
//  {id:\\d+}/author => (?P<id>\\d+)
var repl = strings.NewReplacer(string(nameStart), "(?P<",
	string(regexpSeparator), ">",
	string(nameEnd), ")")

type reg struct {
	name     string
	value    string
	endpoint bool
	expr     *regexp.Regexp
}

func newReg(str string) (Segment, error) {
	index := strings.IndexByte(str, regexpSeparator)

	r := repl.Replace(str)
	expr, err := regexp.Compile(r)
	if err != nil {
		return nil, err
	}

	return &reg{
		value:    str,
		name:     str[1:index],
		expr:     expr,
		endpoint: IsEndpoint(str),
	}, nil
}

func (r *reg) Type() Type {
	return TypeRegexp
}

func (r *reg) Value() string {
	return r.value
}

func (r *reg) Endpoint() bool {
	return r.endpoint
}

func (r *reg) Match(path string, params params.Params) (bool, string) {
	locs := r.expr.FindStringSubmatchIndex(path)
	if locs == nil || locs[0] != 0 { // 不匹配
		return false, path
	}

	params[r.name] = path[:locs[3]]
	return true, path[locs[1]:]
}

func (r *reg) DeleteParams(params params.Params) {
	delete(params, r.name)
}

func (r *reg) URL(buf *bytes.Buffer, params map[string]string) error {
	param, found := params[r.name]
	if !found {
		return fmt.Errorf("缺少参数 %s", r.name)
	}

	index := strings.IndexByte(r.value, nameEnd)
	url := strings.Replace(r.value, r.value[:index+1], param, 1)

	_, err := buf.WriteString(url)
	return err
}
<|endoftext|>


<commit_before>// +build !windows

package termstatus

import (
	"io"

	"golang.org/x/sys/unix"

	isatty "github.com/mattn/go-isatty"
)

// clearCurrentLine removes all characters from the current line and resets the
// cursor position to the first column.
func clearCurrentLine(wr io.Writer, fd uintptr) func(io.Writer, uintptr) {
	return posixClearCurrentLine
}

// moveCursorUp moves the cursor to the line n lines above the current one.
func moveCursorUp(wr io.Writer, fd uintptr) func(io.Writer, uintptr, int) {
	return posixMoveCursorUp
}

// canUpdateStatus returns true if status lines can be printed, the process
// output is not redirected to a file or pipe.
func canUpdateStatus(fd uintptr) bool {
	return isatty.IsTerminal(fd)
}

// getTermSize returns the dimensions of the given terminal.
// the code is taken from "golang.org/x/crypto/ssh/terminal"
func getTermSize(fd uintptr) (width, height int, err error) {
	ws, err := unix.IoctlGetWinsize(int(fd), unix.TIOCGWINSZ)
	if err != nil {
		return -1, -1, err
	}
	return int(ws.Col), int(ws.Row), nil
}
<commit_msg>termstatus: detect and respect dumb terminals on Unix<commit_after>// +build !windows

package termstatus

import (
	"io"
	"os"

	"golang.org/x/sys/unix"

	isatty "github.com/mattn/go-isatty"
)

// clearCurrentLine removes all characters from the current line and resets the
// cursor position to the first column.
func clearCurrentLine(wr io.Writer, fd uintptr) func(io.Writer, uintptr) {
	return posixClearCurrentLine
}

// moveCursorUp moves the cursor to the line n lines above the current one.
func moveCursorUp(wr io.Writer, fd uintptr) func(io.Writer, uintptr, int) {
	return posixMoveCursorUp
}

// canUpdateStatus returns true if status lines can be printed, the process
// output is not redirected to a file or pipe.
func canUpdateStatus(fd uintptr) bool {
	if !isatty.IsTerminal(fd) {
		return false
	}
	term := os.Getenv("TERM")
	if term == "" {
		return false
	}
	// TODO actually read termcap db and detect if terminal supports what we need
	return term != "dumb"
}

// getTermSize returns the dimensions of the given terminal.
// the code is taken from "golang.org/x/crypto/ssh/terminal"
func getTermSize(fd uintptr) (width, height int, err error) {
	ws, err := unix.IoctlGetWinsize(int(fd), unix.TIOCGWINSZ)
	if err != nil {
		return -1, -1, err
	}
	return int(ws.Col), int(ws.Row), nil
}
<|endoftext|>


<commit_before>// Copyright 2016 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vhost

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"

	frpNet "github.com/fatedier/frp/utils/net"
	"github.com/fatedier/frp/utils/pool"
)

type HttpMuxer struct {
	*VhostMuxer
}

func GetHttpRequestInfo(c frpNet.Conn) (_ frpNet.Conn, _ map[string]string, err error) {
	reqInfoMap := make(map[string]string, 0)
	sc, rd := frpNet.NewShareConn(c)

	request, err := http.ReadRequest(bufio.NewReader(rd))
	if err != nil {
		return sc, reqInfoMap, err
	}
	// hostName
	tmpArr := strings.Split(request.Host, ":")
	reqInfoMap["Host"] = tmpArr[0]
	reqInfoMap["Path"] = request.URL.Path
	reqInfoMap["Scheme"] = request.URL.Scheme

	// Authorization
	authStr := request.Header.Get("Authorization")
	if authStr != "" {
		reqInfoMap["Authorization"] = authStr
	}
	request.Body.Close()
	return sc, reqInfoMap, nil
}

func NewHttpMuxer(listener frpNet.Listener, timeout time.Duration) (*HttpMuxer, error) {
	mux, err := NewVhostMuxer(listener, GetHttpRequestInfo, HttpAuthFunc, ModifyHttpRequest, timeout)
	return &HttpMuxer{mux}, err
}

func ModifyHttpRequest(c frpNet.Conn, rewriteHost string) (_ frpNet.Conn, err error) {
	sc, rd := frpNet.NewShareConn(c)
	var buff []byte
	remoteIP := strings.Split(c.RemoteAddr().String(), ":")[0]
	if buff, err = hostNameRewrite(rd, rewriteHost, remoteIP); err != nil {
		return sc, err
	}
	err = sc.WriteBuff(buff)
	return sc, err
}

func hostNameRewrite(request io.Reader, rewriteHost string, remoteIP string) (_ []byte, err error) {
	buf := pool.GetBuf(1024)
	defer pool.PutBuf(buf)

	request.Read(buf)
	retBuffer, err := parseRequest(buf, rewriteHost, remoteIP)
	return retBuffer, err
}

func parseRequest(org []byte, rewriteHost string, remoteIP string) (ret []byte, err error) {
	tp := bytes.NewBuffer(org)
	// First line: GET /index.html HTTP/1.0
	var b []byte
	if b, err = tp.ReadBytes('\n'); err != nil {
		return nil, err
	}
	req := new(http.Request)
	// we invoked ReadRequest in GetHttpHostname before, so we ignore error
	req.Method, req.RequestURI, req.Proto, _ = parseRequestLine(string(b))
	rawurl := req.RequestURI
	// CONNECT www.google.com:443 HTTP/1.1
	justAuthority := req.Method == "CONNECT" && !strings.HasPrefix(rawurl, "/")
	if justAuthority {
		rawurl = "http://" + rawurl
	}
	req.URL, _ = url.ParseRequestURI(rawurl)
	if justAuthority {
		// Strip the bogus "http://" back off.
		req.URL.Scheme = ""
	}

	//  RFC2616: first case
	//  GET /index.html HTTP/1.1
	//  Host: www.google.com
	if req.URL.Host == "" {
		var changedBuf []byte
		if rewriteHost != "" {
			changedBuf, err = changeHostName(tp, rewriteHost)
		}
		buf := new(bytes.Buffer)
		buf.Write(b)
		buf.WriteString(fmt.Sprintf("X-Forwarded-For: %s\n", remoteIP))
		buf.WriteString(fmt.Sprintf("X-Real-IP: %s\n", remoteIP))
		if len(changedBuf) == 0 {
			tp.WriteTo(buf)
		} else {
			buf.Write(changedBuf)
		}
		return buf.Bytes(), err
	}

	// RFC2616: second case
	// GET http://www.google.com/index.html HTTP/1.1
	// Host: doesntmatter
	// In this case, any Host line is ignored.
	if rewriteHost != "" {
		hostPort := strings.Split(req.URL.Host, ":")
		if len(hostPort) == 1 {
			req.URL.Host = rewriteHost
		} else if len(hostPort) == 2 {
			req.URL.Host = fmt.Sprintf("%s:%s", rewriteHost, hostPort[1])
		}
	}
	firstLine := req.Method + " " + req.URL.String() + " " + req.Proto
	buf := new(bytes.Buffer)
	buf.WriteString(firstLine)
	buf.WriteString(fmt.Sprintf("X-Forwarded-For: %s\n", remoteIP))
	buf.WriteString(fmt.Sprintf("X-Real-IP: %s\n", remoteIP))
	tp.WriteTo(buf)
	return buf.Bytes(), err
}

// parseRequestLine parses "GET /foo HTTP/1.1" into its three parts.
func parseRequestLine(line string) (method, requestURI, proto string, ok bool) {
	s1 := strings.Index(line, " ")
	s2 := strings.Index(line[s1+1:], " ")
	if s1 < 0 || s2 < 0 {
		return
	}
	s2 += s1 + 1
	return line[:s1], line[s1+1 : s2], line[s2+1:], true
}

func changeHostName(buff *bytes.Buffer, rewriteHost string) (_ []byte, err error) {
	retBuf := new(bytes.Buffer)

	peek := buff.Bytes()
	for len(peek) > 0 {
		i := bytes.IndexByte(peek, '\n')
		if i < 3 {
			// Not present (-1) or found within the next few bytes,
			// implying we're at the end ("\r\n\r\n" or "\n\n")
			return nil, err
		}
		kv := peek[:i]
		j := bytes.IndexByte(kv, ':')
		if j < 0 {
			return nil, fmt.Errorf("malformed MIME header line: " + string(kv))
		}
		if strings.Contains(strings.ToLower(string(kv[:j])), "host") {
			var hostHeader string
			portPos := bytes.IndexByte(kv[j+1:], ':')
			if portPos == -1 {
				hostHeader = fmt.Sprintf("Host: %s\n", rewriteHost)
			} else {
				hostHeader = fmt.Sprintf("Host: %s:%s\n", rewriteHost, kv[j+portPos+2:])
			}
			retBuf.WriteString(hostHeader)
			peek = peek[i+1:]
			break
		} else {
			retBuf.Write(peek[:i])
			retBuf.WriteByte('\n')
		}

		peek = peek[i+1:]
	}
	retBuf.Write(peek)
	return retBuf.Bytes(), err
}

func HttpAuthFunc(c frpNet.Conn, userName, passWord, authorization string) (bAccess bool, err error) {
	s := strings.SplitN(authorization, " ", 2)
	if len(s) != 2 {
		res := noAuthResponse()
		res.Write(c)
		return
	}
	b, err := base64.StdEncoding.DecodeString(s[1])
	if err != nil {
		return
	}
	pair := strings.SplitN(string(b), ":", 2)
	if len(pair) != 2 {
		return
	}
	if pair[0] != userName || pair[1] != passWord {
		return
	}
	return true, nil
}

func noAuthResponse() *http.Response {
	header := make(map[string][]string)
	header["WWW-Authenticate"] = []string{`Basic realm="Restricted"`}
	res := &http.Response{
		Status:     "401 Not authorized",
		StatusCode: 401,
		Proto:      "HTTP/1.1",
		ProtoMajor: 1,
		ProtoMinor: 1,
		Header:     header,
	}
	return res
}
<commit_msg>vhost: a bug fix of reading request<commit_after>// Copyright 2016 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vhost

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"

	frpNet "github.com/fatedier/frp/utils/net"
	"github.com/fatedier/frp/utils/pool"
)

type HttpMuxer struct {
	*VhostMuxer
}

func GetHttpRequestInfo(c frpNet.Conn) (_ frpNet.Conn, _ map[string]string, err error) {
	reqInfoMap := make(map[string]string, 0)
	sc, rd := frpNet.NewShareConn(c)

	request, err := http.ReadRequest(bufio.NewReader(rd))
	if err != nil {
		return sc, reqInfoMap, err
	}
	// hostName
	tmpArr := strings.Split(request.Host, ":")
	reqInfoMap["Host"] = tmpArr[0]
	reqInfoMap["Path"] = request.URL.Path
	reqInfoMap["Scheme"] = request.URL.Scheme

	// Authorization
	authStr := request.Header.Get("Authorization")
	if authStr != "" {
		reqInfoMap["Authorization"] = authStr
	}
	request.Body.Close()
	return sc, reqInfoMap, nil
}

func NewHttpMuxer(listener frpNet.Listener, timeout time.Duration) (*HttpMuxer, error) {
	mux, err := NewVhostMuxer(listener, GetHttpRequestInfo, HttpAuthFunc, ModifyHttpRequest, timeout)
	return &HttpMuxer{mux}, err
}

func ModifyHttpRequest(c frpNet.Conn, rewriteHost string) (_ frpNet.Conn, err error) {
	sc, rd := frpNet.NewShareConn(c)
	var buff []byte
	remoteIP := strings.Split(c.RemoteAddr().String(), ":")[0]
	if buff, err = hostNameRewrite(rd, rewriteHost, remoteIP); err != nil {
		return sc, err
	}
	err = sc.WriteBuff(buff)
	return sc, err
}

func hostNameRewrite(request io.Reader, rewriteHost string, remoteIP string) (_ []byte, err error) {
	buf := pool.GetBuf(1024)
	defer pool.PutBuf(buf)

	var n int
	n, err = request.Read(buf)
	if err != nil {
		return
	}
	retBuffer, err := parseRequest(buf[:n], rewriteHost, remoteIP)
	return retBuffer, err
}

func parseRequest(org []byte, rewriteHost string, remoteIP string) (ret []byte, err error) {
	tp := bytes.NewBuffer(org)
	// First line: GET /index.html HTTP/1.0
	var b []byte
	if b, err = tp.ReadBytes('\n'); err != nil {
		return nil, err
	}
	req := new(http.Request)
	// we invoked ReadRequest in GetHttpHostname before, so we ignore error
	req.Method, req.RequestURI, req.Proto, _ = parseRequestLine(string(b))
	rawurl := req.RequestURI
	// CONNECT www.google.com:443 HTTP/1.1
	justAuthority := req.Method == "CONNECT" && !strings.HasPrefix(rawurl, "/")
	if justAuthority {
		rawurl = "http://" + rawurl
	}
	req.URL, _ = url.ParseRequestURI(rawurl)
	if justAuthority {
		// Strip the bogus "http://" back off.
		req.URL.Scheme = ""
	}

	//  RFC2616: first case
	//  GET /index.html HTTP/1.1
	//  Host: www.google.com
	if req.URL.Host == "" {
		var changedBuf []byte
		if rewriteHost != "" {
			changedBuf, err = changeHostName(tp, rewriteHost)
		}
		buf := new(bytes.Buffer)
		buf.Write(b)
		buf.WriteString(fmt.Sprintf("X-Forwarded-For: %s\r\n", remoteIP))
		buf.WriteString(fmt.Sprintf("X-Real-IP: %s\r\n", remoteIP))
		if len(changedBuf) == 0 {
			tp.WriteTo(buf)
		} else {
			buf.Write(changedBuf)
		}
		return buf.Bytes(), err
	}

	// RFC2616: second case
	// GET http://www.google.com/index.html HTTP/1.1
	// Host: doesntmatter
	// In this case, any Host line is ignored.
	if rewriteHost != "" {
		hostPort := strings.Split(req.URL.Host, ":")
		if len(hostPort) == 1 {
			req.URL.Host = rewriteHost
		} else if len(hostPort) == 2 {
			req.URL.Host = fmt.Sprintf("%s:%s", rewriteHost, hostPort[1])
		}
	}
	firstLine := req.Method + " " + req.URL.String() + " " + req.Proto
	buf := new(bytes.Buffer)
	buf.WriteString(firstLine)
	buf.WriteString(fmt.Sprintf("X-Forwarded-For: %s\r\n", remoteIP))
	buf.WriteString(fmt.Sprintf("X-Real-IP: %s\r\n", remoteIP))
	tp.WriteTo(buf)
	return buf.Bytes(), err
}

// parseRequestLine parses "GET /foo HTTP/1.1" into its three parts.
func parseRequestLine(line string) (method, requestURI, proto string, ok bool) {
	s1 := strings.Index(line, " ")
	s2 := strings.Index(line[s1+1:], " ")
	if s1 < 0 || s2 < 0 {
		return
	}
	s2 += s1 + 1
	return line[:s1], line[s1+1 : s2], line[s2+1:], true
}

func changeHostName(buff *bytes.Buffer, rewriteHost string) (_ []byte, err error) {
	retBuf := new(bytes.Buffer)

	peek := buff.Bytes()
	for len(peek) > 0 {
		i := bytes.IndexByte(peek, '\n')
		if i < 3 {
			// Not present (-1) or found within the next few bytes,
			// implying we're at the end ("\r\n\r\n" or "\n\n")
			return nil, err
		}
		kv := peek[:i]
		j := bytes.IndexByte(kv, ':')
		if j < 0 {
			return nil, fmt.Errorf("malformed MIME header line: " + string(kv))
		}
		if strings.Contains(strings.ToLower(string(kv[:j])), "host") {
			var hostHeader string
			portPos := bytes.IndexByte(kv[j+1:], ':')
			if portPos == -1 {
				hostHeader = fmt.Sprintf("Host: %s\r\n", rewriteHost)
			} else {
				hostHeader = fmt.Sprintf("Host: %s:%s\r\n", rewriteHost, kv[j+portPos+2:])
			}
			retBuf.WriteString(hostHeader)
			peek = peek[i+1:]
			break
		} else {
			retBuf.Write(peek[:i])
			retBuf.WriteByte('\n')
		}

		peek = peek[i+1:]
	}
	retBuf.Write(peek)
	return retBuf.Bytes(), err
}

func HttpAuthFunc(c frpNet.Conn, userName, passWord, authorization string) (bAccess bool, err error) {
	s := strings.SplitN(authorization, " ", 2)
	if len(s) != 2 {
		res := noAuthResponse()
		res.Write(c)
		return
	}
	b, err := base64.StdEncoding.DecodeString(s[1])
	if err != nil {
		return
	}
	pair := strings.SplitN(string(b), ":", 2)
	if len(pair) != 2 {
		return
	}
	if pair[0] != userName || pair[1] != passWord {
		return
	}
	return true, nil
}

func noAuthResponse() *http.Response {
	header := make(map[string][]string)
	header["WWW-Authenticate"] = []string{`Basic realm="Restricted"`}
	res := &http.Response{
		Status:     "401 Not authorized",
		StatusCode: 401,
		Proto:      "HTTP/1.1",
		ProtoMajor: 1,
		ProtoMinor: 1,
		Header:     header,
	}
	return res
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tarball

import (
	"compress/gzip"
	"io"
	"io/ioutil"
	"os"

	"github.com/google/go-containerregistry/v1"
	"github.com/google/go-containerregistry/v1/v1util"
)

type layer struct {
	digest     v1.Hash
	diffID     v1.Hash
	size       int64
	opener     Opener
	compressed bool
}

func (l *layer) Digest() (v1.Hash, error) {
	return l.digest, nil
}

func (l *layer) DiffID() (v1.Hash, error) {
	return l.diffID, nil
}

func (l *layer) Compressed() (io.ReadCloser, error) {
	rc, err := l.opener()
	if err == nil && !l.compressed {
		return v1util.GzipReadCloser(rc)
	}

	return rc, err
}

func (l *layer) Uncompressed() (io.ReadCloser, error) {
	rc, err := l.opener()
	if err == nil && l.compressed {
		return v1util.GunzipReadCloser(rc)
	}

	return rc, err
}

func (l *layer) Size() (int64, error) {
	return l.size, nil
}

// LayerFromFile returns a v1.Layer given a tarball
func LayerFromFile(path string) (v1.Layer, error) {
	opener := func() (io.ReadCloser, error) {
		return os.Open(path)
	}
	return LayerFromOpener(opener)
}

// LayerFromOpener returns a v1.Layer given an Opener function
func LayerFromOpener(opener Opener) (v1.Layer, error) {
	rc, err := opener()
	if err != nil {
		return nil, err
	}
	defer rc.Close()

	compressed, err := v1util.IsGzipped(rc)
	if err != nil {
		return nil, err
	}

	var digest v1.Hash
	var size int64
	if digest, size, err = computeDigest(opener, compressed); err != nil {
		return nil, err
	}

	diffID, err := computeDiffID(opener, compressed)
	if err != nil {
		return nil, err
	}

	return &layer{
		digest:     digest,
		diffID:     diffID,
		size:       size,
		compressed: compressed,
		opener:     opener,
	}, nil
}

func computeDigest(opener Opener, compressed bool) (v1.Hash, int64, error) {
	rc, err := opener()
	if err != nil {
		return v1.Hash{}, 0, err
	}
	defer rc.Close()

	if compressed {
		return v1.SHA256(rc)
	}

	reader, err := v1util.GzipReadCloser(ioutil.NopCloser(rc))
	if err != nil {
		return v1.Hash{}, 0, err
	}

	return v1.SHA256(reader)
}

func computeDiffID(opener Opener, compressed bool) (v1.Hash, error) {
	rc, err := opener()
	if err != nil {
		return v1.Hash{}, err
	}
	defer rc.Close()

	if !compressed {
		digest, _, err := v1.SHA256(rc)
		return digest, err
	}

	reader, err := gzip.NewReader(rc)
	if err != nil {
		return v1.Hash{}, err
	}

	diffID, _, err := v1.SHA256(reader)
	return diffID, err
}
<commit_msg>gofmt v1/tarball/layer.go (#107)<commit_after>// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tarball

import (
	"compress/gzip"
	"io"
	"io/ioutil"
	"os"

	"github.com/google/go-containerregistry/v1"
	"github.com/google/go-containerregistry/v1/v1util"
)

type layer struct {
	digest     v1.Hash
	diffID     v1.Hash
	size       int64
	opener     Opener
	compressed bool
}

func (l *layer) Digest() (v1.Hash, error) {
	return l.digest, nil
}

func (l *layer) DiffID() (v1.Hash, error) {
	return l.diffID, nil
}

func (l *layer) Compressed() (io.ReadCloser, error) {
	rc, err := l.opener()
	if err == nil && !l.compressed {
		return v1util.GzipReadCloser(rc)
	}

	return rc, err
}

func (l *layer) Uncompressed() (io.ReadCloser, error) {
	rc, err := l.opener()
	if err == nil && l.compressed {
		return v1util.GunzipReadCloser(rc)
	}

	return rc, err
}

func (l *layer) Size() (int64, error) {
	return l.size, nil
}

// LayerFromFile returns a v1.Layer given a tarball
func LayerFromFile(path string) (v1.Layer, error) {
	opener := func() (io.ReadCloser, error) {
		return os.Open(path)
	}
	return LayerFromOpener(opener)
}

// LayerFromOpener returns a v1.Layer given an Opener function
func LayerFromOpener(opener Opener) (v1.Layer, error) {
	rc, err := opener()
	if err != nil {
		return nil, err
	}
	defer rc.Close()

	compressed, err := v1util.IsGzipped(rc)
	if err != nil {
		return nil, err
	}

	var digest v1.Hash
	var size int64
	if digest, size, err = computeDigest(opener, compressed); err != nil {
		return nil, err
	}

	diffID, err := computeDiffID(opener, compressed)
	if err != nil {
		return nil, err
	}

	return &layer{
		digest:     digest,
		diffID:     diffID,
		size:       size,
		compressed: compressed,
		opener:     opener,
	}, nil
}

func computeDigest(opener Opener, compressed bool) (v1.Hash, int64, error) {
	rc, err := opener()
	if err != nil {
		return v1.Hash{}, 0, err
	}
	defer rc.Close()

	if compressed {
		return v1.SHA256(rc)
	}

	reader, err := v1util.GzipReadCloser(ioutil.NopCloser(rc))
	if err != nil {
		return v1.Hash{}, 0, err
	}

	return v1.SHA256(reader)
}

func computeDiffID(opener Opener, compressed bool) (v1.Hash, error) {
	rc, err := opener()
	if err != nil {
		return v1.Hash{}, err
	}
	defer rc.Close()

	if !compressed {
		digest, _, err := v1.SHA256(rc)
		return digest, err
	}

	reader, err := gzip.NewReader(rc)
	if err != nil {
		return v1.Hash{}, err
	}

	diffID, _, err := v1.SHA256(reader)
	return diffID, err
}
<|endoftext|>


<commit_before>package sfml

//#include <SFML/Graphics.h>
import "C"

type Shader struct {
	data *C.sfShader
}

func CreateShaderFromFile(vertexShaderFilename, fragmentShaderFilename string) *Shader {
	return nil
}
<commit_msg>Shader implemented<commit_after>package sfml

//#include <SFML/Graphics.h>
//#include <stdlib.h>
import "C"

import (
	"runtime"
	"unsafe"
)

type Shader struct {
	data *C.sfShader
}

func destroyShader(s *Shader) {
	C.sfShader_destroy(s.data)
}

func CreateShader(vertexShaderFilename, fragmentShaderFilename string) *Shader {
	vshader := C.CString(vertexShaderFilename)
	defer C.free(unsafe.Pointer(vshader))
	fshader := C.CString(fragmentShaderFilename)
	defer C.free(unsafe.Pointer(fshader))
	s := C.sfShader_createFromFile(vshader, fshader)
	if s == nil {
		return nil
	}
	obj := &Shader{s}
	runtime.SetFinalizer(obj, destroyShader)
	return obj
}

func CreateShaderFromMemory(vertexShader, fragmentShader string) *Shader {
	vshader := C.CString(vertexShader)
	defer C.free(unsafe.Pointer(vshader))
	fshader := C.CString(fragmentShader)
	defer C.free(unsafe.Pointer(fshader))
	s := C.sfShader_createFromMemory(vshader, fshader)
	if s == nil {
		return nil
	}
	obj := &Shader{s}
	runtime.SetFinalizer(obj, destroyShader)
	return obj
}

func (s *Shader) SetFloatParameter(name string, xyzw ...float32) {
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	switch len(xyzw) {
	case 1:
		C.sfShader_setFloatParameter(s.data, cname, C.float(xyzw[0]))
	case 2:
		C.sfShader_setFloat2Parameter(s.data, cname, C.float(xyzw[0]), C.float(xyzw[1]))
	case 3:
		C.sfShader_setFloat3Parameter(s.data, cname, C.float(xyzw[0]), C.float(xyzw[1]), C.float(xyzw[2]))
	case 4:
		C.sfShader_setFloat4Parameter(s.data, cname, C.float(xyzw[0]), C.float(xyzw[1]), C.float(xyzw[2]), C.float(xyzw[3]))
	}
}

func (s *Shader) SetVector2Parameter(name string, vector Vector2f) {
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	C.sfShader_setVector2Parameter(s.data, cname, cVector2f(&vector))
}

func (s *Shader) SetVector3Parameter(name string, vector Vector3f) {
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	C.sfShader_setVector3Parameter(s.data, cname, cVector3f(&vector))
}

func (s *Shader) SetColorParameter(name string, color Color) {
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	C.sfShader_setColorParameter(s.data, cname, cColor(&color))
}

func (s *Shader) SetTransformParameter(name string, transform Transform) {
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	C.sfShader_setTransformParameter(s.data, cname, cTransform(&transform))
}

func (s *Shader) SetTextureParameter(name string, texture *Texture) {
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	C.sfShader_setTextureParameter(s.data, cname, texture.data)
}

func (s *Shader) SetCurrentTextureParameter(name string) {
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))
	C.sfShader_setCurrentTextureParameter(s.data, cname)
}

func BindShader(shader *Shader) {
	C.sfShader_bind(shader.data)
}

func ShaderIsAvailable() bool {
	return goBool(C.sfShader_isAvailable())
}<|endoftext|>


<commit_before>package repository

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"io"
	"io/ioutil"
	"os"
	"path"

	. "gopkg.in/check.v1"
)

type FileContentStorageTests struct {
	dir     string
	storage *FileContentStorage
	data    []byte
}

var _ = Suite(&FileContentStorageTests{})

func (t *FileContentStorageTests) SetUpTest(c *C) {
	t.dir = c.MkDir()
	t.storage = &FileContentStorage{StoragePath: t.dir}
	t.data = []byte("This is a test blob storage file input.")
}

func (t *FileContentStorageTests) blobID() string {
	blobIDBytes := sha256.New().Sum(t.data)
	return hex.EncodeToString(blobIDBytes[:])
}

func (t *FileContentStorageTests) blobPath() string {
	return path.Join(t.dir, t.blobID())
}

func (t *FileContentStorageTests) testReader() io.Reader {
	return bytes.NewReader(t.data)
}

func (t *FileContentStorageTests) setData() error {
	return t.storage.Set(t.blobID(), t.testReader())
}

func (t *FileContentStorageTests) TestSet(c *C) {
	err := t.setData()
	c.Assert(err, IsNil)
	_, err = os.Stat(t.blobPath())
	c.Assert(err, IsNil)
}

func (t *FileContentStorageTests) TestSetInputData(c *C) {
	t.setData()
	file, _ := os.Open(t.blobPath())
	fileData, _ := ioutil.ReadAll(file)
	c.Assert(fileData[:], DeepEquals, t.data[:])
}

func (t *FileContentStorageTests) TestExistsNegative(c *C) {
	c.Assert(t.storage.Exists(t.blobID()), Equals, false)
}

func (t *FileContentStorageTests) TestExistsPositive(c *C) {
	t.setData()
	c.Assert(t.storage.Exists(t.blobID()), Equals, true)
}

func (t *FileContentStorageTests) TestGet(c *C) {
	t.storage.Set(t.blobID(), t.testReader())
	_, err := t.storage.Get(t.blobID())
	c.Assert(err, IsNil)
}

func (t *FileContentStorageTests) TestGetData(c *C) {
	t.setData()
	file, _ := t.storage.Get(t.blobID())
	fileData, _ := ioutil.ReadAll(file)
	c.Assert(fileData[:], DeepEquals, t.data)
}

func (t *FileContentStorageTests) TestGetError(c *C) {
	_, err := t.storage.Get(t.blobID())
	c.Assert(err, NotNil)
}

func (t *FileContentStorageTests) TestSetError(c *C) {
	os.RemoveAll(t.dir)

	err := t.storage.Set(t.blobID(),
		t.testReader())
	c.Assert(err, NotNil)
}

func (t *FileContentStorageTests) TestDelete(c *C) {
	t.setData()
	err := t.storage.Delete(t.blobID())
	c.Assert(err, IsNil)
	c.Assert(t.storage.Exists(t.blobID()), Equals, false)
}

func (t *FileContentStorage) TestDeleteError(c *C) {
	err := t.storage.Delete(t.blobID())
	c.Assert(err, NotNil)
}
<commit_msg>repository: fixed a error for the file content storage test.<commit_after>package repository

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"io"
	"io/ioutil"
	"os"
	"path"

	. "gopkg.in/check.v1"
)

type FileContentStorageTests struct {
	dir     string
	storage *FileContentStorage
	data    []byte
}

var _ = Suite(&FileContentStorageTests{})

func (t *FileContentStorageTests) SetUpTest(c *C) {
	t.dir = c.MkDir()
	t.storage = &FileContentStorage{StoragePath: t.dir}
	t.data = []byte("This is a test blob storage file input.")
}

func (t *FileContentStorageTests) blobID() string {
	blobIDBytes := sha256.New().Sum(t.data)
	return hex.EncodeToString(blobIDBytes[:])
}

func (t *FileContentStorageTests) blobPath() string {
	return path.Join(t.dir, t.blobID())
}

func (t *FileContentStorageTests) testReader() io.Reader {
	return bytes.NewReader(t.data)
}

func (t *FileContentStorageTests) setData() error {
	return t.storage.Set(t.blobID(), t.testReader())
}

func (t *FileContentStorageTests) TestSet(c *C) {
	err := t.setData()
	c.Assert(err, IsNil)
	_, err = os.Stat(t.blobPath())
	c.Assert(err, IsNil)
}

func (t *FileContentStorageTests) TestSetInputData(c *C) {
	t.setData()
	file, _ := os.Open(t.blobPath())
	fileData, _ := ioutil.ReadAll(file)
	c.Assert(fileData[:], DeepEquals, t.data[:])
}

func (t *FileContentStorageTests) TestExistsNegative(c *C) {
	c.Assert(t.storage.Exists(t.blobID()), Equals, false)
}

func (t *FileContentStorageTests) TestExistsPositive(c *C) {
	t.setData()
	c.Assert(t.storage.Exists(t.blobID()), Equals, true)
}

func (t *FileContentStorageTests) TestGet(c *C) {
	t.storage.Set(t.blobID(), t.testReader())
	_, err := t.storage.Get(t.blobID())
	c.Assert(err, IsNil)
}

func (t *FileContentStorageTests) TestGetData(c *C) {
	t.setData()
	file, _ := t.storage.Get(t.blobID())
	fileData, _ := ioutil.ReadAll(file)
	c.Assert(fileData[:], DeepEquals, t.data)
}

func (t *FileContentStorageTests) TestGetError(c *C) {
	_, err := t.storage.Get(t.blobID())
	c.Assert(err, NotNil)
}

func (t *FileContentStorageTests) TestSetError(c *C) {
	os.RemoveAll(t.dir)

	err := t.storage.Set(t.blobID(),
		t.testReader())
	c.Assert(err, NotNil)
}

func (t *FileContentStorageTests) TestDelete(c *C) {
	t.setData()
	err := t.storage.Delete(t.blobID())
	c.Assert(err, IsNil)
	c.Assert(t.storage.Exists(t.blobID()), Equals, false)
}

func (t *FileContentStorageTests) TestDeleteError(c *C) {
	err := t.storage.Delete(t.blobID())
	c.Assert(err, NotNil)
}
<|endoftext|>


<commit_before>package engine

import (
	"context"
	"errors"
	"sync"
	"time"

	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagerctx"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/builds"
	"github.com/concourse/concourse/atc/creds"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/event"
	"github.com/concourse/concourse/atc/exec"
	"github.com/concourse/concourse/atc/metric"
	"github.com/concourse/concourse/atc/util"
	"github.com/concourse/concourse/tracing"
)

//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate

func NewEngine(
	stepperFactory StepperFactory,
	secrets creds.Secrets,
	varSourcePool creds.VarSourcePool,
) Engine {
	return Engine{
		stepperFactory: stepperFactory,
		release:        make(chan bool),
		trackedStates:  new(sync.Map),
		waitGroup:      new(sync.WaitGroup),

		globalSecrets: secrets,
		varSourcePool: varSourcePool,
	}
}

type Engine struct {
	stepperFactory StepperFactory
	release        chan bool
	trackedStates  *sync.Map
	waitGroup      *sync.WaitGroup

	globalSecrets creds.Secrets
	varSourcePool creds.VarSourcePool
}

func (engine Engine) Drain(ctx context.Context) {
	logger := lagerctx.FromContext(ctx)

	logger.Info("start")
	defer logger.Info("done")

	close(engine.release)

	logger.Info("waiting")

	engine.waitGroup.Wait()
}

func (engine Engine) NewBuild(build db.Build) builds.Runnable {
	return NewBuild(
		build,
		engine.stepperFactory,
		engine.globalSecrets,
		engine.varSourcePool,
		engine.release,
		engine.trackedStates,
		engine.waitGroup,
	)
}

func NewBuild(
	build db.Build,
	builder StepperFactory,
	globalSecrets creds.Secrets,
	varSourcePool creds.VarSourcePool,
	release chan bool,
	trackedStates *sync.Map,
	waitGroup *sync.WaitGroup,
) builds.Runnable {
	return &engineBuild{
		build:   build,
		builder: builder,

		globalSecrets: globalSecrets,
		varSourcePool: varSourcePool,

		release:       release,
		trackedStates: trackedStates,
		waitGroup:     waitGroup,
	}
}

type engineBuild struct {
	build   db.Build
	builder StepperFactory

	globalSecrets creds.Secrets
	varSourcePool creds.VarSourcePool

	release       chan bool
	trackedStates *sync.Map
	waitGroup     *sync.WaitGroup
}

func (b *engineBuild) Run(ctx context.Context) {
	b.waitGroup.Add(1)
	defer b.waitGroup.Done()

	logger := lagerctx.FromContext(ctx).WithData(b.build.LagerData())

	lock, acquired, err := b.build.AcquireTrackingLock(logger, time.Minute)
	if err != nil {
		logger.Error("failed-to-get-lock", err)
		return
	}

	if !acquired {
		logger.Debug("build-already-tracked")
		return
	}

	defer lock.Release()

	found, err := b.build.Reload()
	if err != nil {
		logger.Error("failed-to-load-build-from-db", err)
		return
	}

	if !found {
		logger.Info("build-not-found")
		return
	}

	if !b.build.IsRunning() {
		logger.Info("build-already-finished")
		return
	}

	ctx, span := tracing.StartSpanFollowing(ctx, b.build, "build", b.build.TracingAttrs())
	defer span.End()

	stepper, err := b.builder.StepperForBuild(b.build)
	if err != nil {
		logger.Error("failed-to-construct-build-stepper", err)

		// Fails the build if BuildStep returned an error because such unrecoverable
		// errors will cause a build to never start to run.
		b.buildStepErrored(logger, err.Error())
		b.finish(logger.Session("finish"), err, false)

		return
	}

	b.trackStarted(logger)
	defer b.trackFinished(logger)

	logger.Info("running")

	state, err := b.runState(logger, stepper)
	if err != nil {
		logger.Error("failed-to-create-run-state", err)

		// Fails the build if fetching the pipeline variables fails, as these errors
		// are unrecoverable - e.g. if pipeline var_sources is wrong
		b.buildStepErrored(logger, err.Error())
		b.finish(logger.Session("finish"), err, false)

		return
	}
	defer b.clearRunState()

	notifier, err := b.build.AbortNotifier()
	if err != nil {
		logger.Error("failed-to-listen-for-aborts", err)
		return
	}

	if notifier != nil {
		defer notifier.Close()

		var cancel context.CancelFunc
		ctx, cancel = context.WithCancel(ctx)

		noleak := make(chan bool)
		defer close(noleak)

		go func() {
			select {
			case <-noleak:
			case <-notifier.Notify():
				logger.Info("aborting")
				cancel()
			}
		}()
	}

	var succeeded bool
	var runErr error

	done := make(chan struct{})
	go func() {
		defer close(done)
		defer func() {
			err := util.DumpPanic(recover(), "running build plan %d", b.build.ID())
			if err != nil {
				logger.Error("panic-in-engine-build-step-run", err)
				runErr = err
			}
		}()
		succeeded, runErr = state.Run(lagerctx.NewContext(ctx, logger), b.build.PrivatePlan())
	}()

	select {
	case <-b.release:
		logger.Info("releasing")

	case <-done:
		if errors.As(runErr, &exec.Retriable{}) {
			return
		}

		// An in-memory build only generates a real build id once start to run,
		// so let's update logger with the latest lager data.
		b.finish(logger.Session("finish").WithData(b.build.LagerData()), runErr, succeeded)
	}
}

func (b *engineBuild) buildStepErrored(logger lager.Logger, message string) {
	err := b.build.SaveEvent(event.Error{
		Message: message,
		Origin: event.Origin{
			ID: event.OriginID(b.build.PrivatePlan().ID),
		},
		Time: time.Now().Unix(),
	})
	if err != nil {
		logger.Error("failed-to-save-error-event", err)
	}
}

func (b *engineBuild) finish(logger lager.Logger, err error, succeeded bool) {
	if errors.Is(err, context.Canceled) {
		b.saveStatus(logger, atc.StatusAborted)
		logger.Info("aborted")

	} else if err != nil {
		b.saveStatus(logger, atc.StatusErrored)
		logger.Info("errored", lager.Data{"error": err.Error()})

	} else if succeeded {
		b.saveStatus(logger, atc.StatusSucceeded)
		logger.Info("succeeded")

	} else {
		b.saveStatus(logger, atc.StatusFailed)
		logger.Info("failed")
	}
}

func (b *engineBuild) saveStatus(logger lager.Logger, status atc.BuildStatus) {
	if err := b.build.Finish(db.BuildStatus(status)); err != nil {
		logger.Error("failed-to-finish-build", err)
	}
}

func (b *engineBuild) trackStarted(logger lager.Logger) {
	if b.build.Name() != db.CheckBuildName {
		metric.BuildStarted{
			Build: b.build,
		}.Emit(logger)
	}
}

func (b *engineBuild) trackFinished(logger lager.Logger) {
	found, err := b.build.Reload()
	if err != nil {
		logger.Error("failed-to-load-build-from-db", err)
		return
	}

	if !found {
		logger.Info("build-removed")
		return
	}

	if !b.build.IsRunning() {
		if b.build.Name() != db.CheckBuildName {
			metric.BuildFinished{
				Build: b.build,
			}.Emit(logger)
		}
	}
}

func (b *engineBuild) runState(logger lager.Logger, stepper exec.Stepper) (exec.RunState, error) {
	id := b.build.RunStateID()
	existingState, ok := b.trackedStates.Load(id)
	if ok {
		return existingState.(exec.RunState), nil
	}
	credVars, err := b.build.Variables(logger, b.globalSecrets, b.varSourcePool)
	if err != nil {
		return nil, err
	}
	state, _ := b.trackedStates.LoadOrStore(id, exec.NewRunState(stepper, credVars, atc.EnableRedactSecrets))
	return state.(exec.RunState), nil
}

func (b *engineBuild) clearRunState() {
	b.trackedStates.Delete(b.build.RunStateID())
}
<commit_msg>check build should not auto retry.<commit_after>package engine

import (
	"context"
	"errors"
	"sync"
	"time"

	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagerctx"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/builds"
	"github.com/concourse/concourse/atc/creds"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/event"
	"github.com/concourse/concourse/atc/exec"
	"github.com/concourse/concourse/atc/metric"
	"github.com/concourse/concourse/atc/util"
	"github.com/concourse/concourse/tracing"
)

//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate

func NewEngine(
	stepperFactory StepperFactory,
	secrets creds.Secrets,
	varSourcePool creds.VarSourcePool,
) Engine {
	return Engine{
		stepperFactory: stepperFactory,
		release:        make(chan bool),
		trackedStates:  new(sync.Map),
		waitGroup:      new(sync.WaitGroup),

		globalSecrets: secrets,
		varSourcePool: varSourcePool,
	}
}

type Engine struct {
	stepperFactory StepperFactory
	release        chan bool
	trackedStates  *sync.Map
	waitGroup      *sync.WaitGroup

	globalSecrets creds.Secrets
	varSourcePool creds.VarSourcePool
}

func (engine Engine) Drain(ctx context.Context) {
	logger := lagerctx.FromContext(ctx)

	logger.Info("start")
	defer logger.Info("done")

	close(engine.release)

	logger.Info("waiting")

	engine.waitGroup.Wait()
}

func (engine Engine) NewBuild(build db.Build) builds.Runnable {
	return NewBuild(
		build,
		engine.stepperFactory,
		engine.globalSecrets,
		engine.varSourcePool,
		engine.release,
		engine.trackedStates,
		engine.waitGroup,
	)
}

func NewBuild(
	build db.Build,
	builder StepperFactory,
	globalSecrets creds.Secrets,
	varSourcePool creds.VarSourcePool,
	release chan bool,
	trackedStates *sync.Map,
	waitGroup *sync.WaitGroup,
) builds.Runnable {
	return &engineBuild{
		build:   build,
		builder: builder,

		globalSecrets: globalSecrets,
		varSourcePool: varSourcePool,

		release:       release,
		trackedStates: trackedStates,
		waitGroup:     waitGroup,
	}
}

type engineBuild struct {
	build   db.Build
	builder StepperFactory

	globalSecrets creds.Secrets
	varSourcePool creds.VarSourcePool

	release       chan bool
	trackedStates *sync.Map
	waitGroup     *sync.WaitGroup
}

func (b *engineBuild) Run(ctx context.Context) {
	b.waitGroup.Add(1)
	defer b.waitGroup.Done()

	logger := lagerctx.FromContext(ctx).WithData(b.build.LagerData())

	lock, acquired, err := b.build.AcquireTrackingLock(logger, time.Minute)
	if err != nil {
		logger.Error("failed-to-get-lock", err)
		return
	}

	if !acquired {
		logger.Debug("build-already-tracked")
		return
	}

	defer lock.Release()

	found, err := b.build.Reload()
	if err != nil {
		logger.Error("failed-to-load-build-from-db", err)
		return
	}

	if !found {
		logger.Info("build-not-found")
		return
	}

	if !b.build.IsRunning() {
		logger.Info("build-already-finished")
		return
	}

	ctx, span := tracing.StartSpanFollowing(ctx, b.build, "build", b.build.TracingAttrs())
	defer span.End()

	stepper, err := b.builder.StepperForBuild(b.build)
	if err != nil {
		logger.Error("failed-to-construct-build-stepper", err)

		// Fails the build if BuildStep returned an error because such unrecoverable
		// errors will cause a build to never start to run.
		b.buildStepErrored(logger, err.Error())
		b.finish(logger.Session("finish"), err, false)

		return
	}

	b.trackStarted(logger)
	defer b.trackFinished(logger)

	logger.Info("running")

	state, err := b.runState(logger, stepper)
	if err != nil {
		logger.Error("failed-to-create-run-state", err)

		// Fails the build if fetching the pipeline variables fails, as these errors
		// are unrecoverable - e.g. if pipeline var_sources is wrong
		b.buildStepErrored(logger, err.Error())
		b.finish(logger.Session("finish"), err, false)

		return
	}
	defer b.clearRunState()

	notifier, err := b.build.AbortNotifier()
	if err != nil {
		logger.Error("failed-to-listen-for-aborts", err)
		return
	}

	if notifier != nil {
		defer notifier.Close()

		var cancel context.CancelFunc
		ctx, cancel = context.WithCancel(ctx)

		noleak := make(chan bool)
		defer close(noleak)

		go func() {
			select {
			case <-noleak:
			case <-notifier.Notify():
				logger.Info("aborting")
				cancel()
			}
		}()
	}

	var succeeded bool
	var runErr error

	done := make(chan struct{})
	go func() {
		defer close(done)
		defer func() {
			err := util.DumpPanic(recover(), "running build plan %d", b.build.ID())
			if err != nil {
				logger.Error("panic-in-engine-build-step-run", err)
				runErr = err
			}
		}()
		succeeded, runErr = state.Run(lagerctx.NewContext(ctx, logger), b.build.PrivatePlan())
	}()

	select {
	case <-b.release:
		logger.Info("releasing")

	case <-done:
		// Don't retry check build because if a check build drops into endless retry,
		// there is no way to abort it.
		if b.build.Name() != db.CheckBuildName && errors.As(runErr, &exec.Retriable{}) {
			return
		}

		// An in-memory build only generates a real build id once start to run,
		// so let's update logger with the latest lager data.
		b.finish(logger.Session("finish").WithData(b.build.LagerData()), runErr, succeeded)
	}
}

func (b *engineBuild) buildStepErrored(logger lager.Logger, message string) {
	err := b.build.SaveEvent(event.Error{
		Message: message,
		Origin: event.Origin{
			ID: event.OriginID(b.build.PrivatePlan().ID),
		},
		Time: time.Now().Unix(),
	})
	if err != nil {
		logger.Error("failed-to-save-error-event", err)
	}
}

func (b *engineBuild) finish(logger lager.Logger, err error, succeeded bool) {
	if errors.Is(err, context.Canceled) {
		b.saveStatus(logger, atc.StatusAborted)
		logger.Info("aborted")

	} else if err != nil {
		b.saveStatus(logger, atc.StatusErrored)
		logger.Info("errored", lager.Data{"error": err.Error()})

	} else if succeeded {
		b.saveStatus(logger, atc.StatusSucceeded)
		logger.Info("succeeded")

	} else {
		b.saveStatus(logger, atc.StatusFailed)
		logger.Info("failed")
	}
}

func (b *engineBuild) saveStatus(logger lager.Logger, status atc.BuildStatus) {
	if err := b.build.Finish(db.BuildStatus(status)); err != nil {
		logger.Error("failed-to-finish-build", err)
	}
}

func (b *engineBuild) trackStarted(logger lager.Logger) {
	if b.build.Name() != db.CheckBuildName {
		metric.BuildStarted{
			Build: b.build,
		}.Emit(logger)
	}
}

func (b *engineBuild) trackFinished(logger lager.Logger) {
	found, err := b.build.Reload()
	if err != nil {
		logger.Error("failed-to-load-build-from-db", err)
		return
	}

	if !found {
		logger.Info("build-removed")
		return
	}

	if !b.build.IsRunning() {
		if b.build.Name() != db.CheckBuildName {
			metric.BuildFinished{
				Build: b.build,
			}.Emit(logger)
		}
	}
}

func (b *engineBuild) runState(logger lager.Logger, stepper exec.Stepper) (exec.RunState, error) {
	id := b.build.RunStateID()
	existingState, ok := b.trackedStates.Load(id)
	if ok {
		return existingState.(exec.RunState), nil
	}
	credVars, err := b.build.Variables(logger, b.globalSecrets, b.varSourcePool)
	if err != nil {
		return nil, err
	}
	state, _ := b.trackedStates.LoadOrStore(id, exec.NewRunState(stepper, credVars, atc.EnableRedactSecrets))
	return state.(exec.RunState), nil
}

func (b *engineBuild) clearRunState() {
	b.trackedStates.Delete(b.build.RunStateID())
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package servemux

import (
	"net/http"
	"testing"

	"github.com/google/martian/martiantest"
	"github.com/google/martian/proxyutil"
)

func TestModifyRequest(t *testing.T) {
	mux := http.NewServeMux()
	mux.HandleFunc("example.com/test", nil)

	f := NewFilter(mux)
	tm := martiantest.NewModifier()
	f.RequestWhenTrue(tm)
	fm := martiantest.NewModifier()
	f.RequestWhenFalse(fm)

	req, err := http.NewRequest("GET", "http://example.com/test", nil)
	if err != nil {
		t.Fatalf("http.NewRequest(): got %v, want no error", err)
	}

	if err := f.ModifyRequest(req); err != nil {
		t.Errorf("ModifyRequest(): got %v, want no error", err)
	}

	if got, want := tm.RequestModified(), true; got != want {
		t.Errorf("tm.RequestModified(): got %v, want %v", got, want)
	}

	if got, want := fm.RequestModified(), false; got != want {
		t.Errorf("fm.RequestModified(): got %v, want %v", got, want)
	}

	tm.Reset()
	fm.Reset()

	req, err = http.NewRequest("GET", "http://example.com/nomatch", nil)
	if err != nil {
		t.Fatalf("http.NewRequest(): got %v, want no error", err)
	}

	if err := f.ModifyRequest(req); err != nil {
		t.Errorf("ModifyRequest(): got %v, want no error", err)
	}

	if got, want := tm.RequestModified(), false; got != want {
		t.Errorf("tm.RequestModified(): got %v, want %v", got, want)
	}

	if got, want := fm.RequestModified(), true; got != want {
		t.Errorf("fm.RequestModified(): got %v, want %v", got, want)
	}
}

func TestModifyResponse(t *testing.T) {
	mux := http.NewServeMux()
	mux.HandleFunc("example.com/restest", nil)

	f := NewFilter(mux)
	tm := martiantest.NewModifier()
	f.ResponseWhenTrue(tm)
	fm := martiantest.NewModifier()
	f.ResponseWhenFalse(fm)

	req, err := http.NewRequest("GET", "http://example.com/restest", nil)
	if err != nil {
		t.Fatalf("http.NewRequest(): got %v, want no error", err)
	}
	res := proxyutil.NewResponse(200, nil, req)

	if err := f.ModifyResponse(res); err != nil {
		t.Errorf("ModifyResponse(): got %v, want no error", err)
	}

	if got, want := tm.ResponseModified(), true; got != want {
		t.Errorf("tm.ResponseModified(): got %v, want %v", got, want)
	}

	if got, want := fm.ResponseModified(), false; got != want {
		t.Errorf("fm.ResponseModified(): got %v, want %v", got, want)
	}

	tm.Reset()
	fm.Reset()

	req, err = http.NewRequest("GET", "http://example.com/nomatch", nil)
	if err != nil {
		t.Fatalf("http.NewRequest(): got %v, want no error", err)
	}
	res = proxyutil.NewResponse(200, nil, req)

	if err := f.ModifyResponse(res); err != nil {
		t.Errorf("ModifyResponse(): got %v, want no error", err)
	}

	if tm.ResponseModified() != false {
		t.Errorf("tm.ResponseModified(): got %t, want %t", tm.ResponseModified(), false)
	}

	if got, want := tm.ResponseModified(), false; got != want {
		t.Errorf("tm.ResponseModified(): got %v, want %v", got, want)
	}

	if got, want := fm.ResponseModified(), true; got != want {
		t.Errorf("fm.ResponseModified(): got %v, want %v", got, want)
	}
}
<commit_msg>do not pass in a nil handler to mux.HandleFunc (#230)<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package servemux

import (
	"net/http"
	"testing"

	"github.com/google/martian/martiantest"
	"github.com/google/martian/proxyutil"
)

func TestModifyRequest(t *testing.T) {
	mux := http.NewServeMux()
	mux.HandleFunc("example.com/test", func(rw http.ResponseWriter, req *http.Request) {
		return
	})

	f := NewFilter(mux)
	tm := martiantest.NewModifier()
	f.RequestWhenTrue(tm)
	fm := martiantest.NewModifier()
	f.RequestWhenFalse(fm)

	req, err := http.NewRequest("GET", "http://example.com/test", nil)
	if err != nil {
		t.Fatalf("http.NewRequest(): got %v, want no error", err)
	}

	if err := f.ModifyRequest(req); err != nil {
		t.Errorf("ModifyRequest(): got %v, want no error", err)
	}

	if got, want := tm.RequestModified(), true; got != want {
		t.Errorf("tm.RequestModified(): got %v, want %v", got, want)
	}

	if got, want := fm.RequestModified(), false; got != want {
		t.Errorf("fm.RequestModified(): got %v, want %v", got, want)
	}

	tm.Reset()
	fm.Reset()

	req, err = http.NewRequest("GET", "http://example.com/nomatch", nil)
	if err != nil {
		t.Fatalf("http.NewRequest(): got %v, want no error", err)
	}

	if err := f.ModifyRequest(req); err != nil {
		t.Errorf("ModifyRequest(): got %v, want no error", err)
	}

	if got, want := tm.RequestModified(), false; got != want {
		t.Errorf("tm.RequestModified(): got %v, want %v", got, want)
	}

	if got, want := fm.RequestModified(), true; got != want {
		t.Errorf("fm.RequestModified(): got %v, want %v", got, want)
	}
}

func TestModifyResponse(t *testing.T) {
	mux := http.NewServeMux()
	mux.HandleFunc("example.com/restest", func(rw http.ResponseWriter, req *http.Request) {
		return
	})

	f := NewFilter(mux)
	tm := martiantest.NewModifier()
	f.ResponseWhenTrue(tm)
	fm := martiantest.NewModifier()
	f.ResponseWhenFalse(fm)

	req, err := http.NewRequest("GET", "http://example.com/restest", nil)
	if err != nil {
		t.Fatalf("http.NewRequest(): got %v, want no error", err)
	}
	res := proxyutil.NewResponse(200, nil, req)

	if err := f.ModifyResponse(res); err != nil {
		t.Errorf("ModifyResponse(): got %v, want no error", err)
	}

	if got, want := tm.ResponseModified(), true; got != want {
		t.Errorf("tm.ResponseModified(): got %v, want %v", got, want)
	}

	if got, want := fm.ResponseModified(), false; got != want {
		t.Errorf("fm.ResponseModified(): got %v, want %v", got, want)
	}

	tm.Reset()
	fm.Reset()

	req, err = http.NewRequest("GET", "http://example.com/nomatch", nil)
	if err != nil {
		t.Fatalf("http.NewRequest(): got %v, want no error", err)
	}
	res = proxyutil.NewResponse(200, nil, req)

	if err := f.ModifyResponse(res); err != nil {
		t.Errorf("ModifyResponse(): got %v, want no error", err)
	}

	if tm.ResponseModified() != false {
		t.Errorf("tm.ResponseModified(): got %t, want %t", tm.ResponseModified(), false)
	}

	if got, want := tm.ResponseModified(), false; got != want {
		t.Errorf("tm.ResponseModified(): got %v, want %v", got, want)
	}

	if got, want := fm.ResponseModified(), true; got != want {
		t.Errorf("fm.ResponseModified(): got %v, want %v", got, want)
	}
}
<|endoftext|>


<commit_before>package auth

import "net/http"

func CopyRequest(r *http.Request) *http.Request {
	requestCopy := *r
	requestCopy.Header = make(http.Header)
	for k, s := range r.Header {
		requestCopy.Header[k] = s
	}
	return &requestCopy
}
<commit_msg>remove now-unused CopyRequest function<commit_after><|endoftext|>


<commit_before>package impl

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path"
	"sort"

	"veyron/lib/cmdline"
)

// Root returns a command that represents the root of the veyron tool.
func Root() *cmdline.Command {
	return &cmdline.Command{
		Name:  "veyron",
		Short: "Command-line tool for managing the veyron project",
		Long: `
The veyron tool facilitates interaction with the veyron project.
In particular, it can be used to install different veyron profiles.
`,
		Children: []*cmdline.Command{cmdSetup},
	}
}

var (
	profiles = map[string]string{
		"android":           "Android veyron development",
		"cross-compilation": "cross-compilation for Linux/ARM",
		"developer":         "core veyron development",
	}
)

func profilesDescription() string {
	result := `
<profiles> is a list of profiles to set up. Currently, the veyron tool
supports the following profiles:
`
	sortedProfiles := make([]string, 0)
	maxLength := 0
	for profile, _ := range profiles {
		sortedProfiles = append(sortedProfiles, profile)
		if len(profile) > maxLength {
			maxLength = len(profile)
		}
	}
	sort.Strings(sortedProfiles)
	for _, profile := range sortedProfiles {
		result += fmt.Sprintf("  %*s: %s\n", maxLength, profile, profiles[profile])
	}
	return result
}

// cmdSetup represents the 'setup' command of the veyron tool.
var cmdSetup = &cmdline.Command{
	Run:   runSetup,
	Name:  "setup",
	Short: "Set up the given veyron profiles",
	Long: `
To facilitate development across different platforms, veyron defines
platform-independent profiles that map different platforms to a set
of libraries and tools that can be used for a factor of veyron
development. The "setup" command can be used to install the libraries
and tools identified by the combination of the given profiles and
the host platform.
`,
	ArgsName: "<profiles>",
	ArgsLong: profilesDescription(),
}

func runSetup(cmd *cmdline.Command, args []string) error {
	// Check that the profiles to be set up exist.
	for _, arg := range args {
		if _, ok := profiles[arg]; !ok {
			cmd.Errorf("Unknown profile '%s'", arg)
			return cmdline.ErrUsage
		}
	}
	// Setup the profiles.
	root := os.Getenv("VEYRON_ROOT")
	script := path.Join(root, "environment/scripts/setup/machine/init.sh")
	for _, arg := range args {
		cmd := exec.Command(script, "-p", arg)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			return errors.New("profile setup failed")
		}
	}
	return nil
}
<commit_msg>TBR<commit_after>package impl

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path"
	"sort"

	"veyron/lib/cmdline"
)

// Root returns a command that represents the root of the veyron tool.
func Root() *cmdline.Command {
	return &cmdline.Command{
		Name:  "veyron",
		Short: "Command-line tool for managing the veyron project",
		Long: `
The veyron tool facilitates interaction with the veyron project.
In particular, it can be used to install different veyron profiles.
`,
		Children: []*cmdline.Command{cmdSetup},
	}
}

var (
	profiles = map[string]string{
		"android":           "Android veyron development",
		"cross-compilation": "cross-compilation for Linux/ARM",
		"developer":         "core veyron development",
	}
)

func profilesDescription() string {
	result := `
<profiles> is a list of profiles to set up. Currently, the veyron tool
supports the following profiles:
`
	sortedProfiles := make([]string, 0)
	maxLength := 0
	for profile, _ := range profiles {
		sortedProfiles = append(sortedProfiles, profile)
		if len(profile) > maxLength {
			maxLength = len(profile)
		}
	}
	sort.Strings(sortedProfiles)
	for _, profile := range sortedProfiles {
		result += fmt.Sprintf("  %*s: %s\n", maxLength, profile, profiles[profile])
	}
	return result
}

// cmdSetup represents the 'setup' command of the veyron tool.
var cmdSetup = &cmdline.Command{
	Run:   runSetup,
	Name:  "setup",
	Short: "Set up the given veyron profiles",
	Long: `
To facilitate development across different platforms, veyron defines
platform-independent profiles that map different platforms to a set
of libraries and tools that can be used for a factor of veyron
development. The "setup" command can be used to install the libraries
and tools identified by the combination of the given profiles and
the host platform.
`,
	ArgsName: "<profiles>",
	ArgsLong: profilesDescription(),
}

func runSetup(cmd *cmdline.Command, args []string) error {
	// Check that the profiles to be set up exist.
	for _, arg := range args {
		if _, ok := profiles[arg]; !ok {
			cmd.Errorf("Unknown profile '%s'", arg)
			return cmdline.ErrUsage
		}
	}
	// Setup the profiles.
	root := os.Getenv("VEYRON_ROOT")
	script := path.Join(root, "environment/scripts/setup/machine/init.sh")
	for _, arg := range args {
		checkpoints := path.Join(root, ".checkpoints", arg)
		if err := os.Setenv("CHECKPOINT_DIR", checkpoints); err != nil {
			return errors.New("checkpoint setup failed")
		}
		if err := os.MkdirAll(checkpoints, 0777); err != nil {
			return errors.New("checkpoint setup failed")
		}
		cmd := exec.Command(script, "-p", arg)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			return errors.New("profile setup failed")
		}
		if err := os.RemoveAll(checkpoints); err != nil {
			return errors.New("checkpoint setup failed")
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

//go:generate git-version

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"os/signal"
	"path"
	"reflect"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/cblomart/vsphere-graphite/backend"
	"github.com/cblomart/vsphere-graphite/config"
	"github.com/cblomart/vsphere-graphite/vsphere"

	"github.com/takama/daemon"

	"code.cloudfoundry.org/bytefmt"
)

const (
	// name of the service
	name        = "vsphere-graphite"
	description = "send vsphere stats to graphite"
)

var dependencies = []string{}

// Service has embedded daemon
type Service struct {
	daemon.Daemon
}

func queryVCenter(vcenter vsphere.VCenter, conf config.Configuration, channel *chan backend.Point, wg *sync.WaitGroup) {
	vcenter.Query(conf.Interval, conf.Domain, conf.ReplacePoint, conf.Properties, channel, wg)
}

// Manage by daemon commands or run the daemon
func (service *Service) Manage() (string, error) {

	usage := "Usage: vsphere-graphite install | remove | start | stop | status"

	// if received any kind of command, do it
	if len(os.Args) > 1 {
		command := os.Args[1]
		text := usage
		var err error
		switch command {
		case "install":
			text, err = service.Install()
		case "remove":
			text, err = service.Remove()
		case "start":
			text, err = service.Start()
		case "stop":
			text, err = service.Stop()
		case "status":
			text, err = service.Status()
		}
		return text, err
	}

	log.Println("Starting daemon:", path.Base(os.Args[0]))

	// read the configuration
	file, err := os.Open("/etc/" + path.Base(os.Args[0]) + ".json")
	if err != nil {
		return "Could not open configuration file", err
	}
	jsondec := json.NewDecoder(file)
	conf := config.Configuration{}
	err = jsondec.Decode(&conf)
	if err != nil {
		return "Could not decode configuration file", err
	}

	// replace all by all properties
	all := false
	if conf.Properties == nil {
		all = true
	} else {
		for _, property := range conf.Properties {
			if strings.ToLower(property) == "all" {
				all = true
				break
			}
		}
	}
	if all {
		// Reset properties
		conf.Properties = []string{}
		// Fill it with all properties keys
		for propkey := range vsphere.Properties {
			conf.Properties = append(conf.Properties, propkey)
		}
	}

	// default flush size 1000
	if conf.FlushSize == 0 {
		conf.FlushSize = 1000
	}

	// default backend prefix to "vsphere"
	if len(conf.Backend.Prefix) == 0 {
		conf.Backend.Prefix = "vsphere"
	}

	if conf.CPUProfiling {
		f, err := os.OpenFile("/tmp/vsphere-graphite-cpu.pb.gz", os.O_RDWR|os.O_CREATE, 0600) // nolint: vetshadow
		if err != nil {
			log.Fatal("could not create CPU profile: ", err)
		}
		log.Println("Will write cpu profiling to: ", f.Name())
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatal("could not start CPU profile: ", err)
		}
		defer pprof.StopCPUProfile()
	}

	//force backend values to environement varialbles if present
	s := reflect.ValueOf(conf.Backend).Elem()
	numfields := s.NumField()
	for i := 0; i < numfields; i++ {
		f := s.Field(i)
		if f.CanSet() {
			//exported field
			envname := strings.ToUpper(s.Type().Name() + "_" + s.Type().Field(i).Name)
			envval := os.Getenv(envname)
			if len(envval) > 0 {
				//environment variable set with name
				switch ftype := f.Type().Name(); ftype {
				case "string":
					f.SetString(envval)
				case "int":
					val, err := strconv.ParseInt(envval, 10, 64) // nolint: vetshadow
					if err == nil {
						f.SetInt(val)
					}
				}
			}
		}
	}

	for _, vcenter := range conf.VCenters {
		vcenter.Init(conf.Metrics)
	}

	queries, err := conf.Backend.Init()
	if err != nil {
		return "Could not initialize backend", err
	}
	defer conf.Backend.Disconnect()

	//check properties in function of backend support of metadata
	if !conf.Backend.HasMetadata() {
		properties := []string{}
		for _, confproperty := range conf.Properties {
			found := false
			for _, metricproperty := range vsphere.MetricProperties {
				if strings.ToLower(confproperty) == strings.ToLower(metricproperty) {
					found = true
					break
				}
			}
			if found {
				properties = append(properties, confproperty)
			}
		}
		conf.Properties = properties
	}

	// Set up channel on which to send signal notifications.
	// We must use a buffered channel or risk missing the signal
	// if we're not ready to receive when the signal is sent.
	interrupt := make(chan os.Signal, 1)
	//lint:ignore SA1016 in this case we wan't to quit
	signal.Notify(interrupt, os.Interrupt, os.Kill, syscall.SIGTERM) // nolint: megacheck

	// Set up a channel to receive the metrics
	metrics := make(chan backend.Point, conf.FlushSize)

	ticker := time.NewTicker(time.Second * time.Duration(conf.Interval))
	defer ticker.Stop()

	// Set up a ticker to collect metrics at givent interval (except for non scheduled backend)
	if !conf.Backend.Scheduled() {
		ticker.Stop()
	} else {
		// Start retriveing and sending metrics
		log.Println("Retrieving metrics")
		for _, vcenter := range conf.VCenters {
			go queryVCenter(*vcenter, conf, &metrics, nil)
		}
	}

	// Memory statisctics
	var memstats runtime.MemStats
	// timer to execute memory collection
	memtimer := time.NewTimer(time.Second * time.Duration(10))
	// channel to cleanup
	cleanup := make(chan bool, 1)

	// buffer for points to send
	pointbuffer := make([]*backend.Point, conf.FlushSize)
	bufferindex := 0

	// wait group for non scheduled metric retrival
	var wg sync.WaitGroup

	for {
		select {
		case value := <-metrics:
			// reset timer as a point has been recieved.
			// do that in the main thread to avoid collisions
			if !memtimer.Stop() {
				<-memtimer.C
			}
			memtimer.Reset(time.Second * time.Duration(5))
			pointbuffer[bufferindex] = &value
			bufferindex++
			if bufferindex == len(pointbuffer) {
				go conf.Backend.SendMetrics(pointbuffer)
				log.Printf("sent %d logs to backend\n", bufferindex)
				ClearBuffer(pointbuffer)
				bufferindex = 0
			}
		case request := <-*queries:
			go func() {
				log.Println("adhoc metric retrieval")
				wg.Add(len(conf.VCenters))
				for _, vcenter := range conf.VCenters {
					go queryVCenter(*vcenter, conf, request.Request, &wg)
				}
				wg.Wait()
				*request.Done <- true
				cleanup <- true
			}()
		case <-ticker.C:
			// not doing go func as it will create threads itself
			log.Println("scheduled metric retrieval")
			for _, vcenter := range conf.VCenters {
				go queryVCenter(*vcenter, conf, &metrics, nil)
			}
		case <-memtimer.C:
			if !conf.Backend.Scheduled() {
				cleanup <- true
				continue
			}
			// sent remaining values
			go conf.Backend.SendMetrics(pointbuffer)
			log.Printf("sent last %d logs to backend\n", bufferindex)
			// empty point buffer
			bufferindex = 0
			ClearBuffer(pointbuffer)
			cleanup <- true
		case <-cleanup:
			go func() {
				runtime.GC()
				debug.FreeOSMemory()
				runtime.ReadMemStats(&memstats)
				log.Printf("Memory usage : sys=%s alloc=%s\n", bytefmt.ByteSize(memstats.Sys), bytefmt.ByteSize(memstats.Alloc))
				if conf.MEMProfiling {
					f, err := os.OpenFile("/tmp/vsphere-graphite-mem.pb.gz", os.O_RDWR|os.O_CREATE, 0600) // nolin.vetshaddow
					if err != nil {
						log.Fatal("could not create Mem profile: ", err)
					}
					defer f.Close()
					log.Println("Will write mem profiling to: ", f.Name())
					if err := pprof.WriteHeapProfile(f); err != nil {
						log.Fatal("could not write Mem profile: ", err)
					}
					if err := f.Close(); err != nil {
						log.Fatal("could close Mem profile: ", err)
					}
				}
			}()
		case killSignal := <-interrupt:
			log.Println("Got signal:", killSignal)
			if bufferindex > 0 {
				conf.Backend.SendMetrics(pointbuffer[:bufferindex])
				log.Printf("Sent %d logs to backend", bufferindex)
			}
			if killSignal == os.Interrupt {
				return "Daemon was interrupted by system signal", nil
			}
			return "Daemon was killed", nil
		}
	}
}

// ClearBuffer : set all values in pointer array to nil
func ClearBuffer(buffer []*backend.Point) {
	for i := 0; i < len(buffer); i++ {
		buffer[i] = nil
	}
}

func main() {
	log.Printf("Version information: %s - %s@%s (%s)", gitTag, gitShortCommit, gitBranch, gitStatus)
	srv, err := daemon.New(name, description, dependencies...)
	if err != nil {
		log.Println("Error: ", err)
		os.Exit(1)
	}
	service := &Service{srv}
	status, err := service.Manage()
	if err != nil {
		log.Println(status, "Error: ", err)
		os.Exit(1)
	}
	fmt.Println(status)
}
<commit_msg>separate accumulating buffer and send buffer<commit_after>package main

//go:generate git-version

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"os/signal"
	"path"
	"reflect"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/cblomart/vsphere-graphite/backend"
	"github.com/cblomart/vsphere-graphite/config"
	"github.com/cblomart/vsphere-graphite/vsphere"

	"github.com/takama/daemon"

	"code.cloudfoundry.org/bytefmt"
)

const (
	// name of the service
	name        = "vsphere-graphite"
	description = "send vsphere stats to graphite"
)

var dependencies = []string{}

// Service has embedded daemon
type Service struct {
	daemon.Daemon
}

func queryVCenter(vcenter vsphere.VCenter, conf config.Configuration, channel *chan backend.Point, wg *sync.WaitGroup) {
	vcenter.Query(conf.Interval, conf.Domain, conf.ReplacePoint, conf.Properties, channel, wg)
}

// Manage by daemon commands or run the daemon
func (service *Service) Manage() (string, error) {

	usage := "Usage: vsphere-graphite install | remove | start | stop | status"

	// if received any kind of command, do it
	if len(os.Args) > 1 {
		command := os.Args[1]
		text := usage
		var err error
		switch command {
		case "install":
			text, err = service.Install()
		case "remove":
			text, err = service.Remove()
		case "start":
			text, err = service.Start()
		case "stop":
			text, err = service.Stop()
		case "status":
			text, err = service.Status()
		}
		return text, err
	}

	log.Println("Starting daemon:", path.Base(os.Args[0]))

	// read the configuration
	file, err := os.Open("/etc/" + path.Base(os.Args[0]) + ".json")
	if err != nil {
		return "Could not open configuration file", err
	}
	jsondec := json.NewDecoder(file)
	conf := config.Configuration{}
	err = jsondec.Decode(&conf)
	if err != nil {
		return "Could not decode configuration file", err
	}

	// replace all by all properties
	all := false
	if conf.Properties == nil {
		all = true
	} else {
		for _, property := range conf.Properties {
			if strings.ToLower(property) == "all" {
				all = true
				break
			}
		}
	}
	if all {
		// Reset properties
		conf.Properties = []string{}
		// Fill it with all properties keys
		for propkey := range vsphere.Properties {
			conf.Properties = append(conf.Properties, propkey)
		}
	}

	// default flush size 1000
	if conf.FlushSize == 0 {
		conf.FlushSize = 1000
	}

	// default backend prefix to "vsphere"
	if len(conf.Backend.Prefix) == 0 {
		conf.Backend.Prefix = "vsphere"
	}

	if conf.CPUProfiling {
		f, err := os.OpenFile("/tmp/vsphere-graphite-cpu.pb.gz", os.O_RDWR|os.O_CREATE, 0600) // nolint: vetshadow
		if err != nil {
			log.Fatal("could not create CPU profile: ", err)
		}
		log.Println("Will write cpu profiling to: ", f.Name())
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatal("could not start CPU profile: ", err)
		}
		defer pprof.StopCPUProfile()
	}

	//force backend values to environement varialbles if present
	s := reflect.ValueOf(conf.Backend).Elem()
	numfields := s.NumField()
	for i := 0; i < numfields; i++ {
		f := s.Field(i)
		if f.CanSet() {
			//exported field
			envname := strings.ToUpper(s.Type().Name() + "_" + s.Type().Field(i).Name)
			envval := os.Getenv(envname)
			if len(envval) > 0 {
				//environment variable set with name
				switch ftype := f.Type().Name(); ftype {
				case "string":
					f.SetString(envval)
				case "int":
					val, err := strconv.ParseInt(envval, 10, 64) // nolint: vetshadow
					if err == nil {
						f.SetInt(val)
					}
				}
			}
		}
	}

	for _, vcenter := range conf.VCenters {
		vcenter.Init(conf.Metrics)
	}

	queries, err := conf.Backend.Init()
	if err != nil {
		return "Could not initialize backend", err
	}
	defer conf.Backend.Disconnect()

	//check properties in function of backend support of metadata
	if !conf.Backend.HasMetadata() {
		properties := []string{}
		for _, confproperty := range conf.Properties {
			found := false
			for _, metricproperty := range vsphere.MetricProperties {
				if strings.ToLower(confproperty) == strings.ToLower(metricproperty) {
					found = true
					break
				}
			}
			if found {
				properties = append(properties, confproperty)
			}
		}
		conf.Properties = properties
	}

	// Set up channel on which to send signal notifications.
	// We must use a buffered channel or risk missing the signal
	// if we're not ready to receive when the signal is sent.
	interrupt := make(chan os.Signal, 1)
	//lint:ignore SA1016 in this case we wan't to quit
	signal.Notify(interrupt, os.Interrupt, os.Kill, syscall.SIGTERM) // nolint: megacheck

	// Set up a channel to receive the metrics
	metrics := make(chan backend.Point, conf.FlushSize)

	ticker := time.NewTicker(time.Second * time.Duration(conf.Interval))
	defer ticker.Stop()

	// Set up a ticker to collect metrics at givent interval (except for non scheduled backend)
	if !conf.Backend.Scheduled() {
		ticker.Stop()
	} else {
		// Start retriveing and sending metrics
		log.Println("Retrieving metrics")
		for _, vcenter := range conf.VCenters {
			go queryVCenter(*vcenter, conf, &metrics, nil)
		}
	}

	// Memory statisctics
	var memstats runtime.MemStats
	// timer to execute memory collection
	memtimer := time.NewTimer(time.Second * time.Duration(10))
	// channel to cleanup
	cleanup := make(chan bool, 1)

	// buffer for points to send
	pointbuffer := make([]*backend.Point, conf.FlushSize)
	bufferindex := 0

	// wait group for non scheduled metric retrival
	var wg sync.WaitGroup

	for {
		select {
		case value := <-metrics:
			// reset timer as a point has been recieved.
			// do that in the main thread to avoid collisions
			if !memtimer.Stop() {
				<-memtimer.C
			}
			memtimer.Reset(time.Second * time.Duration(5))
			pointbuffer[bufferindex] = &value
			bufferindex++
			if bufferindex == len(pointbuffer) {
				t := make([]*backend.Point, len(pointbuffer))
				copy(t,pointbuffer)
				ClearBuffer(pointbuffer)
				bufferindex = 0
				go conf.Backend.SendMetrics(t)
				log.Printf("sent %d logs to backend\n", len(t))
			}
		case request := <-*queries:
			go func() {
				log.Println("adhoc metric retrieval")
				wg.Add(len(conf.VCenters))
				for _, vcenter := range conf.VCenters {
					go queryVCenter(*vcenter, conf, request.Request, &wg)
				}
				wg.Wait()
				*request.Done <- true
				cleanup <- true
			}()
		case <-ticker.C:
			// not doing go func as it will create threads itself
			log.Println("scheduled metric retrieval")
			for _, vcenter := range conf.VCenters {
				go queryVCenter(*vcenter, conf, &metrics, nil)
			}
		case <-memtimer.C:
			if !conf.Backend.Scheduled() {
				continue
			}
			// sent remaining values
			// copy to send point to appart buffer
			t := make([]*backend.Point, len(pointbuffer))
			copy(t,pointbuffer)
			// clear main buffer
			ClearBuffer(pointbuffer)
			bufferindex = 0
			// send sent buffer
			go conf.Backend.SendMetrics(t)
			log.Printf("sent last %d logs to backend\n", len(t))
			// empty point buffer
			cleanup <- true
		case <-cleanup:
			go func() {
				runtime.GC()
				debug.FreeOSMemory()
				runtime.ReadMemStats(&memstats)
				log.Printf("Memory usage : sys=%s alloc=%s\n", bytefmt.ByteSize(memstats.Sys), bytefmt.ByteSize(memstats.Alloc))
				if conf.MEMProfiling {
					f, err := os.OpenFile("/tmp/vsphere-graphite-mem.pb.gz", os.O_RDWR|os.O_CREATE, 0600) // nolin.vetshaddow
					if err != nil {
						log.Fatal("could not create Mem profile: ", err)
					}
					defer f.Close()
					log.Println("Will write mem profiling to: ", f.Name())
					if err := pprof.WriteHeapProfile(f); err != nil {
						log.Fatal("could not write Mem profile: ", err)
					}
					if err := f.Close(); err != nil {
						log.Fatal("could close Mem profile: ", err)
					}
				}
			}()
		case killSignal := <-interrupt:
			log.Println("Got signal:", killSignal)
			if bufferindex > 0 {
				conf.Backend.SendMetrics(pointbuffer[:bufferindex])
				log.Printf("Sent %d logs to backend", bufferindex)
			}
			if killSignal == os.Interrupt {
				return "Daemon was interrupted by system signal", nil
			}
			return "Daemon was killed", nil
		}
	}
}

// ClearBuffer : set all values in pointer array to nil
func ClearBuffer(buffer []*backend.Point) {
	for i := 0; i < len(buffer); i++ {
		buffer[i] = nil
	}
}

func main() {
	log.Printf("Version information: %s - %s@%s (%s)", gitTag, gitShortCommit, gitBranch, gitStatus)
	srv, err := daemon.New(name, description, dependencies...)
	if err != nil {
		log.Println("Error: ", err)
		os.Exit(1)
	}
	service := &Service{srv}
	status, err := service.Manage()
	if err != nil {
		log.Println(status, "Error: ", err)
		os.Exit(1)
	}
	fmt.Println(status)
}
<|endoftext|>