<commit_before>package btn

import (
	"image/color"
	"strconv"

	"github.com/oakmound/oak/dlog"

	"github.com/oakmound/oak/event"
	"github.com/oakmound/oak/mouse"
	"github.com/oakmound/oak/render"
	"github.com/oakmound/oak/render/mod"
)

// A Generator defines the variables used to create buttons from optional arguments
type Generator struct {
	X, Y           float64
	W, H           float64
	TxtX, TxtY     float64
	Color          color.Color
	Color2         color.Color
	ProgressFunc   func(x, y, w, h int) float64
	Mod            mod.Transform
	R              render.Modifiable
	R1             render.Modifiable
	R2             render.Modifiable
	RS             []render.Modifiable
	Cid            event.CID
	Font           *render.Font
	Layers         []int
	Text           string
	Children       []Generator
	Bindings       map[string][]event.Bindable
	Trigger        string
	Toggle         *bool
	ListChoice     *int
	Group          *Group
	DisallowRevert bool
}

func defGenerator() Generator {
	// A number of these fields could be removed, because they are the zero
	// value, but are left for documentation
	return Generator{
		X:     0,
		Y:     0,
		W:     1,
		H:     1,
		TxtX:  0,
		TxtY:  0,
		Color: color.RGBA{255, 0, 0, 255},
		Mod:   nil,
		R:     nil,
		R1:    nil,
		R2:    nil,

		Children: []Generator{},
		Cid:      0,
		Font:     nil,
		Layers:   []int{0},
		Text:     "Button",
		Bindings: make(map[string][]event.Bindable),
		Trigger:  "MouseClickOn",

		Toggle: nil,
	}
}

// Generate creates a Button from a generator.
func (g Generator) Generate() Btn {
	return g.generate(nil)
}

func (g Generator) generate(parent *Generator) Btn {
	var box render.Modifiable
	switch {
	case g.Toggle != nil:
		//Handles checks and other toggle situations
		start := "on"
		if !(*g.Toggle) {
			start = "off"
		}
		if _, ok := g.R1.(*render.Reverting); !ok {
			g.R1 = render.NewReverting(g.R1)
		}
		if _, ok := g.R2.(*render.Reverting); !ok {
			g.R2 = render.NewReverting(g.R2)
		}
		box = render.NewSwitch(start, map[string]render.Modifiable{
			"on":  g.R1,
			"off": g.R2,
		})
		g.Bindings["MouseClickOn"] = append(g.Bindings["MouseClickOn"], toggleFxn(g))
	case g.ListChoice != nil:

		start := "list" + strconv.Itoa(*g.ListChoice)
		mp := make(map[string]render.Modifiable)
		for i, r := range g.RS {
			if _, ok := r.(*render.Reverting); !ok {
				r = render.NewReverting(r)
			}
			mp["list"+strconv.Itoa(i)] = r
		}
		box = render.NewSwitch(start, mp)

		g.Bindings["MouseClickOn"] = append(g.Bindings["MouseClickOn"], listFxn(g))
	case g.R != nil:
		box = g.R
	case g.ProgressFunc != nil:
		box = render.NewGradientBox(int(g.W), int(g.H), g.Color, g.Color2, g.ProgressFunc)
	default:
		box = render.NewColorBox(int(g.W), int(g.H), g.Color)
	}
	if !g.DisallowRevert {
		box = render.NewReverting(box)
	}

	if g.Mod != nil {
		box.Modify(g.Mod)
	}
	font := g.Font
	if font == nil {
		font = render.DefFont()
	}
	var btn Btn
	if g.Text != "" {
		txtbx := NewTextBox(g.Cid, g.X, g.Y, g.W, g.H, g.TxtX, g.TxtY, font, box, g.Layers...)
		txtbx.SetString(g.Text)
		btn = txtbx
	} else {
		btn = NewBox(g.Cid, g.X, g.Y, g.W, g.H, box, g.Layers...)
	}

	for k, v := range g.Bindings {
		for _, b := range v {
			btn.Bind(b, k)
		}
	}

	err := mouse.PhaseCollision(btn.GetSpace())
	dlog.ErrorCheck(err)

	if g.Group != nil {
		g.Group.members = append(g.Group.members, btn)
	}

	return btn
}

// An Option is used to populate generator fields prior to generation of a button
type Option func(Generator) Generator

// New creates a button with the given options and defaults for all variables not set.
func New(opts ...Option) Btn {
	g := defGenerator()
	for _, opt := range opts {
		if opt == nil {
			continue
		}
		g = opt(g)
	}
	return g.Generate()
}

// toggleFxn sets up the mouseclick binding for toggle buttons created for goreport cyclo decrease
func toggleFxn(g Generator) func(id int, nothing interface{}) int {
	return func(id int, nothing interface{}) int {
		btn := event.GetEntity(id).(Btn)
		if btn.GetRenderable().(*render.Switch).Get() == "on" {
			if g.Group != nil && g.Group.active == btn {
				g.Group.active = nil
			}
			btn.GetRenderable().(*render.Switch).Set("off")
		} else {
			// We can pull this out to separate binding if group != nil
			if g.Group != nil {
				g.Group.active = btn
				for _, b := range g.Group.members {
					if b.GetRenderable().(*render.Switch).Get() == "on" {
						b.Trigger("MouseClickOn", nil)
					}
				}
			}
			btn.GetRenderable().(*render.Switch).Set("on")

		}
		*g.Toggle = !*g.Toggle

		return 0
	}
}

// listFxn sets up the mouseclick binding for list buttons created for goreport cyclo decrease
func listFxn(g Generator) func(id int, button interface{}) int {
	return func(id int, button interface{}) int {
		btn := event.GetEntity(id).(Btn)
		i := *g.ListChoice
		mEvent := button.(mouse.Event)

		if mEvent.Button == "LeftMouse" {
			i++
			if i == len(g.RS) {
				i = 0
			}

		} else if mEvent.Button == "RightMouse" {
			i--
			if i < 0 {
				i += len(g.RS)
			}
		}

		btn.GetRenderable().(*render.Switch).Set("list" + strconv.Itoa(i))

		*g.ListChoice = i

		return 0
	}
}
<commit_msg>Fix btn toggle for revertings<commit_after>package btn

import (
	"image/color"
	"strconv"

	"github.com/oakmound/oak/dlog"

	"github.com/oakmound/oak/event"
	"github.com/oakmound/oak/mouse"
	"github.com/oakmound/oak/render"
	"github.com/oakmound/oak/render/mod"
)

// A Generator defines the variables used to create buttons from optional arguments
type Generator struct {
	X, Y           float64
	W, H           float64
	TxtX, TxtY     float64
	Color          color.Color
	Color2         color.Color
	ProgressFunc   func(x, y, w, h int) float64
	Mod            mod.Transform
	R              render.Modifiable
	R1             render.Modifiable
	R2             render.Modifiable
	RS             []render.Modifiable
	Cid            event.CID
	Font           *render.Font
	Layers         []int
	Text           string
	Children       []Generator
	Bindings       map[string][]event.Bindable
	Trigger        string
	Toggle         *bool
	ListChoice     *int
	Group          *Group
	DisallowRevert bool
}

func defGenerator() Generator {
	// A number of these fields could be removed, because they are the zero
	// value, but are left for documentation
	return Generator{
		X:     0,
		Y:     0,
		W:     1,
		H:     1,
		TxtX:  0,
		TxtY:  0,
		Color: color.RGBA{255, 0, 0, 255},
		Mod:   nil,
		R:     nil,
		R1:    nil,
		R2:    nil,

		Children: []Generator{},
		Cid:      0,
		Font:     nil,
		Layers:   []int{0},
		Text:     "Button",
		Bindings: make(map[string][]event.Bindable),
		Trigger:  "MouseClickOn",

		Toggle: nil,
	}
}

// Generate creates a Button from a generator.
func (g Generator) Generate() Btn {
	return g.generate(nil)
}

func (g Generator) generate(parent *Generator) Btn {
	var box render.Modifiable
	switch {
	case g.Toggle != nil:
		//Handles checks and other toggle situations
		start := "on"
		if !(*g.Toggle) {
			start = "off"
		}
		if _, ok := g.R1.(*render.Reverting); !ok {
			g.R1 = render.NewReverting(g.R1)
		}
		if _, ok := g.R2.(*render.Reverting); !ok {
			g.R2 = render.NewReverting(g.R2)
		}
		box = render.NewSwitch(start, map[string]render.Modifiable{
			"on":  g.R1,
			"off": g.R2,
		})
		g.Bindings["MouseClickOn"] = append(g.Bindings["MouseClickOn"], toggleFxn(g))
	case g.ListChoice != nil:

		start := "list" + strconv.Itoa(*g.ListChoice)
		mp := make(map[string]render.Modifiable)
		for i, r := range g.RS {
			if _, ok := r.(*render.Reverting); !ok {
				r = render.NewReverting(r)
			}
			mp["list"+strconv.Itoa(i)] = r
		}
		box = render.NewSwitch(start, mp)

		g.Bindings["MouseClickOn"] = append(g.Bindings["MouseClickOn"], listFxn(g))
	case g.R != nil:
		box = g.R
	case g.ProgressFunc != nil:
		box = render.NewGradientBox(int(g.W), int(g.H), g.Color, g.Color2, g.ProgressFunc)
	default:
		box = render.NewColorBox(int(g.W), int(g.H), g.Color)
	}
	if !g.DisallowRevert {
		box = render.NewReverting(box)
	}

	if g.Mod != nil {
		box.Modify(g.Mod)
	}
	font := g.Font
	if font == nil {
		font = render.DefFont()
	}
	var btn Btn
	if g.Text != "" {
		txtbx := NewTextBox(g.Cid, g.X, g.Y, g.W, g.H, g.TxtX, g.TxtY, font, box, g.Layers...)
		txtbx.SetString(g.Text)
		btn = txtbx
	} else {
		btn = NewBox(g.Cid, g.X, g.Y, g.W, g.H, box, g.Layers...)
	}

	for k, v := range g.Bindings {
		for _, b := range v {
			btn.Bind(b, k)
		}
	}

	err := mouse.PhaseCollision(btn.GetSpace())
	dlog.ErrorCheck(err)

	if g.Group != nil {
		g.Group.members = append(g.Group.members, btn)
	}

	return btn
}

// An Option is used to populate generator fields prior to generation of a button
type Option func(Generator) Generator

// New creates a button with the given options and defaults for all variables not set.
func New(opts ...Option) Btn {
	g := defGenerator()
	for _, opt := range opts {
		if opt == nil {
			continue
		}
		g = opt(g)
	}
	return g.Generate()
}

type switcher interface {
	Get() string
	Set(string) error
}

// toggleFxn sets up the mouseclick binding for toggle buttons created for goreport cyclo decrease
func toggleFxn(g Generator) func(id int, nothing interface{}) int {
	return func(id int, nothing interface{}) int {
		btn := event.GetEntity(id).(Btn)
		if btn.GetRenderable().(switcher).Get() == "on" {
			if g.Group != nil && g.Group.active == btn {
				g.Group.active = nil
			}
			btn.GetRenderable().(switcher).Set("off")
		} else {
			// We can pull this out to separate binding if group != nil
			if g.Group != nil {
				g.Group.active = btn
				for _, b := range g.Group.members {
					if b.GetRenderable().(switcher).Get() == "on" {
						b.Trigger("MouseClickOn", nil)
					}
				}
			}
			btn.GetRenderable().(switcher).Set("on")

		}
		*g.Toggle = !*g.Toggle

		return 0
	}
}

// listFxn sets up the mouseclick binding for list buttons created for goreport cyclo decrease
func listFxn(g Generator) func(id int, button interface{}) int {
	return func(id int, button interface{}) int {
		btn := event.GetEntity(id).(Btn)
		i := *g.ListChoice
		mEvent := button.(mouse.Event)

		if mEvent.Button == "LeftMouse" {
			i++
			if i == len(g.RS) {
				i = 0
			}

		} else if mEvent.Button == "RightMouse" {
			i--
			if i < 0 {
				i += len(g.RS)
			}
		}

		btn.GetRenderable().(*render.Switch).Set("list" + strconv.Itoa(i))

		*g.ListChoice = i

		return 0
	}
}
<|endoftext|>


<commit_before>package tcp_output

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"time"

	"github.com/karimra/gnmic/formatters"
	"github.com/karimra/gnmic/outputs"
	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/protobuf/proto"
)

const (
	defaultRetryTimer = 2 * time.Second
	defaultNumWorkers = 1
	loggingPrefix     = "[tcp_output] "
)

func init() {
	outputs.Register("tcp", func() outputs.Output {
		return &TCPOutput{
			Cfg:    &Config{},
			logger: log.New(ioutil.Discard, loggingPrefix, log.LstdFlags|log.Lmicroseconds),
		}
	})
}

type TCPOutput struct {
	Cfg *Config

	cancelFn context.CancelFunc
	buffer   chan []byte
	limiter  *time.Ticker
	logger   *log.Logger
	mo       *formatters.MarshalOptions
	evps     []formatters.EventProcessor
}

type Config struct {
	Address            string        `mapstructure:"address,omitempty"` // ip:port
	Rate               time.Duration `mapstructure:"rate,omitempty"`
	BufferSize         uint          `mapstructure:"buffer-size,omitempty"`
	Format             string        `mapstructure:"format,omitempty"`
	OverrideTimestamps bool          `mapstructure:"override-timestamps,omitempty"`
	KeepAlive          time.Duration `mapstructure:"keep-alive,omitempty"`
	RetryInterval      time.Duration `mapstructure:"retry-interval,omitempty"`
	NumWorkers         int           `mapstructure:"num-workers,omitempty"`
	EnableMetrics      bool          `mapstructure:"enable-metrics,omitempty"`
	EventProcessors    []string      `mapstructure:"event-processors,omitempty"`
}

func (t *TCPOutput) SetLogger(logger *log.Logger) {
	if logger != nil && t.logger != nil {
		t.logger.SetOutput(logger.Writer())
		t.logger.SetFlags(logger.Flags())
	}
}

func (t *TCPOutput) SetEventProcessors(ps map[string]map[string]interface{}, logger *log.Logger, tcs map[string]interface{}) {
	for _, epName := range t.Cfg.EventProcessors {
		if epCfg, ok := ps[epName]; ok {
			epType := ""
			for k := range epCfg {
				epType = k
				break
			}
			if in, ok := formatters.EventProcessors[epType]; ok {
				ep := in()
				err := ep.Init(epCfg[epType], formatters.WithLogger(logger), formatters.WithTargets(tcs))
				if err != nil {
					t.logger.Printf("failed initializing event processor '%s' of type='%s': %v", epName, epType, err)
					continue
				}
				t.evps = append(t.evps, ep)
				t.logger.Printf("added event processor '%s' of type=%s to tcp output", epName, epType)
				continue
			}
			t.logger.Printf("%q event processor has an unknown type=%q", epName, epType)
			continue
		}
		t.logger.Printf("%q event processor not found!", epName)
	}
}

func (t *TCPOutput) Init(ctx context.Context, name string, cfg map[string]interface{}, opts ...outputs.Option) error {
	err := outputs.DecodeConfig(cfg, t.Cfg)
	if err != nil {
		return err
	}
	for _, opt := range opts {
		opt(t)
	}
	_, _, err = net.SplitHostPort(t.Cfg.Address)
	if err != nil {
		return fmt.Errorf("wrong address format: %v", err)
	}
	t.buffer = make(chan []byte, t.Cfg.BufferSize)
	if t.Cfg.Rate > 0 {
		t.limiter = time.NewTicker(t.Cfg.Rate)
	}
	if t.Cfg.RetryInterval == 0 {
		t.Cfg.RetryInterval = defaultRetryTimer
	}
	if t.Cfg.NumWorkers < 1 {
		t.Cfg.NumWorkers = defaultNumWorkers
	}
	t.mo = &formatters.MarshalOptions{
		Format:     t.Cfg.Format,
		OverrideTS: t.Cfg.OverrideTimestamps,
	}
	go func() {
		<-ctx.Done()
		t.Close()
	}()

	ctx, t.cancelFn = context.WithCancel(ctx)
	for i := 0; i < t.Cfg.NumWorkers; i++ {
		go t.start(ctx, i)
	}
	return nil
}

func (t *TCPOutput) Write(ctx context.Context, m proto.Message, meta outputs.Meta) {
	if m == nil {
		return
	}
	select {
	case <-ctx.Done():
		return
	default:
		b, err := t.mo.Marshal(m, meta, t.evps...)
		if err != nil {
			t.logger.Printf("failed marshaling proto msg: %v", err)
			return
		}
		t.buffer <- b
	}
}

func (t *TCPOutput) WriteEvent(ctx context.Context, ev *formatters.EventMsg) {}

func (t *TCPOutput) Close() error {
	t.cancelFn()
	if t.limiter != nil {
		t.limiter.Stop()
	}
	return nil
}
func (t *TCPOutput) RegisterMetrics(reg *prometheus.Registry) {}

func (t *TCPOutput) String() string {
	b, err := json.Marshal(t)
	if err != nil {
		return ""
	}
	return string(b)
}

func (t *TCPOutput) start(ctx context.Context, idx int) {
	workerLogPrefix := fmt.Sprintf("worker-%d", idx)
START:
	tcpAddr, err := net.ResolveTCPAddr("tcp", t.Cfg.Address)
	if err != nil {
		t.logger.Printf("%s failed to resolve address: %v", workerLogPrefix, err)
		time.Sleep(t.Cfg.RetryInterval)
		goto START
	}
	conn, err := net.DialTCP("tcp", nil, tcpAddr)
	if err != nil {
		t.logger.Printf("%s failed to dial TCP: %v", workerLogPrefix, err)
		time.Sleep(t.Cfg.RetryInterval)
		goto START
	}
	defer conn.Close()
	if t.Cfg.KeepAlive > 0 {
		conn.SetKeepAlive(true)
		conn.SetKeepAlivePeriod(t.Cfg.KeepAlive)
	}
	defer t.Close()
	for {
		select {
		case <-ctx.Done():
			return
		case b := <-t.buffer:
			if t.limiter != nil {
				<-t.limiter.C
			}
			_, err = conn.Write(b)
			if err != nil {
				t.logger.Printf("%s failed sending tcp bytes: %v", workerLogPrefix, err)
				conn.Close()
				time.Sleep(t.Cfg.RetryInterval)
				goto START
			}
		}
	}
}

func (t *TCPOutput) SetName(name string)        {}
func (t *TCPOutput) SetClusterName(name string) {}
<commit_msg>tcp output<commit_after>package tcp_output

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"text/template"
	"time"

	"github.com/karimra/gnmic/formatters"
	"github.com/karimra/gnmic/outputs"
	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/protobuf/proto"
)

const (
	defaultRetryTimer = 2 * time.Second
	defaultNumWorkers = 1
	loggingPrefix     = "[tcp_output] "
)

func init() {
	outputs.Register("tcp", func() outputs.Output {
		return &TCPOutput{
			Cfg:    &Config{},
			logger: log.New(ioutil.Discard, loggingPrefix, log.LstdFlags|log.Lmicroseconds),
		}
	})
}

type TCPOutput struct {
	Cfg *Config

	cancelFn context.CancelFunc
	buffer   chan []byte
	limiter  *time.Ticker
	logger   *log.Logger
	mo       *formatters.MarshalOptions
	evps     []formatters.EventProcessor

	targetTpl *template.Template
}

type Config struct {
	Address            string        `mapstructure:"address,omitempty"` // ip:port
	Rate               time.Duration `mapstructure:"rate,omitempty"`
	BufferSize         uint          `mapstructure:"buffer-size,omitempty"`
	Format             string        `mapstructure:"format,omitempty"`
	AddTarget          string        `mapstructure:"add-target,omitempty"`
	TargetTemplate     string        `mapstructure:"target-template,omitempty"`
	OverrideTimestamps bool          `mapstructure:"override-timestamps,omitempty"`
	KeepAlive          time.Duration `mapstructure:"keep-alive,omitempty"`
	RetryInterval      time.Duration `mapstructure:"retry-interval,omitempty"`
	NumWorkers         int           `mapstructure:"num-workers,omitempty"`
	EnableMetrics      bool          `mapstructure:"enable-metrics,omitempty"`
	EventProcessors    []string      `mapstructure:"event-processors,omitempty"`
}

func (t *TCPOutput) SetLogger(logger *log.Logger) {
	if logger != nil && t.logger != nil {
		t.logger.SetOutput(logger.Writer())
		t.logger.SetFlags(logger.Flags())
	}
}

func (t *TCPOutput) SetEventProcessors(ps map[string]map[string]interface{}, logger *log.Logger, tcs map[string]interface{}) {
	for _, epName := range t.Cfg.EventProcessors {
		if epCfg, ok := ps[epName]; ok {
			epType := ""
			for k := range epCfg {
				epType = k
				break
			}
			if in, ok := formatters.EventProcessors[epType]; ok {
				ep := in()
				err := ep.Init(epCfg[epType], formatters.WithLogger(logger), formatters.WithTargets(tcs))
				if err != nil {
					t.logger.Printf("failed initializing event processor '%s' of type='%s': %v", epName, epType, err)
					continue
				}
				t.evps = append(t.evps, ep)
				t.logger.Printf("added event processor '%s' of type=%s to tcp output", epName, epType)
				continue
			}
			t.logger.Printf("%q event processor has an unknown type=%q", epName, epType)
			continue
		}
		t.logger.Printf("%q event processor not found!", epName)
	}
}

func (t *TCPOutput) Init(ctx context.Context, name string, cfg map[string]interface{}, opts ...outputs.Option) error {
	err := outputs.DecodeConfig(cfg, t.Cfg)
	if err != nil {
		return err
	}
	for _, opt := range opts {
		opt(t)
	}
	_, _, err = net.SplitHostPort(t.Cfg.Address)
	if err != nil {
		return fmt.Errorf("wrong address format: %v", err)
	}
	t.buffer = make(chan []byte, t.Cfg.BufferSize)
	if t.Cfg.Rate > 0 {
		t.limiter = time.NewTicker(t.Cfg.Rate)
	}
	if t.Cfg.RetryInterval == 0 {
		t.Cfg.RetryInterval = defaultRetryTimer
	}
	if t.Cfg.NumWorkers < 1 {
		t.Cfg.NumWorkers = defaultNumWorkers
	}
	t.mo = &formatters.MarshalOptions{
		Format:     t.Cfg.Format,
		OverrideTS: t.Cfg.OverrideTimestamps,
	}

	if t.Cfg.TargetTemplate == "" {
		t.targetTpl = outputs.DefaultTargetTemplate
	} else if t.Cfg.AddTarget != "" {
		t.targetTpl, err = template.New("target-template").
			Funcs(outputs.TemplateFuncs).
			Parse(t.Cfg.TargetTemplate)
		if err != nil {
			return err
		}
	}
	go func() {
		<-ctx.Done()
		t.Close()
	}()

	ctx, t.cancelFn = context.WithCancel(ctx)
	for i := 0; i < t.Cfg.NumWorkers; i++ {
		go t.start(ctx, i)
	}
	return nil
}

func (t *TCPOutput) Write(ctx context.Context, m proto.Message, meta outputs.Meta) {
	if m == nil {
		return
	}
	var err error
	select {
	case <-ctx.Done():
		return
	default:
		err = outputs.AddSubscriptionTarget(m, meta, t.Cfg.AddTarget, t.targetTpl)
		if err != nil {
			t.logger.Printf("failed to add target to the response: %v", err)
		}
		b, err := t.mo.Marshal(m, meta, t.evps...)
		if err != nil {
			t.logger.Printf("failed marshaling proto msg: %v", err)
			return
		}
		t.buffer <- b
	}
}

func (t *TCPOutput) WriteEvent(ctx context.Context, ev *formatters.EventMsg) {}

func (t *TCPOutput) Close() error {
	t.cancelFn()
	if t.limiter != nil {
		t.limiter.Stop()
	}
	return nil
}
func (t *TCPOutput) RegisterMetrics(reg *prometheus.Registry) {}

func (t *TCPOutput) String() string {
	b, err := json.Marshal(t)
	if err != nil {
		return ""
	}
	return string(b)
}

func (t *TCPOutput) start(ctx context.Context, idx int) {
	workerLogPrefix := fmt.Sprintf("worker-%d", idx)
START:
	tcpAddr, err := net.ResolveTCPAddr("tcp", t.Cfg.Address)
	if err != nil {
		t.logger.Printf("%s failed to resolve address: %v", workerLogPrefix, err)
		time.Sleep(t.Cfg.RetryInterval)
		goto START
	}
	conn, err := net.DialTCP("tcp", nil, tcpAddr)
	if err != nil {
		t.logger.Printf("%s failed to dial TCP: %v", workerLogPrefix, err)
		time.Sleep(t.Cfg.RetryInterval)
		goto START
	}
	defer conn.Close()
	if t.Cfg.KeepAlive > 0 {
		conn.SetKeepAlive(true)
		conn.SetKeepAlivePeriod(t.Cfg.KeepAlive)
	}
	defer t.Close()
	for {
		select {
		case <-ctx.Done():
			return
		case b := <-t.buffer:
			if t.limiter != nil {
				<-t.limiter.C
			}
			_, err = conn.Write(b)
			if err != nil {
				t.logger.Printf("%s failed sending tcp bytes: %v", workerLogPrefix, err)
				conn.Close()
				time.Sleep(t.Cfg.RetryInterval)
				goto START
			}
		}
	}
}

func (t *TCPOutput) SetName(name string)        {}
func (t *TCPOutput) SetClusterName(name string) {}
<|endoftext|>


<commit_before>package leveldb

import (
	"encoding/hex"
	"fmt"
	"path/filepath"
	"strings"

	"gnd.la/blobstore/driver"
	"gnd.la/config"
	"gnd.la/internal"
	"gnd.la/util/pathutil"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/iterator"
	"github.com/syndtr/goleveldb/leveldb/opt"
)

var (
	syncOptions       = &opt.WriteOptions{Sync: true}
	checkChunkOptions = &opt.ReadOptions{DontFillCache: true, Strict: opt.NoStrict}
)

type leveldbDriver struct {
	files  *leveldb.DB
	chunks *leveldb.DB
	dir    string
}

func (d *leveldbDriver) Create(id string) (driver.WFile, error) {
	return newWFile(d, id), nil
}

func (d *leveldbDriver) Open(id string) (driver.RFile, error) {
	value, err := d.files.Get(internal.StringToBytes(id), nil)
	if err != nil {
		if err == leveldb.ErrNotFound {
			return nil, fmt.Errorf("file %s not found", id)
		}
		return nil, err
	}
	count := int(littleEndian.Uint32(value))
	if count == 0 {
		// Data is inline
		return &rfile{chunks: [][]byte{value[4:]}}, nil
	}
	pos := 4
	chunks := make([][]byte, count)
	for ii := 0; ii < count; ii++ {
		size := int(littleEndian.Uint32(value[pos:]))
		pos += 4
		key := value[pos : pos+size]
		chunk, err := d.chunks.Get(key, nil)
		if err != nil {
			if err == leveldb.ErrNotFound {
				return nil, fmt.Errorf("chunk %s in file %s not found", hex.EncodeToString(key), id)
			}
			return nil, err
		}
		chunks[ii] = chunk
		pos += size
	}
	return &rfile{chunks: chunks}, nil
}

func (d *leveldbDriver) Remove(id string) error {
	return d.files.Delete([]byte(id), syncOptions)
}

func (d *leveldbDriver) Close() error {
	if err := d.files.Close(); err != nil {
		return err
	}
	if err := d.chunks.Close(); err != nil {
		return err
	}
	return nil
}

func (d *leveldbDriver) Iter() (driver.Iter, error) {
	iter := d.files.NewIterator(nil, nil)
	return &leveldbIter{iter: iter}, nil
}

func leveldbOpener(url *config.URL) (driver.Driver, error) {
	value := url.Value
	if !filepath.IsAbs(value) {
		value = pathutil.Relative(value)
	}
	opts := &opt.Options{}
	if url.Fragment["nocompress"] != "" {
		opts.Compression = opt.NoCompression
	}
	if url.Fragment["nocreate"] != "" {
		opts.ErrorIfMissing = true
	}
	filesDir := filepath.Join(value, "files")
	files, err := leveldb.OpenFile(filesDir, opts)
	if err != nil {
		return nil, err
	}
	chunksDir := filepath.Join(value, "chunks")
	chunks, err := leveldb.OpenFile(chunksDir, opts)
	if err != nil {
		return nil, err
	}
	return &leveldbDriver{
		files:  files,
		chunks: chunks,
		dir:    value,
	}, nil
}

type leveldbIter struct {
	iter iterator.Iterator
}

func (i *leveldbIter) Next(id *string) bool {
	for i.iter.Next() {
		key := string(i.iter.Key())
		if strings.HasSuffix(key, ".meta") {
			continue
		}
		if id != nil {
			*id = key
		}
		return true
	}
	return false
}

func (i *leveldbIter) Err() error {
	return i.iter.Error()
}

func (i *leveldbIter) Close() error {
	i.iter.Release()
	return nil
}

func init() {
	driver.Register("leveldb", leveldbOpener)
}
<commit_msg>Configure the chunks db to use a bloom filter<commit_after>package leveldb

import (
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"path/filepath"
	"strings"

	"gnd.la/blobstore/driver"
	"gnd.la/config"
	"gnd.la/internal"
	"gnd.la/util/pathutil"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/filter"
	"github.com/syndtr/goleveldb/leveldb/iterator"
	"github.com/syndtr/goleveldb/leveldb/opt"
)

var (
	syncOptions       = &opt.WriteOptions{Sync: true}
	checkChunkOptions = &opt.ReadOptions{DontFillCache: true, Strict: opt.NoStrict}
)

type leveldbDriver struct {
	files  *leveldb.DB
	chunks *leveldb.DB
	dir    string
}

func (d *leveldbDriver) Create(id string) (driver.WFile, error) {
	return newWFile(d, id), nil
}

func (d *leveldbDriver) Open(id string) (driver.RFile, error) {
	value, err := d.files.Get(internal.StringToBytes(id), nil)
	if err != nil {
		if err == leveldb.ErrNotFound {
			return nil, fmt.Errorf("file %s not found", id)
		}
		return nil, err
	}
	count := int(littleEndian.Uint32(value))
	if count == 0 {
		// Data is inline
		return &rfile{chunks: [][]byte{value[4:]}}, nil
	}
	pos := 4
	chunks := make([][]byte, count)
	for ii := 0; ii < count; ii++ {
		size := int(littleEndian.Uint32(value[pos:]))
		pos += 4
		key := value[pos : pos+size]
		chunk, err := d.chunks.Get(key, nil)
		if err != nil {
			if err == leveldb.ErrNotFound {
				return nil, fmt.Errorf("chunk %s in file %s not found", hex.EncodeToString(key), id)
			}
			return nil, err
		}
		chunks[ii] = chunk
		pos += size
	}
	return &rfile{chunks: chunks}, nil
}

func (d *leveldbDriver) Remove(id string) error {
	return d.files.Delete([]byte(id), syncOptions)
}

func (d *leveldbDriver) Close() error {
	if err := d.files.Close(); err != nil {
		return err
	}
	if err := d.chunks.Close(); err != nil {
		return err
	}
	return nil
}

func (d *leveldbDriver) Iter() (driver.Iter, error) {
	iter := d.files.NewIterator(nil, nil)
	return &leveldbIter{iter: iter}, nil
}

func leveldbOpener(url *config.URL) (driver.Driver, error) {
	value := url.Value
	if !filepath.IsAbs(value) {
		value = pathutil.Relative(value)
	}
	opts := &opt.Options{}
	if url.Fragment["nocompress"] != "" {
		opts.Compression = opt.NoCompression
	}
	if url.Fragment["nocreate"] != "" {
		opts.ErrorIfMissing = true
	}
	filesDir := filepath.Join(value, "files")
	files, err := leveldb.OpenFile(filesDir, opts)
	if err != nil {
		return nil, err
	}
	copts := *opts
	copts.Filter = filter.NewBloomFilter(8 * sha1.Size)
	chunksDir := filepath.Join(value, "chunks")
	chunks, err := leveldb.OpenFile(chunksDir, &copts)
	if err != nil {
		return nil, err
	}
	return &leveldbDriver{
		files:  files,
		chunks: chunks,
		dir:    value,
	}, nil
}

type leveldbIter struct {
	iter iterator.Iterator
}

func (i *leveldbIter) Next(id *string) bool {
	for i.iter.Next() {
		key := string(i.iter.Key())
		if strings.HasSuffix(key, ".meta") {
			continue
		}
		if id != nil {
			*id = key
		}
		return true
	}
	return false
}

func (i *leveldbIter) Err() error {
	return i.iter.Error()
}

func (i *leveldbIter) Close() error {
	i.iter.Release()
	return nil
}

func init() {
	driver.Register("leveldb", leveldbOpener)
}
<|endoftext|>


<commit_before>// File: ./blockfreight/bft/leveldb/leveldb.go
// Summary: Application code for Blockfreight™ | The blockchain of global freight.
// License: MIT License
// Company: Blockfreight, Inc.
// Author: Julian Nunez, Neil Tran, Julian Smith & contributors
// Site: https://blockfreight.com
// Support: <support@blockfreight.com>

// Copyright 2017 Blockfreight, Inc.

// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

// =================================================================================================================================================
// =================================================================================================================================================
//
// BBBBBBBBBBBb     lll                                kkk             ffff                         iii                  hhh            ttt
// BBBB``````BBBB   lll                                kkk            fff                           ```                  hhh            ttt
// BBBB      BBBB   lll      oooooo        ccccccc     kkk    kkkk  fffffff  rrr  rrr    eeeee      iii     gggggg ggg   hhh  hhhhh   tttttttt
// BBBBBBBBBBBB     lll    ooo    oooo    ccc    ccc   kkk   kkk    fffffff  rrrrrrrr eee    eeee   iii   gggg   ggggg   hhhh   hhhh  tttttttt
// BBBBBBBBBBBBBB   lll   ooo      ooo   ccc           kkkkkkk        fff    rrrr    eeeeeeeeeeeee  iii  gggg      ggg   hhh     hhh    ttt
// BBBB       BBB   lll   ooo      ooo   ccc           kkkk kkkk      fff    rrr     eeeeeeeeeeeee  iii   ggg      ggg   hhh     hhh    ttt
// BBBB      BBBB   lll   oooo    oooo   cccc    ccc   kkk   kkkk     fff    rrr      eee      eee  iii    ggg    gggg   hhh     hhh    tttt    ....
// BBBBBBBBBBBBB    lll     oooooooo       ccccccc     kkk     kkkk   fff    rrr       eeeeeeeee    iii     gggggg ggg   hhh     hhh     ttttt  ....
//                                                                                ggg      ggg
//   Blockfreight™ | The blockchain of global freight.                                                      ggggggggg
//
// =================================================================================================================================================
// =================================================================================================================================================

// Package leveldb provides some useful functions to work with LevelDB.
// It has common database functions as OpenDB, CloseDB, Insert and Iterate.
package leveldb

import (
    // =======================
    // Golang Standard library
    // =======================
    "encoding/json" // Implements encoding and decoding of JSON as defined in RFC 4627.
    "errors"        // Implements functions to manipulate errors.
    "strconv"       // Implements conversions to and from string representations of basic data types.

    // ====================
    // Third-party packages
    // ====================
    "github.com/syndtr/goleveldb/leveldb"   // Implementation of the LevelDB key/value database in the Go programming language.

    // ======================
    // Blockfreight™ packages
    // ======================
    "github.com/blockfreight/blockfreight-alpha/blockfreight/bft/bf_tx" // Defines the Blockfreight™ Transaction (BF_TX) transaction standard and provides some useful functions to work with the BF_TX.
)

var db_path string = "bft-db"   //Folder name where is going to be the LevelDB

// OpenDB is a function that receives the path of the DB, creates or opens that DB and return ir with a possible error if that ocurred.
func OpenDB(db_path string) (db *leveldb.DB, err error) {
    db, err = leveldb.OpenFile(db_path, nil)
    return db, err

}

// CloseDB is a function that receives a DB pointer that closes the connection to DB.
func CloseDB(db *leveldb.DB) {
    db.Close()
}

// InsertBF_TX is a function that receives the key and value strings to insert a tuple in determined DB, the final parameter. As result, it returns a true or false bool.
func InsertBF_TX(key string, value string, db *leveldb.DB) error {
    return db.Put([]byte(key), []byte(value), nil)
}

// Total is a function that returns the total of BF_TX stored in the DB.
func Total() (int, error) {
    db, err := OpenDB(db_path)
    defer CloseDB(db)
    if err != nil {
        return 0, err
    }

    iter := db.NewIterator(nil, nil)
    n := 0
    for iter.Next() {
        n += 1
    }
    iter.Release()
    return n, nil
}

// RecordOnDB is a function that receives the content of the BF_RX JSON to insert it into the DB and return true or false according to the result.
func RecordOnDB( id int, json string) error { //TODO: Check the id
    db, err := OpenDB(db_path)
    defer CloseDB(db)
    if err != nil {
        return err
    }
    err = InsertBF_TX(strconv.Itoa(id), json, db)   //TODO: Check the id
    if err != nil {
        return err
    }
    return nil
}

// GetBfTx is a function that receives a bf_tx id, and returns the BF_TX if it exists.
func GetBfTx(id string) (bf_tx.BF_TX, error) {
    var bftx bf_tx.BF_TX
    db, err := OpenDB(db_path)
    defer CloseDB(db)
    if err != nil {
        return bftx, err
    }

    data, err := db.Get([]byte(id), nil)
    if err != nil {
        if(err.Error() == "leveldb: not found"){
            return bftx, errors.New("LevelDB Get function: BF_TX not found.")
        }
        return bftx, errors.New("LevelDB Get function: "+err.Error())
    }

    json.Unmarshal(data, &bftx)
    return bftx, nil
}

// =================================================
// Blockfreight™ | The blockchain of global freight.
// =================================================

// BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
// BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
// BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
// BBBBBBB                    BBBBBBBBBBBBBBBBBBB
// BBBBBBB                       BBBBBBBBBBBBBBBB
// BBBBBBB                        BBBBBBBBBBBBBBB
// BBBBBBB       BBBBBBBBB        BBBBBBBBBBBBBBB
// BBBBBBB       BBBBBBBBB        BBBBBBBBBBBBBBB
// BBBBBBB       BBBBBBB         BBBBBBBBBBBBBBBB
// BBBBBBB                     BBBBBBBBBBBBBBBBBB
// BBBBBBB                        BBBBBBBBBBBBBBB
// BBBBBBB       BBBBBBBBBB        BBBBBBBBBBBBBB
// BBBBBBB       BBBBBBBBBBB       BBBBBBBBBBBBBB
// BBBBBBB       BBBBBBBBBB        BBBBBBBBBBBBBB
// BBBBBBB       BBBBBBBBB        BBB       BBBBB
// BBBBBBB                       BBBB       BBBBB
// BBBBBBB                    BBBBBBB       BBBBB
// BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
// BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
// BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB

// ==================================================
// Blockfreight™ | The blockchain for global freight.
// ==================================================
<commit_msg>Create Verify function to match a content with BF_TX<commit_after>// File: ./blockfreight/bft/leveldb/leveldb.go
// Summary: Application code for Blockfreight™ | The blockchain of global freight.
// License: MIT License
// Company: Blockfreight, Inc.
// Author: Julian Nunez, Neil Tran, Julian Smith & contributors
// Site: https://blockfreight.com
// Support: <support@blockfreight.com>

// Copyright 2017 Blockfreight, Inc.

// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

// =================================================================================================================================================
// =================================================================================================================================================
//
// BBBBBBBBBBBb     lll                                kkk             ffff                         iii                  hhh            ttt
// BBBB``````BBBB   lll                                kkk            fff                           ```                  hhh            ttt
// BBBB      BBBB   lll      oooooo        ccccccc     kkk    kkkk  fffffff  rrr  rrr    eeeee      iii     gggggg ggg   hhh  hhhhh   tttttttt
// BBBBBBBBBBBB     lll    ooo    oooo    ccc    ccc   kkk   kkk    fffffff  rrrrrrrr eee    eeee   iii   gggg   ggggg   hhhh   hhhh  tttttttt
// BBBBBBBBBBBBBB   lll   ooo      ooo   ccc           kkkkkkk        fff    rrrr    eeeeeeeeeeeee  iii  gggg      ggg   hhh     hhh    ttt
// BBBB       BBB   lll   ooo      ooo   ccc           kkkk kkkk      fff    rrr     eeeeeeeeeeeee  iii   ggg      ggg   hhh     hhh    ttt
// BBBB      BBBB   lll   oooo    oooo   cccc    ccc   kkk   kkkk     fff    rrr      eee      eee  iii    ggg    gggg   hhh     hhh    tttt    ....
// BBBBBBBBBBBBB    lll     oooooooo       ccccccc     kkk     kkkk   fff    rrr       eeeeeeeee    iii     gggggg ggg   hhh     hhh     ttttt  ....
//                                                                                ggg      ggg
//   Blockfreight™ | The blockchain of global freight.                                                      ggggggggg
//
// =================================================================================================================================================
// =================================================================================================================================================

// Package leveldb provides some useful functions to work with LevelDB.
// It has common database functions as OpenDB, CloseDB, Insert and Iterate.
package leveldb

import (
    // =======================
    // Golang Standard library
    // =======================
    "encoding/json" // Implements encoding and decoding of JSON as defined in RFC 4627.
    "errors"        // Implements functions to manipulate errors.
    "strconv"       // Implements conversions to and from string representations of basic data types.

    // ====================
    // Third-party packages
    // ====================
    "github.com/syndtr/goleveldb/leveldb"   // Implementation of the LevelDB key/value database in the Go programming language.

    // ======================
    // Blockfreight™ packages
    // ======================
    "github.com/blockfreight/blockfreight-alpha/blockfreight/bft/bf_tx" // Defines the Blockfreight™ Transaction (BF_TX) transaction standard and provides some useful functions to work with the BF_TX.
)

var db_path string = "bft-db"   //Folder name where is going to be the LevelDB

// OpenDB is a function that receives the path of the DB, creates or opens that DB and return ir with a possible error if that ocurred.
func OpenDB(db_path string) (db *leveldb.DB, err error) {
    db, err = leveldb.OpenFile(db_path, nil)
    return db, err

}

// CloseDB is a function that receives a DB pointer that closes the connection to DB.
func CloseDB(db *leveldb.DB) {
    db.Close()
}

// InsertBF_TX is a function that receives the key and value strings to insert a tuple in determined DB, the final parameter. As result, it returns a true or false bool.
func InsertBF_TX(key string, value string, db *leveldb.DB) error {
    return db.Put([]byte(key), []byte(value), nil)
}

// Total is a function that returns the total of BF_TX stored in the DB.
func Total() (int, error) {
    db, err := OpenDB(db_path)
    defer CloseDB(db)
    if err != nil {
        return 0, err
    }

    iter := db.NewIterator(nil, nil)
    n := 0
    for iter.Next() {
        n += 1
    }
    iter.Release()
    return n, iter.Error()
}

// RecordOnDB is a function that receives the content of the BF_RX JSON to insert it into the DB and return true or false according to the result.
func RecordOnDB( id int, json string) error { //TODO: Check the id
    db, err := OpenDB(db_path)
    defer CloseDB(db)
    if err != nil {
        return err
    }
    err = InsertBF_TX(strconv.Itoa(id), json, db)   //TODO: Check the id
    if err != nil {
        return err
    }
    return nil
}

// GetBfTx is a function that receives a bf_tx id, and returns the BF_TX if it exists.
func GetBfTx(id string) (bf_tx.BF_TX, error) {
    var bftx bf_tx.BF_TX
    db, err := OpenDB(db_path)
    defer CloseDB(db)
    if err != nil {
        return bftx, err
    }

    data, err := db.Get([]byte(id), nil)
    if err != nil {
        if(err.Error() == "leveldb: not found"){
            return bftx, errors.New("LevelDB Get function: BF_TX not found.")
        }
        return bftx, errors.New("LevelDB Get function: "+err.Error())
    }

    json.Unmarshal(data, &bftx)
    return bftx, nil
}

// Verify is a function that receives a content and look for a BF_TX that has the same content.
func Verify(jcontent string) ([]byte, error){
    var bftx bf_tx.BF_TX
    db, err := OpenDB(db_path)
    defer CloseDB(db)
    if err != nil {
        return nil, err
    }

    iter := db.NewIterator(nil, nil)
    for iter.Next() {
        key := iter.Key()
        value := iter.Value()

        // Get a BF_TX by id
        json.Unmarshal(value, &bftx)

        // Reinitialize the BF_TX
        bftx = bf_tx.Reinitialize(bftx)

        // Get the BF_TX old_content in string format
        content, err := bf_tx.BF_TXContent(bftx)
        if err != nil {
            return nil, err
        }

        if jcontent == content {
            iter.Release()
            //strconv.Atoi(string(buf))
            return key, nil
        }
    }
    iter.Release()

    return nil, iter.Error()
}

// =================================================
// Blockfreight™ | The blockchain of global freight.
// =================================================

// BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
// BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
// BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
// BBBBBBB                    BBBBBBBBBBBBBBBBBBB
// BBBBBBB                       BBBBBBBBBBBBBBBB
// BBBBBBB                        BBBBBBBBBBBBBBB
// BBBBBBB       BBBBBBBBB        BBBBBBBBBBBBBBB
// BBBBBBB       BBBBBBBBB        BBBBBBBBBBBBBBB
// BBBBBBB       BBBBBBB         BBBBBBBBBBBBBBBB
// BBBBBBB                     BBBBBBBBBBBBBBBBBB
// BBBBBBB                        BBBBBBBBBBBBBBB
// BBBBBBB       BBBBBBBBBB        BBBBBBBBBBBBBB
// BBBBBBB       BBBBBBBBBBB       BBBBBBBBBBBBBB
// BBBBBBB       BBBBBBBBBB        BBBBBBBBBBBBBB
// BBBBBBB       BBBBBBBBB        BBB       BBBBB
// BBBBBBB                       BBBB       BBBBB
// BBBBBBB                    BBBBBBB       BBBBB
// BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
// BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
// BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB

// ==================================================
// Blockfreight™ | The blockchain for global freight.
// ==================================================
<|endoftext|>


<commit_before>package event

import (
	"fmt"
	"time"
)

// PrecisionTiming keeps min/max/avg information about a timer over a certain interval
type PrecisionTiming struct {
	Name  string
	Min   time.Duration
	Max   time.Duration
	Value time.Duration
	Count int64
}

// NewPrecisionTiming is a factory for a Timing event, setting the Count to 1 to prevent div_by_0 errors
func NewPrecisionTiming(k string, delta time.Duration) *PrecisionTiming {
	return &PrecisionTiming{Name: k, Min: delta, Max: delta, Value: delta, Count: 1}
}

// Update the event with metrics coming from a new one of the same type and with the same key
func (e *PrecisionTiming) Update(e2 Event) error {
	if e.Type() != e2.Type() {
		return fmt.Errorf("statsd event type conflict: %s vs %s ", e.String(), e2.String())
	}
	p := e2.Payload().(PrecisionTiming)
	e.Count += p.Count
	e.Value += p.Value
	e.Min = time.Duration(minInt64(int64(e.Min), int64(p.Min)))
	e.Max = time.Duration(maxInt64(int64(e.Max), int64(p.Min)))
	return nil
}

// Payload returns the aggregated value for this event
func (e PrecisionTiming) Payload() interface{} {
	return e
}

// Stats returns an array of StatsD events as they travel over UDP
func (e PrecisionTiming) Stats() []string {
	return []string{
		fmt.Sprintf("%s.count:%d|a", e.Name, e.Count),
		fmt.Sprintf("%s.avg:%.6f|ms", e.Name, float64(int64(e.Value)/e.Count)/1000000), // make sure e.Count != 0
		fmt.Sprintf("%s.min:%.6f|ms", e.Name, e.durationToMs(e.Min)),
		fmt.Sprintf("%s.max:%.6f|ms", e.Name, e.durationToMs(e.Max)),
	}
}

// durationToMs converts time.Duration into the corresponding value in milliseconds
func (e PrecisionTiming) durationToMs(x time.Duration) float64 {
	return float64(x) / float64(time.Millisecond)
}

// Key returns the name of this metric
func (e PrecisionTiming) Key() string {
	return e.Name
}

// SetKey sets the name of this metric
func (e *PrecisionTiming) SetKey(key string) {
	e.Name = key
}

// Type returns an integer identifier for this type of metric
func (e PrecisionTiming) Type() int {
	return EventPrecisionTiming
}

// TypeString returns a name for this type of metric
func (e PrecisionTiming) TypeString() string {
	return "PrecisionTiming"
}

// String returns a debug-friendly representation of this metric
func (e PrecisionTiming) String() string {
	return fmt.Sprintf("{Type: %s, Key: %s, Value: %+v}", e.TypeString(), e.Name, e.Payload())
}
<commit_msg>Update precisiontiming.go<commit_after>package event

import (
	"fmt"
	"time"
)

// PrecisionTiming keeps min/max/avg information about a timer over a certain interval
type PrecisionTiming struct {
	Name  string
	Min   time.Duration
	Max   time.Duration
	Value time.Duration
	Count int64
}

// NewPrecisionTiming is a factory for a Timing event, setting the Count to 1 to prevent div_by_0 errors
func NewPrecisionTiming(k string, delta time.Duration) *PrecisionTiming {
	return &PrecisionTiming{Name: k, Min: delta, Max: delta, Value: delta, Count: 1}
}

// Update the event with metrics coming from a new one of the same type and with the same key
func (e *PrecisionTiming) Update(e2 Event) error {
	if e.Type() != e2.Type() {
		return fmt.Errorf("statsd event type conflict: %s vs %s ", e.String(), e2.String())
	}
	p := e2.Payload().(PrecisionTiming)
	e.Count += p.Count
	e.Value += p.Value
	e.Min = time.Duration(minInt64(int64(e.Min), int64(p.Min)))
	e.Max = time.Duration(maxInt64(int64(e.Max), int64(p.Min)))
	return nil
}

// Payload returns the aggregated value for this event
func (e PrecisionTiming) Payload() interface{} {
	return e
}

// Stats returns an array of StatsD events as they travel over UDP
func (e PrecisionTiming) Stats() []string {
	return []string{
		fmt.Sprintf("%s.count:%d|a", e.Name, e.Count),
		fmt.Sprintf("%s.avg:%.6f|ms", e.Name, float64(int64(e.Value/time.Millisecond)/e.Count)), // make sure e.Count != 0
		fmt.Sprintf("%s.min:%.6f|ms", e.Name, e.durationToMs(e.Min)),
		fmt.Sprintf("%s.max:%.6f|ms", e.Name, e.durationToMs(e.Max)),
	}
}

// durationToMs converts time.Duration into the corresponding value in milliseconds
func (e PrecisionTiming) durationToMs(x time.Duration) float64 {
	return float64(x) / float64(time.Millisecond)
}

// Key returns the name of this metric
func (e PrecisionTiming) Key() string {
	return e.Name
}

// SetKey sets the name of this metric
func (e *PrecisionTiming) SetKey(key string) {
	e.Name = key
}

// Type returns an integer identifier for this type of metric
func (e PrecisionTiming) Type() int {
	return EventPrecisionTiming
}

// TypeString returns a name for this type of metric
func (e PrecisionTiming) TypeString() string {
	return "PrecisionTiming"
}

// String returns a debug-friendly representation of this metric
func (e PrecisionTiming) String() string {
	return fmt.Sprintf("{Type: %s, Key: %s, Value: %+v}", e.TypeString(), e.Name, e.Payload())
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"github.com/hashicorp/nomad/api"
	"github.com/posener/complete"
)

type OperatorSnapshotSaveCommand struct {
	Meta
}

func (c *OperatorSnapshotSaveCommand) Help() string {
	helpText := `
Usage: nomad operator snapshot save [options] <file>

  Retrieves an atomic, point-in-time snapshot of the state of the Nomad servers
  which includes jobs, nodes, allocations, periodic jobs, and ACLs.

  If ACLs are enabled, a management token must be supplied in order to perform
  snapshot operations.

  To create a snapshot from the leader server and save it to "backup.snap":

    $ nomad snapshot save backup.snap

  To create a potentially stale snapshot from any available server (useful if no
  leader is available):

    $ consul snapshot save -stale backup.snap

  This is useful for situations where a cluster is in a degraded state and no
  leader is available. To target a specific server for a snapshot, you can run
  the 'nomad operator snapshot save' command on that specific server.

General Options:

  ` + generalOptionsUsage() + `

Snapshot Save Options:

  -stale=[true|false]
    The -stale argument defaults to "false" which means the leader provides the
    result. If the cluster is in an outage state without a leader, you may need
    to set -stale to "true" to get the configuration from a non-leader server.
`
	return strings.TrimSpace(helpText)
}

func (c *OperatorSnapshotSaveCommand) AutocompleteFlags() complete.Flags {
	return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient),
		complete.Flags{
			"-stale": complete.PredictAnything,
		})
}

func (c *OperatorSnapshotSaveCommand) AutocompleteArgs() complete.Predictor {
	return complete.PredictNothing
}

func (c *OperatorSnapshotSaveCommand) Synopsis() string {
	return "Saves snapshot of Nomad server state"
}

func (c *OperatorSnapshotSaveCommand) Name() string { return "operator snapshot save" }

func (c *OperatorSnapshotSaveCommand) Run(args []string) int {
	var stale bool

	flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
	flags.Usage = func() { c.Ui.Output(c.Help()) }

	flags.BoolVar(&stale, "stale", false, "")
	if err := flags.Parse(args); err != nil {
		c.Ui.Error(fmt.Sprintf("Failed to parse args: %v", err))
		return 1
	}

	// Check for misuse
	// Check that we either got no filename or exactly one.
	args = flags.Args()
	if len(args) > 1 {
		c.Ui.Error("This command takes either no arguments or one: <filename>")
		c.Ui.Error(commandErrorText(c))
		return 1
	}

	now := time.Now()
	filename := fmt.Sprintf("nomad-state-%04d%02d%0d-%d.snap", now.Year(), now.Month(), now.Day(), now.Unix())

	if len(args) == 1 {
		filename = args[0]
	}

	if _, err := os.Lstat(filename); err == nil {
		c.Ui.Error(fmt.Sprintf("Destination file already exists: %q", filename))
		c.Ui.Error(commandErrorText(c))
		return 1
	} else if !os.IsNotExist(err) {
		c.Ui.Error(fmt.Sprintf("Unexpected failure checking %q: %v", filename, err))
		return 1
	}

	// Set up a client.
	client, err := c.Meta.Client()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
		return 1
	}

	tmpFile, err := os.Create(filename + ".tmp")
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Failed to create file: %v", err))
		return 1
	}

	// Fetch the current configuration.
	q := &api.QueryOptions{
		AllowStale: stale,
	}
	snapIn, err := client.Operator().Snapshot(q)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Failed to get snapshot file: %v", err))
		return 1
	}

	defer snapIn.Close()

	_, err = io.Copy(tmpFile, snapIn)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Filed to download snapshot file: %v", err))
		return 1
	}

	err = os.Rename(tmpFile.Name(), filename)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Filed to finalize snapshot file: %v", err))
		return 1
	}

	c.Ui.Output(fmt.Sprintf("State file written to %v", filename))
	return 0
}
<commit_msg>Apply suggestions from code review<commit_after>package command

import (
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"github.com/hashicorp/nomad/api"
	"github.com/posener/complete"
)

type OperatorSnapshotSaveCommand struct {
	Meta
}

func (c *OperatorSnapshotSaveCommand) Help() string {
	helpText := `
Usage: nomad operator snapshot save [options] <file>

  Retrieves an atomic, point-in-time snapshot of the state of the Nomad servers
  which includes jobs, nodes, allocations, periodic jobs, and ACLs.

  If ACLs are enabled, a management token must be supplied in order to perform
  snapshot operations.

  To create a snapshot from the leader server and save it to "backup.snap":

    $ nomad snapshot save backup.snap

  To create a potentially stale snapshot from any available server (useful if no
  leader is available):

    $ nomad snapshot save -stale backup.snap

  This is useful for situations where a cluster is in a degraded state and no
  leader is available. To target a specific server for a snapshot, you can run
  the 'nomad operator snapshot save' command on that specific server.

General Options:

  ` + generalOptionsUsage() + `

Snapshot Save Options:

  -stale=[true|false]
    The -stale argument defaults to "false" which means the leader provides the
    result. If the cluster is in an outage state without a leader, you may need
    to set -stale to "true" to get the configuration from a non-leader server.
`
	return strings.TrimSpace(helpText)
}

func (c *OperatorSnapshotSaveCommand) AutocompleteFlags() complete.Flags {
	return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient),
		complete.Flags{
			"-stale": complete.PredictAnything,
		})
}

func (c *OperatorSnapshotSaveCommand) AutocompleteArgs() complete.Predictor {
	return complete.PredictNothing
}

func (c *OperatorSnapshotSaveCommand) Synopsis() string {
	return "Saves snapshot of Nomad server state"
}

func (c *OperatorSnapshotSaveCommand) Name() string { return "operator snapshot save" }

func (c *OperatorSnapshotSaveCommand) Run(args []string) int {
	var stale bool

	flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
	flags.Usage = func() { c.Ui.Output(c.Help()) }

	flags.BoolVar(&stale, "stale", false, "")
	if err := flags.Parse(args); err != nil {
		c.Ui.Error(fmt.Sprintf("Failed to parse args: %v", err))
		return 1
	}

	// Check for misuse
	// Check that we either got no filename or exactly one.
	args = flags.Args()
	if len(args) > 1 {
		c.Ui.Error("This command takes either no arguments or one: <filename>")
		c.Ui.Error(commandErrorText(c))
		return 1
	}

	now := time.Now()
	filename := fmt.Sprintf("nomad-state-%04d%02d%0d-%d.snap", now.Year(), now.Month(), now.Day(), now.Unix())

	if len(args) == 1 {
		filename = args[0]
	}

	if _, err := os.Lstat(filename); err == nil {
		c.Ui.Error(fmt.Sprintf("Destination file already exists: %q", filename))
		c.Ui.Error(commandErrorText(c))
		return 1
	} else if !os.IsNotExist(err) {
		c.Ui.Error(fmt.Sprintf("Unexpected failure checking %q: %v", filename, err))
		return 1
	}

	// Set up a client.
	client, err := c.Meta.Client()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
		return 1
	}

	tmpFile, err := os.Create(filename + ".tmp")
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Failed to create file: %v", err))
		return 1
	}

	// Fetch the current configuration.
	q := &api.QueryOptions{
		AllowStale: stale,
	}
	snapIn, err := client.Operator().Snapshot(q)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Failed to get snapshot file: %v", err))
		return 1
	}

	defer snapIn.Close()

	_, err = io.Copy(tmpFile, snapIn)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Filed to download snapshot file: %v", err))
		return 1
	}

	err = os.Rename(tmpFile.Name(), filename)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Filed to finalize snapshot file: %v", err))
		return 1
	}

	c.Ui.Output(fmt.Sprintf("State file written to %v", filename))
	return 0
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"

	"../../goble"
)

func main() {
	verbose := flag.Bool("verbose", false, "dump all events")
	dups := flag.Bool("allow-duplicates", false, "allow duplicates when scanning")
	flag.Parse()

	if len(flag.Args()) != 1 {
		fmt.Println("usage: explorer [options] peripheral-uuid")
		return
	}

	peripheralUuid := flag.Args()[0]

	var done chan bool

	ble := goble.New()
	ble.SetVerbose(*verbose)

	if *verbose {
		ble.On(goble.ALL, func(ev goble.Event) {
			log.Println("Event", ev)
		})
	}

	ble.On("stateChange", func(ev goble.Event) {
		if ev.State == "poweredOn" {
			ble.StartScanning(nil, *dups)
		} else {
			ble.StopScanning()
		}
	})

	ble.On("discover", func(ev goble.Event) {
		if peripheralUuid == ev.DeviceUUID.String() {
			ble.StopScanning()

			fmt.Println()
			fmt.Println("peripheral with UUID", ev.DeviceUUID, "found")

			advertisement := ev.Peripheral.Advertisement

			localName := advertisement.LocalName
			txPowerLevel := advertisement.TxPowerLevel
			manufacturerData := advertisement.ManufacturerData
			serviceData := advertisement.ServiceData
			//serviceUuids := advertisement.ServiceUuids

			if len(localName) > 0 {
				fmt.Println("  Local Name        =", localName)
			}

			if txPowerLevel != 0 {
				fmt.Println("  TX Power Level    =", txPowerLevel)
			}

			if len(manufacturerData) > 0 {
				fmt.Println("  Manufacturer Data =", manufacturerData)
			}

			if len(serviceData) > 0 {
				fmt.Println("  Service Data      =", serviceData)
			}

			fmt.Println()
			//explore(peripheral)

			done <- true
		}
	})

	if *verbose {
		log.Println("Init...")
	}

	ble.Init()
	_ = <-done
}
<commit_msg>start of "peripheral explorer" function (connect, discover services, disconnect)<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"../../goble"
)

func explore(ble *goble.BLE, peripheral *goble.Peripheral) {

	ble.On("connect", func(ev goble.Event) (done bool) {
		log.Println("connected", ev)
		ble.DiscoverServices(ev.DeviceUUID, nil)
		return
	})

	ble.On("disconnect", func(ev goble.Event) (done bool) {
		log.Println("disconnected", ev)
		os.Exit(0)
		return true
	})

	ble.On("servicesDiscover", func(ev goble.Event) (done bool) {
		log.Println("services", ev)
		return
	})

	fmt.Println("services and characteristics:")
	ble.Connect(peripheral.Uuid)

	/*

	     peripheral.on('disconnect', function() {
	       process.exit(0);
	     });

	     peripheral.connect(function(error) {
	       peripheral.discoverServices([], function(error, services) {
	         var serviceIndex = 0;

	         async.whilst(
	           function () {
	             return (serviceIndex < services.length);
	           },
	           function(callback) {
	             var service = services[serviceIndex];
	             var serviceInfo = service.uuid;

	             if (service.name) {
	               serviceInfo += ' (' + service.name + ')';
	             }
	             console.log(serviceInfo);

	             service.discoverCharacteristics([], function(error, characteristics) {
	               var characteristicIndex = 0;

	               async.whilst(
	                 function () {
	                   return (characteristicIndex < characteristics.length);
	                 },
	                 function(callback) {
	                   var characteristic = characteristics[characteristicIndex];
	                   var characteristicInfo = '  ' + characteristic.uuid;

	                   if (characteristic.name) {
	                     characteristicInfo += ' (' + characteristic.name + ')';
	                   }

	                   async.series([
	                     function(callback) {
	                       characteristic.discoverDescriptors(function(error, descriptors) {
	                         async.detect(
	                           descriptors,
	                           function(descriptor, callback) {
	                             return callback(descriptor.uuid === '2901")
	                           },
	                           function(userDescriptionDescriptor){
	                             if (userDescriptionDescriptor) {
	                               userDescriptionDescriptor.readValue(function(error, data) {
	                                 characteristicInfo += ' (' + data.toString() + ')';
	                                 callback();
	                               });
	                             } else {
	                               callback();
	                             }
	                           }
	                         );
	                       });
	                     },
	                     function(callback) {
	                           characteristicInfo += '\n    properties  ' + characteristic.properties.join(', ")

	                       if (characteristic.properties.indexOf('read') !== -1) {
	                         characteristic.read(function(error, data) {
	                           if (data) {
	                             var string = data.toString('ascii")

	                             characteristicInfo += '\n    value       ' + data.toString('hex') + ' | \'' + string + '\'';
	                           }
	                           callback();
	                         });
	                       } else {
	                         callback();
	                       }
	                     },
	                     function() {
	                       console.log(characteristicInfo);
	                       characteristicIndex++;
	                       callback();
	                     }
	                   ]);
	                 },
	                 function(error) {
	                   serviceIndex++;
	                   callback();
	                 }
	               );
	             });
	           },
	           function (err) {
	             peripheral.disconnect();
	           }
	         );
	       });
	     });
	   }
	*/
}

func main() {
	verbose := flag.Bool("verbose", false, "dump all events")
	dups := flag.Bool("allow-duplicates", false, "allow duplicates when scanning")
	flag.Parse()

	if len(flag.Args()) != 1 {
		fmt.Println("usage:", os.Args[0], "[options] peripheral-uuid")
		os.Exit(1)
	}

	peripheralUuid := flag.Args()[0]

	var done chan bool

	ble := goble.New()
	ble.SetVerbose(*verbose)

	if *verbose {
		ble.On(goble.ALL, func(ev goble.Event) (done bool) {
			log.Println("Event", ev)
			return
		})
	}

	ble.On("stateChange", func(ev goble.Event) (done bool) {
		if ev.State == "poweredOn" {
			ble.StartScanning(nil, *dups)
		} else {
			ble.StopScanning()
			done = true
		}

		return
	})

	ble.On("discover", func(ev goble.Event) (done bool) {
		if peripheralUuid == ev.DeviceUUID.String() {
			ble.StopScanning()

			fmt.Println()
			fmt.Println("peripheral with UUID", ev.DeviceUUID, "found")

			advertisement := ev.Peripheral.Advertisement

			localName := advertisement.LocalName
			txPowerLevel := advertisement.TxPowerLevel
			manufacturerData := advertisement.ManufacturerData
			serviceData := advertisement.ServiceData
			//serviceUuids := advertisement.ServiceUuids

			if len(localName) > 0 {
				fmt.Println("  Local Name        =", localName)
			}

			if txPowerLevel != 0 {
				fmt.Println("  TX Power Level    =", txPowerLevel)
			}

			if len(manufacturerData) > 0 {
				fmt.Println("  Manufacturer Data =", manufacturerData)
			}

			if len(serviceData) > 0 {
				fmt.Println("  Service Data      =", serviceData)
			}

			fmt.Println()
			explore(ble, &ev.Peripheral)
		}

		return
	})

	if *verbose {
		log.Println("Init...")
	}

	ble.Init()

	fmt.Println("waiting...")
	<-done

	fmt.Println("goodbye!")
	os.Exit(0)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/salviati/go-qt5/qt5"
)

var exit = make(chan bool)

func main() {
	qt5.Main(func() {
		go ui_main()
		qt5.Run()
		exit <- true
	})
}

func ui_main() {
	w := loadUIFile("main.ui")
	name := w.Name()
	fmt.Println(name)

	w.SetWindowTitle(qt5.Version())
	w.SetSizev(300, 200)
	defer w.Close()
	w.Show()
	<-exit
}

func loadUIFile(designFile string) *qt5.Widget {
	file := qt5.NewFileWithFilename(designFile)
	file.Open(qt5.OpenModeReadOnly)
	uiLoader := qt5.NewUILoader()
	return uiLoader.Load(file)
}
<commit_msg>use QPainter<commit_after>package main

import (
	"fmt"
	"image/color"

	"github.com/salviati/go-qt5/qt5"
)

func main() {
	qt5.Main(ui_main)
}

func ui_main() {
	w := qt5.NewWidget()
	defer w.Close()

	// canvas := qt5.NewWidget()

	w.OnPaintEvent(func(e *qt5.PaintEvent) {
		paint := qt5.NewPainter()
		defer paint.Close()
		paint.Begin(w)
		pen := qt5.NewPen()
		pen.SetColor(color.RGBA{255, 128, 0, 0})
		pen.SetWidth(2)
		fmt.Println(pen, pen.Color(), pen.Width())
		paint.SetPen(pen)
		brush := qt5.NewBrush()
		brush.SetStyle(qt5.SolidPattern)
		brush.SetColor(color.RGBA{128, 128, 0, 255})
		paint.SetBrush(brush)
		paint.DrawRect(qt5.Rect{10, 10, 100, 100})
	})

	w.SetSize(qt5.Size{400, 400})

	w.Show()

	qt5.Run()
}
<|endoftext|>


<commit_before>package azure

import (
	"encoding/base64"
	"io"
	"io/ioutil"
	"math/rand"
	"strings"
	"time"

	"github.com/cenkalti/backoff"

	"github.com/Azure/go-autorest/autorest"

	"github.com/Azure/azure-sdk-for-go/storage"
	"github.com/Azure/go-autorest/autorest/azure"

	"net/http"

	"fmt"

	bitsgo "github.com/cloudfoundry-incubator/bits-service"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/validate"
	"github.com/cloudfoundry-incubator/bits-service/config"
	"github.com/cloudfoundry-incubator/bits-service/logger"
	"github.com/pkg/errors"
)

type Blobstore struct {
	containerName  string
	client         storage.BlobStorageClient
	putBlockSize   int64
	maxListResults uint
	metricsService bitsgo.MetricsService
}

func NewBlobstore(config config.AzureBlobstoreConfig, metricsService bitsgo.MetricsService) *Blobstore {
	return NewBlobstoreWithDetails(config, 50<<20, 5000, metricsService)
}

// NetworkErrorRetryingSender is a replacement for the storage.DefaultSender.
// storage.DefaultSender does not retry on network errors which is rarely what we want in
// a production system.
type NetworkErrorRetryingSender struct{}

var retryableStatusCodes = []int{
	http.StatusRequestTimeout,      // 408
	http.StatusInternalServerError, // 500
	http.StatusBadGateway,          // 502
	http.StatusServiceUnavailable,  // 503
	http.StatusGatewayTimeout,      // 504
}

// Send is mostly emulating storage.DefaultSender.Send, so most code was copied.
// But we use the backoff library for convenience and retry on errors returned from
// HTTPClient.Do.
func (sender *NetworkErrorRetryingSender) Send(c *storage.Client, req *http.Request) (*http.Response, error) {
	rr := autorest.NewRetriableRequest(req)
	var resp *http.Response

	err := backoff.Retry(func() error {
		err := rr.Prepare()
		if err != nil {
			return backoff.Permanent(err)
		}
		resp, err = c.HTTPClient.Do(rr.Request())
		// We deliberately mark errors *not* as permanent, because an error
		// here means network connectivity or similar. This is different to the
		// storage.DefaultSender which stops on any error.
		if err != nil {
			return err
		}
		if !autorest.ResponseHasStatusCode(resp, retryableStatusCodes...) {
			return backoff.Permanent(err)
		}
		io.Copy(ioutil.Discard, resp.Body)
		resp.Body.Close()
		return nil
	}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 5))

	return resp, err
}

func NewBlobstoreWithDetails(config config.AzureBlobstoreConfig, putBlockSize int64, maxListResults uint, metricsService bitsgo.MetricsService) *Blobstore {
	validate.NotEmpty(config.AccountKey)
	validate.NotEmpty(config.AccountName)
	validate.NotEmpty(config.ContainerName)
	validate.NotEmpty(config.EnvironmentName())
	if metricsService == nil {
		panic("metricsService must not be nil")
	}

	environment, e := azure.EnvironmentFromName(config.EnvironmentName())
	if e != nil {
		logger.Log.Fatalw("Could not get Azure Environment from Name", "error", e, "environment", config.EnvironmentName())
	}
	client, e := storage.NewBasicClientOnSovereignCloud(config.AccountName, config.AccountKey, environment)
	if e != nil {
		logger.Log.Fatalw("Could not instantiate Azure Basic Client", "error", e)
	}
	client.Sender = &NetworkErrorRetryingSender{}
	return &Blobstore{
		client:         client.GetBlobService(),
		containerName:  config.ContainerName,
		putBlockSize:   putBlockSize,
		maxListResults: maxListResults,
		metricsService: metricsService,
	}
}

func (blobstore *Blobstore) Exists(path string) (bool, error) {
	exists, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).Exists()
	if e != nil {
		return false, errors.Wrapf(e, "Failed to check for %v/%v", blobstore.containerName, path)
	}
	return exists, nil
}

func (blobstore *Blobstore) Get(path string) (body io.ReadCloser, err error) {
	logger.Log.Debugw("Get", "bucket", blobstore.containerName, "path", path)

	reader, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).Get(nil)
	if e != nil {
		return nil, blobstore.handleError(e, "Path %v", path)
	}
	return reader, nil
}

func (blobstore *Blobstore) GetOrRedirect(path string) (body io.ReadCloser, redirectLocation string, err error) {
	signedUrl, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).GetSASURI(storage.BlobSASOptions{
		BlobServiceSASPermissions: storage.BlobServiceSASPermissions{Read: true},
		SASOptions:                storage.SASOptions{Expiry: time.Now().Add(time.Hour)},
	})
	return nil, signedUrl, e
}

func (blobstore *Blobstore) Put(path string, src io.ReadSeeker) error {
	putRequestID := rand.Int63()
	l := logger.Log.With("put-request-id", putRequestID)
	l.Debugw("Put", "bucket", blobstore.containerName, "path", path)

	blob := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path)

	e := blob.CreateBlockBlob(nil)
	if e != nil {
		return errors.Wrapf(e, "create block blob failed. container: %v, path: %v, put-request-id: %v", blobstore.containerName, path, putRequestID)
	}

	uncommittedBlocksList := make([]storage.Block, 0)
	eof := false
	for i := 0; !eof; i++ {
		// using information from https://docs.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs
		if i >= 50000 {
			return errors.Errorf("block blob cannot have more than 50,000 blocks. path: %v, put-request-id: %v", path, putRequestID)
		}
		block := storage.Block{
			ID:     base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%05d", i))),
			Status: storage.BlockStatusUncommitted,
		}
		data := make([]byte, blobstore.putBlockSize)
		numBytesRead, e := src.Read(data)
		if e != nil {
			if e.Error() == "EOF" {
				eof = true
			} else {
				return errors.Wrapf(e, "put block failed. path: %v, put-request-id: %v", path, putRequestID)
			}
		}
		if numBytesRead == 0 {
			l.Debugw("Empty read", "block-index", i, "block-id", block.ID, "is-eof", eof)
			continue
		}
		l.Debugw("PutBlock", "block-index", i, "block-id", block.ID, "block-size", numBytesRead, "is-eof", eof)
		e = backoff.RetryNotify(func() error {
			return blob.PutBlock(block.ID, data[:numBytesRead], nil)
		}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 2), func(error, time.Duration) {
			l.Infow("Retry PutBlock", "block-index", i, "block-id", block.ID, "block-size", numBytesRead, "is-eof", eof)
			blobstore.metricsService.SendCounterMetric("put-block-retry", 1)
		})
		if e != nil {
			return errors.Wrapf(e, "put block failed. path: %v, put-request-id: %v", path, putRequestID)
		}
		uncommittedBlocksList = append(uncommittedBlocksList, block)
	}
	l.Debugw("PutBlockList", "uncommitted-block-list", uncommittedBlocksList)
	e = backoff.RetryNotify(func() error {
		return blob.PutBlockList(uncommittedBlocksList, nil)
	}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 2), func(error, time.Duration) {
		l.Infow("Retry PutBlockList", "uncommitted-block-list", uncommittedBlocksList)
		blobstore.metricsService.SendCounterMetric("put-block-list-retry", 1)
	})
	if e != nil {
		return errors.Wrapf(e, "put block list failed. path: %v, put-request-id: %v", path, putRequestID)
	}

	return nil
}

func (blobstore *Blobstore) Copy(src, dest string) error {
	logger.Log.Debugw("Copy in Azure", "container", blobstore.containerName, "src", src, "dest", dest)
	e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(dest).Copy(
		blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(src).GetURL(), nil)

	if e != nil {
		return blobstore.handleError(e, "Error while trying to copy src %v to dest %v in bucket %v", src, dest, blobstore.containerName)
	}
	return nil
}

func (blobstore *Blobstore) Delete(path string) error {
	deleted, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).DeleteIfExists(nil)
	if e != nil {
		return errors.Wrapf(e, "Path %v", path)
	}
	if !deleted {
		return bitsgo.NewNotFoundError()
	}
	return nil
}

func (blobstore *Blobstore) DeleteDir(prefix string) error {
	deletionErrs := []error{}
	marker := ""
	for {
		response, e := blobstore.client.GetContainerReference(blobstore.containerName).ListBlobs(storage.ListBlobsParameters{
			Prefix:     prefix,
			MaxResults: blobstore.maxListResults,
			Marker:     marker,
		})
		if e != nil {
			return errors.Wrapf(e, "Prefix %v", prefix)
		}
		for _, blob := range response.Blobs {
			e = blobstore.Delete(blob.Name)
			if e != nil {
				if _, isNotFoundError := e.(*bitsgo.NotFoundError); !isNotFoundError {
					deletionErrs = append(deletionErrs, e)
				}
			}
		}
		if response.NextMarker == "" {
			break
		}
		marker = response.NextMarker
	}

	if len(deletionErrs) != 0 {
		return errors.Errorf("Prefix %v, errors from deleting: %v", prefix, deletionErrs)
	}
	return nil
}

func (blobstore *Blobstore) Sign(resource string, method string, expirationTime time.Time) (signedURL string) {
	var e error
	switch strings.ToLower(method) {
	case "put":
		signedURL, e = blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(resource).GetSASURI(storage.BlobSASOptions{
			BlobServiceSASPermissions: storage.BlobServiceSASPermissions{Write: true, Create: true},
			SASOptions:                storage.SASOptions{Expiry: expirationTime},
		})
	case "get":
		signedURL, e = blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(resource).GetSASURI(storage.BlobSASOptions{
			BlobServiceSASPermissions: storage.BlobServiceSASPermissions{Read: true},
			SASOptions:                storage.SASOptions{Expiry: expirationTime},
		})
	default:
		panic("The only supported methods are 'put' and 'get'")
	}
	if e != nil {
		panic(e)
	}
	return
}

func (blobstore *Blobstore) handleError(e error, context string, args ...interface{}) error {
	if azse, ok := e.(storage.AzureStorageServiceError); ok && azse.StatusCode == http.StatusNotFound {
		exists, e := blobstore.client.GetContainerReference(blobstore.containerName).Exists()
		if e != nil {
			return errors.Wrapf(e, context, args...)
		}
		if !exists {
			return errors.Errorf("Container does not exist '%v", blobstore.containerName)
		}
		return bitsgo.NewNotFoundError()
	}
	return errors.Wrapf(e, context, args...)
}
<commit_msg>Use improved strategy when uploading blobs in Azure<commit_after>package azure

import (
	"encoding/base64"
	"io"
	"io/ioutil"
	"math/rand"
	"strings"
	"time"

	"github.com/cenkalti/backoff"

	"github.com/Azure/go-autorest/autorest"

	"github.com/Azure/azure-sdk-for-go/storage"
	"github.com/Azure/go-autorest/autorest/azure"

	"net/http"

	"fmt"

	bitsgo "github.com/cloudfoundry-incubator/bits-service"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/validate"
	"github.com/cloudfoundry-incubator/bits-service/config"
	"github.com/cloudfoundry-incubator/bits-service/logger"
	"github.com/pkg/errors"
)

type Blobstore struct {
	containerName  string
	client         storage.BlobStorageClient
	putBlockSize   int64
	maxListResults uint
	metricsService bitsgo.MetricsService
}

func NewBlobstore(config config.AzureBlobstoreConfig, metricsService bitsgo.MetricsService) *Blobstore {
	return NewBlobstoreWithDetails(config, 50<<20, 5000, metricsService)
}

// NetworkErrorRetryingSender is a replacement for the storage.DefaultSender.
// storage.DefaultSender does not retry on network errors which is rarely what we want in
// a production system.
type NetworkErrorRetryingSender struct{}

var retryableStatusCodes = []int{
	http.StatusRequestTimeout,      // 408
	http.StatusInternalServerError, // 500
	http.StatusBadGateway,          // 502
	http.StatusServiceUnavailable,  // 503
	http.StatusGatewayTimeout,      // 504
}

// Send is mostly emulating storage.DefaultSender.Send, so most code was copied.
// But we use the backoff library for convenience and retry on errors returned from
// HTTPClient.Do.
func (sender *NetworkErrorRetryingSender) Send(c *storage.Client, req *http.Request) (*http.Response, error) {
	rr := autorest.NewRetriableRequest(req)
	var resp *http.Response

	err := backoff.Retry(func() error {
		err := rr.Prepare()
		if err != nil {
			return backoff.Permanent(err)
		}
		resp, err = c.HTTPClient.Do(rr.Request())
		// We deliberately mark errors *not* as permanent, because an error
		// here means network connectivity or similar. This is different to the
		// storage.DefaultSender which stops on any error.
		if err != nil {
			return err
		}
		if !autorest.ResponseHasStatusCode(resp, retryableStatusCodes...) {
			return backoff.Permanent(err)
		}
		io.Copy(ioutil.Discard, resp.Body)
		resp.Body.Close()
		return nil
	}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 5))

	return resp, err
}

func NewBlobstoreWithDetails(config config.AzureBlobstoreConfig, putBlockSize int64, maxListResults uint, metricsService bitsgo.MetricsService) *Blobstore {
	validate.NotEmpty(config.AccountKey)
	validate.NotEmpty(config.AccountName)
	validate.NotEmpty(config.ContainerName)
	validate.NotEmpty(config.EnvironmentName())
	if metricsService == nil {
		panic("metricsService must not be nil")
	}

	environment, e := azure.EnvironmentFromName(config.EnvironmentName())
	if e != nil {
		logger.Log.Fatalw("Could not get Azure Environment from Name", "error", e, "environment", config.EnvironmentName())
	}
	client, e := storage.NewBasicClientOnSovereignCloud(config.AccountName, config.AccountKey, environment)
	if e != nil {
		logger.Log.Fatalw("Could not instantiate Azure Basic Client", "error", e)
	}
	client.Sender = &NetworkErrorRetryingSender{}
	return &Blobstore{
		client:         client.GetBlobService(),
		containerName:  config.ContainerName,
		putBlockSize:   putBlockSize,
		maxListResults: maxListResults,
		metricsService: metricsService,
	}
}

func (blobstore *Blobstore) Exists(path string) (bool, error) {
	exists, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).Exists()
	if e != nil {
		return false, errors.Wrapf(e, "Failed to check for %v/%v", blobstore.containerName, path)
	}
	return exists, nil
}

func (blobstore *Blobstore) Get(path string) (body io.ReadCloser, err error) {
	logger.Log.Debugw("Get", "bucket", blobstore.containerName, "path", path)

	reader, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).Get(nil)
	if e != nil {
		return nil, blobstore.handleError(e, "Path %v", path)
	}
	return reader, nil
}

func (blobstore *Blobstore) GetOrRedirect(path string) (body io.ReadCloser, redirectLocation string, err error) {
	signedUrl, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).GetSASURI(storage.BlobSASOptions{
		BlobServiceSASPermissions: storage.BlobServiceSASPermissions{Read: true},
		SASOptions:                storage.SASOptions{Expiry: time.Now().Add(time.Hour)},
	})
	return nil, signedUrl, e
}

func (blobstore *Blobstore) Put(path string, src io.ReadSeeker) error {
	putRequestID := rand.Int63()
	l := logger.Log.With("put-request-id", putRequestID)
	l.Debugw("Put", "bucket", blobstore.containerName, "path", path)

	pathWithRequestIDSuffix := fmt.Sprintf("%v_%v", path, putRequestID)
	blob := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(pathWithRequestIDSuffix)

	e := blob.CreateBlockBlob(nil)
	if e != nil {
		return errors.Wrapf(e, "create block blob failed. container: %v, path: %v, put-request-id: %v", blobstore.containerName, pathWithRequestIDSuffix, putRequestID)
	}

	uncommittedBlocksList := make([]storage.Block, 0)
	eof := false
	for i := 0; !eof; i++ {
		// using information from https://docs.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs
		if i >= 50000 {
			return errors.Errorf("block blob cannot have more than 50,000 blocks. path: %v, put-request-id: %v", pathWithRequestIDSuffix, putRequestID)
		}
		block := storage.Block{
			ID:     base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%05d", i))),
			Status: storage.BlockStatusUncommitted,
		}
		data := make([]byte, blobstore.putBlockSize)
		numBytesRead, e := src.Read(data)
		if e != nil {
			if e.Error() == "EOF" {
				eof = true
			} else {
				return errors.Wrapf(e, "put block failed. path: %v, put-request-id: %v", pathWithRequestIDSuffix, putRequestID)
			}
		}
		if numBytesRead == 0 {
			l.Debugw("Empty read", "block-index", i, "block-id", block.ID, "is-eof", eof)
			continue
		}
		l.Debugw("PutBlock", "block-index", i, "block-id", block.ID, "block-size", numBytesRead, "is-eof", eof)
		e = backoff.RetryNotify(func() error {
			return blob.PutBlock(block.ID, data[:numBytesRead], nil)
		}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 2), func(error, time.Duration) {
			l.Infow("Retry PutBlock", "block-index", i, "block-id", block.ID, "block-size", numBytesRead, "is-eof", eof)
			blobstore.metricsService.SendCounterMetric("put-block-retry", 1)
		})
		if e != nil {
			return errors.Wrapf(e, "put block failed. path: %v, put-request-id: %v", pathWithRequestIDSuffix, putRequestID)
		}
		uncommittedBlocksList = append(uncommittedBlocksList, block)
	}
	l.Debugw("PutBlockList", "uncommitted-block-list", uncommittedBlocksList)
	e = backoff.RetryNotify(func() error {
		return blob.PutBlockList(uncommittedBlocksList, nil)
	}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 2), func(error, time.Duration) {
		l.Infow("Retry PutBlockList", "uncommitted-block-list", uncommittedBlocksList)
		blobstore.metricsService.SendCounterMetric("put-block-list-retry", 1)
	})
	if e != nil {
		return errors.Wrapf(e, "put block list failed. path: %v, put-request-id: %v", pathWithRequestIDSuffix, putRequestID)
	}

	e = backoff.RetryNotify(func() error {
		return blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).Copy(
			blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(pathWithRequestIDSuffix).GetURL(), nil)
	}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 100), func(error, time.Duration) {
		l.Infow("Retry Copy", "src", pathWithRequestIDSuffix, "dest", path)
		blobstore.metricsService.SendCounterMetric("copy-retry", 1)
	})
	if e != nil {
		return blobstore.handleError(e, "Error while trying to copy src %v to dest %v in bucket %v", pathWithRequestIDSuffix, path, blobstore.containerName)
	}

	e = backoff.RetryNotify(func() error {
		_, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(pathWithRequestIDSuffix).DeleteIfExists(nil)
		return e
	}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 2), func(error, time.Duration) {
		l.Infow("Retry DeleteIfExists", "path", pathWithRequestIDSuffix)
		blobstore.metricsService.SendCounterMetric("delete-retry", 1)
	})
	if e != nil {
		return blobstore.handleError(e, "Error while trying to delete path %v in bucket %v", pathWithRequestIDSuffix, blobstore.containerName)
	}

	return nil
}

func (blobstore *Blobstore) Copy(src, dest string) error {
	logger.Log.Debugw("Copy in Azure", "container", blobstore.containerName, "src", src, "dest", dest)
	e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(dest).Copy(
		blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(src).GetURL(), nil)

	if e != nil {
		return blobstore.handleError(e, "Error while trying to copy src %v to dest %v in bucket %v", src, dest, blobstore.containerName)
	}
	return nil
}

func (blobstore *Blobstore) Delete(path string) error {
	deleted, e := blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(path).DeleteIfExists(nil)
	if e != nil {
		return errors.Wrapf(e, "Path %v", path)
	}
	if !deleted {
		return bitsgo.NewNotFoundError()
	}
	return nil
}

func (blobstore *Blobstore) DeleteDir(prefix string) error {
	deletionErrs := []error{}
	marker := ""
	for {
		response, e := blobstore.client.GetContainerReference(blobstore.containerName).ListBlobs(storage.ListBlobsParameters{
			Prefix:     prefix,
			MaxResults: blobstore.maxListResults,
			Marker:     marker,
		})
		if e != nil {
			return errors.Wrapf(e, "Prefix %v", prefix)
		}
		for _, blob := range response.Blobs {
			e = blobstore.Delete(blob.Name)
			if e != nil {
				if _, isNotFoundError := e.(*bitsgo.NotFoundError); !isNotFoundError {
					deletionErrs = append(deletionErrs, e)
				}
			}
		}
		if response.NextMarker == "" {
			break
		}
		marker = response.NextMarker
	}

	if len(deletionErrs) != 0 {
		return errors.Errorf("Prefix %v, errors from deleting: %v", prefix, deletionErrs)
	}
	return nil
}

func (blobstore *Blobstore) Sign(resource string, method string, expirationTime time.Time) (signedURL string) {
	var e error
	switch strings.ToLower(method) {
	case "put":
		signedURL, e = blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(resource).GetSASURI(storage.BlobSASOptions{
			BlobServiceSASPermissions: storage.BlobServiceSASPermissions{Write: true, Create: true},
			SASOptions:                storage.SASOptions{Expiry: expirationTime},
		})
	case "get":
		signedURL, e = blobstore.client.GetContainerReference(blobstore.containerName).GetBlobReference(resource).GetSASURI(storage.BlobSASOptions{
			BlobServiceSASPermissions: storage.BlobServiceSASPermissions{Read: true},
			SASOptions:                storage.SASOptions{Expiry: expirationTime},
		})
	default:
		panic("The only supported methods are 'put' and 'get'")
	}
	if e != nil {
		panic(e)
	}
	return
}

func (blobstore *Blobstore) handleError(e error, context string, args ...interface{}) error {
	if azse, ok := e.(storage.AzureStorageServiceError); ok && azse.StatusCode == http.StatusNotFound {
		exists, e := blobstore.client.GetContainerReference(blobstore.containerName).Exists()
		if e != nil {
			return errors.Wrapf(e, context, args...)
		}
		if !exists {
			return errors.Errorf("Container does not exist '%v", blobstore.containerName)
		}
		return bitsgo.NewNotFoundError()
	}
	return errors.Wrapf(e, context, args...)
}
<|endoftext|>


<commit_before>package sidekick

import (
	"os"
	"testing"
	"strings"
	"errors"
)

/////////////////
// Test tools  //
/////////////////

type TestContext struct {
	Links   map[string]string
	Renames map[string]string
	Removed []string
	Command string
	Signal  os.Signal
	Writes  map[string]string
}

var context TestContext

func initContext() {
	context = TestContext{Removed: []string{}, Command: "", Writes:make(map[string]string), Renames:make(map[string]string), Links:make(map[string]string)}
}

type MockCommands struct{}

func (mc MockCommands) Writer(path string, content []byte, perm os.FileMode, isPanic bool) error {
	context.Writes[path] = string(content)
	return nil
}

func (mc MockCommands) Renamer(oldPath, newPath string, isPanic bool) error {
	context.Renames[newPath] = oldPath
	return nil
}

func MockCommand(name string, arg ...string) ([]byte, error) {
	context.Command = name + " " + strings.Join(arg, " ")
	return []byte("ok"), nil
}

func MockFailedCommand(name string, arg ...string) ([]byte, error) {
	return nil, errors.New("fails...")
}

func (mc MockCommands) Reader(path string, isPanic bool) ([]byte, error) {
	if strings.HasSuffix(path, "pid") {
		return []byte("1234"), nil
	} else if strings.HasSuffix(path, "conf") {
		return []byte("my conf"), nil
	} else {
		return []byte("nothing"), nil
	}
}

func (mc MockCommands) ReaderEmpty(path string) ([]byte, error) {
	if strings.HasSuffix(path, "pid") {
		return nil, errors.New("pid is not present")
	} else if strings.HasSuffix(path, "conf") {
		return nil, errors.New("conf is not present")
	} else {
		return []byte("nothing"), nil
	}
}

func (mc MockCommands) Linker(origin, destination string, isPanic bool) error {
	context.Links[destination] = origin
	return nil
}

func (mc MockCommands) Exists(newVersion string) bool {
	return true
}

func (mc MockCommands) CheckerAbsent(newVersion string, isPanic bool) bool {
	return false
}

func (mc MockCommands) Remover(path string, isPanic bool) error {
	if strings.Contains(path, "version") {
		if isPanic {
			panic("archive file " + path + " is not present")
		}
		return errors.New("archive file " + path + " is not present")
	}
	context.Removed = append(context.Removed, path)
	return nil
}

func MockSignal(pid int, signal os.Signal) error {
	context.Signal = signal
	return nil
}

func (mc MockCommands) MkdirAll(directory string) error {
	return nil
}

func (mc MockCommands)  ReadLinker(link string, isPanic bool) (string, error) {
	if strings.Contains(link, "archived") {
		return "/export/product/haproxy/product/2/bin/haproxy", nil
	}
	return "/export/product/haproxy/product/1/bin/haproxy", nil
}

func newMockHaproxy() *Haproxy {
	hap := NewHaproxy(&Config{HapHome: "/HOME"}, Context{Application: "TST", Platform: "DEV"})
	hap.Filesystem.Commands = MockCommands{}
	hap.Command = MockCommand
	hap.Config.HapVersions = []string{"1", "2", "3"}
	return hap
}

func check(t *testing.T, message string, actual, expected interface{}) {
	if actual != expected {
		t.Errorf(message, actual, expected)
		t.Fail()
	}
}

func checkMap(t *testing.T, expectedKey string, expectedValue string, data map[string]string) {
	actualValue, contains := data[expectedKey]
	if contains {
		if actualValue == expectedValue {
			//
		} else {
			t.Errorf("should contains value %s for key %s, but value is %s", expectedValue, expectedKey, actualValue)
			t.Fail()
		}
	}
}

func checkContains(t *testing.T, expectedValue string, data []string) {
	contains := false
	for _, v := range data {
		contains = contains || (v == expectedValue)
	}
	if !contains {
		t.Errorf("expected value %s is not present in %s", expectedValue, data)
		t.Fail()
	}
}

func checkError(t *testing.T, err error) {
	if err != nil {
		t.Errorf("unexpected error: %s", err)
		t.Fail()
	}
}
<commit_msg>[sidekick] fix unit test<commit_after>package sidekick

import (
	"os"
	"testing"
	"strings"
	"errors"
)

/////////////////
// Test tools  //
/////////////////

type TestContext struct {
	Links   map[string]string
	Renames map[string]string
	Removed []string
	Command string
	Signal  os.Signal
	Writes  map[string]string
}

var context TestContext

func initContext() {
	context = TestContext{Removed: []string{}, Command: "", Writes:make(map[string]string), Renames:make(map[string]string), Links:make(map[string]string)}
}

type MockCommands struct{}

func (mc MockCommands) Writer(path string, content []byte, perm os.FileMode, isPanic bool) error {
	context.Writes[path] = string(content)
	return nil
}

func (mc MockCommands) Renamer(oldPath, newPath string, isPanic bool) error {
	context.Renames[newPath] = oldPath
	return nil
}

func MockCommand(name string, arg ...string) ([]byte, error) {
	context.Command = name + " " + strings.Join(arg, " ")
	return []byte("ok"), nil
}

func MockFailedCommand(name string, arg ...string) ([]byte, error) {
	return nil, errors.New("fails...")
}

func (mc MockCommands) Reader(path string, isPanic bool) ([]byte, error) {
	if strings.HasSuffix(path, "pid") {
		return []byte("1234"), nil
	} else if strings.HasSuffix(path, "conf") {
		return []byte("my conf"), nil
	} else if strings.HasSuffix(path, "VERSION") {
		return []byte("1"), nil
	} else {
		return []byte("nothing"), nil
	}
}

func (mc MockCommands) ReaderEmpty(path string) ([]byte, error) {
	if strings.HasSuffix(path, "pid") {
		return nil, errors.New("pid is not present")
	} else if strings.HasSuffix(path, "conf") {
		return nil, errors.New("conf is not present")
	} else {
		return []byte("nothing"), nil
	}
}

func (mc MockCommands) Linker(origin, destination string, isPanic bool) error {
	context.Links[destination] = origin
	return nil
}

func (mc MockCommands) Exists(newVersion string) bool {
	return true
}

func (mc MockCommands) CheckerAbsent(newVersion string, isPanic bool) bool {
	return false
}

func (mc MockCommands) Remover(path string, isPanic bool) error {
	if strings.Contains(path, "version") {
		if isPanic {
			panic("archive file " + path + " is not present")
		}
		return errors.New("archive file " + path + " is not present")
	}
	context.Removed = append(context.Removed, path)
	return nil
}

func MockSignal(pid int, signal os.Signal) error {
	context.Signal = signal
	return nil
}

func (mc MockCommands) MkdirAll(directory string) error {
	return nil
}

func (mc MockCommands)  ReadLinker(link string, isPanic bool) (string, error) {
	if strings.Contains(link, "archived") {
		return "/export/product/haproxy/product/2/bin/haproxy", nil
	}
	return "/export/product/haproxy/product/1/bin/haproxy", nil
}

func newMockHaproxy() *Haproxy {
	hap := NewHaproxy(&Config{HapHome: "/HOME"}, Context{Application: "TST", Platform: "DEV"})
	hap.Filesystem.Commands = MockCommands{}
	hap.Command = MockCommand
	hap.Config.HapVersions = []string{"1", "2", "3"}
	return hap
}

func check(t *testing.T, message string, actual, expected interface{}) {
	if actual != expected {
		t.Errorf(message, actual, expected)
		t.Fail()
	}
}

func checkMap(t *testing.T, expectedKey string, expectedValue string, data map[string]string) {
	actualValue, contains := data[expectedKey]
	if contains {
		if actualValue == expectedValue {
			//
		} else {
			t.Errorf("should contains value %s for key %s, but value is %s", expectedValue, expectedKey, actualValue)
			t.Fail()
		}
	}
}

func checkContains(t *testing.T, expectedValue string, data []string) {
	contains := false
	for _, v := range data {
		contains = contains || (v == expectedValue)
	}
	if !contains {
		t.Errorf("expected value %s is not present in %s", expectedValue, data)
		t.Fail()
	}
}

func checkError(t *testing.T, err error) {
	if err != nil {
		t.Errorf("unexpected error: %s", err)
		t.Fail()
	}
}
<|endoftext|>