<commit_before>// Package tracer implements a Dapper-style tracing system. It is
// compatible with the Open Tracing specification.
package tracer

import (
	"crypto/rand"
	"encoding/binary"
	"encoding/hex"
	"io"
	"log"
	"strings"
	"time"

	"github.com/opentracing/opentracing-go"
)

type Joiner func(carrier interface{}) (traceID, parentID, spanID uint64, baggage map[string]string, err error)
type Injecter func(sp *Span, carrier interface{}) error

var joiners = map[interface{}]Joiner{
	opentracing.TextMap: textJoiner,
	opentracing.Binary:  binaryJoiner,
}

var injecters = map[interface{}]Injecter{
	opentracing.TextMap: textInjecter,
	opentracing.Binary:  binaryInjecter,
}

func RegisterJoiner(format interface{}, joiner Joiner) {
	joiners[format] = joiner
}

func RegisterInjecter(format interface{}, injecter Injecter) {
	injecters[format] = injecter
}

func textInjecter(sp *Span, carrier interface{}) error {
	w, ok := carrier.(opentracing.TextMapWriter)
	if !ok {
		return opentracing.ErrInvalidCarrier
	}
	w.Set("X-B3-TraceId", idToHex(sp.TraceID))
	w.Set("X-B3-SpanId", idToHex(sp.SpanID))
	w.Set("X-B3-ParentSpanId", idToHex(sp.ParentID))
	for k, v := range sp.Baggage {
		w.Set("X-B3-Baggage-"+k, v)
	}
	return nil
}

func textJoiner(carrier interface{}) (traceID, parentID, spanID uint64, baggage map[string]string, err error) {
	r, ok := carrier.(opentracing.TextMapReader)
	if !ok {
		return 0, 0, 0, nil, opentracing.ErrInvalidCarrier
	}
	baggage = map[string]string{}
	err = r.ForeachKey(func(key string, val string) error {
		lower := strings.ToLower(key)
		switch lower {
		case "x-b3-traceid":
			traceID = idFromHex(val)
		case "x-b3-spanid":
			spanID = idFromHex(val)
		case "x-b3-parentspanid":
			parentID = idFromHex(val)
		default:
			if strings.HasPrefix(lower, "x-b3-baggage-") {
				key = key[len("X-B3-Baggage-"):]
				baggage[key] = val
			}
		}
		return nil
	})
	if traceID == 0 {
		return 0, 0, 0, nil, opentracing.ErrTraceNotFound
	}
	return traceID, parentID, spanID, baggage, err
}

func binaryInjecter(sp *Span, carrier interface{}) error {
	w, ok := carrier.(io.Writer)
	if !ok {
		return opentracing.ErrInvalidCarrier
	}
	b := make([]byte, 24)
	binary.BigEndian.PutUint64(b, sp.TraceID)
	binary.BigEndian.PutUint64(b[8:], sp.TraceID)
	binary.BigEndian.PutUint64(b[16:], sp.TraceID)
	_, err := w.Write(b)
	return err
}

func binaryJoiner(carrier interface{}) (traceID, parentID, spanID uint64, baggage map[string]string, err error) {
	r, ok := carrier.(io.Reader)
	if !ok {
		return 0, 0, 0, nil, opentracing.ErrInvalidCarrier
	}
	b := make([]byte, 24)
	if _, err := io.ReadFull(r, b); err != nil {
		if err == io.ErrUnexpectedEOF {
			return 0, 0, 0, nil, opentracing.ErrTraceNotFound
		}
		return 0, 0, 0, nil, err
	}
	traceID = binary.BigEndian.Uint64(b)
	spanID = binary.BigEndian.Uint64(b[8:])
	parentID = binary.BigEndian.Uint64(b[16:])
	return traceID, parentID, spanID, nil, err
}

type RawTrace struct {
	TraceID uint64
	Spans   []RawSpan
}

// Span is an implementation of the Open Tracing Span interface.
type Span struct {
	tracer *Tracer
	RawSpan
}

type RawSpan struct {
	SpanID        uint64
	ParentID      uint64
	TraceID       uint64
	OperationName string
	StartTime     time.Time
	FinishTime    time.Time

	Tags    map[string]interface{}
	Baggage map[string]string
	Logs    []opentracing.LogData
}

func (sp *Span) SetOperationName(name string) opentracing.Span {
	sp.OperationName = name
	return sp
}

func (sp *Span) SetTag(key string, value interface{}) opentracing.Span {
	if sp.Tags == nil {
		sp.Tags = map[string]interface{}{}
	}
	sp.Tags[key] = value
	return sp
}

func (sp *Span) Finish() {
	sp.FinishWithOptions(opentracing.FinishOptions{})
}

func (sp *Span) FinishWithOptions(opts opentracing.FinishOptions) {
	if opts.FinishTime.IsZero() {
		opts.FinishTime = time.Now()
	}
	sp.FinishTime = opts.FinishTime
	for _, log := range opts.BulkLogData {
		sp.Log(log)
	}
	if err := sp.tracer.storer.Store(sp.RawSpan); err != nil {
		log.Println("error while storing tracing span:", err)
	}
}

func (sp *Span) LogEvent(event string) {
	sp.Log(opentracing.LogData{
		Event: event,
	})
}

func (sp *Span) LogEventWithPayload(event string, payload interface{}) {
	sp.Log(opentracing.LogData{
		Event:   event,
		Payload: payload,
	})
}

func (sp *Span) Log(data opentracing.LogData) {
	if data.Timestamp.IsZero() {
		data.Timestamp = time.Now()
	}
	sp.Logs = append(sp.Logs, data)
}

func (sp *Span) SetBaggageItem(key, value string) opentracing.Span {
	sp.Baggage[key] = value
	return sp
}

func (sp *Span) BaggageItem(key string) string {
	return sp.Baggage[key]
}

func (sp *Span) ForeachBaggageItem(handler func(k, v string) bool) {
	for k, v := range sp.Baggage {
		if !handler(k, v) {
			return
		}
	}
}

func (sp *Span) Tracer() opentracing.Tracer {
	return sp.tracer
}

// Tracer is an implementation of the Open Tracing Tracer interface.
type Tracer struct {
	storer      Storer
	idGenerator IDGenerator
}

func NewTracer(storer Storer, idGenerator IDGenerator) *Tracer {
	return &Tracer{storer, idGenerator}
}

func (tr *Tracer) StartSpan(operationName string) opentracing.Span {
	return tr.StartSpanWithOptions(opentracing.StartSpanOptions{
		OperationName: operationName,
	})
}

func (tr *Tracer) StartSpanWithOptions(opts opentracing.StartSpanOptions) opentracing.Span {
	if opts.StartTime.IsZero() {
		opts.StartTime = time.Now()
	}
	var traceID uint64
	var parentID uint64
	if opts.Parent != nil {
		parent, ok := opts.Parent.(*Span)
		if !ok {
			panic("parent span must be of type *Span")
		}
		parentID = parent.SpanID
		traceID = parent.TraceID
	}
	id := tr.idGenerator.GenerateID()
	if traceID == 0 {
		traceID = id
	}
	sp := &Span{
		tracer: tr,
		RawSpan: RawSpan{
			OperationName: opts.OperationName,
			SpanID:        id,
			ParentID:      parentID,
			TraceID:       traceID,
			StartTime:     opts.StartTime,
		},
	}
	return sp
}

func idToHex(id uint64) string {
	b := make([]byte, 8)
	binary.BigEndian.PutUint64(b, id)
	return hex.EncodeToString(b)
}

func idFromHex(s string) uint64 {
	b, _ := hex.DecodeString(s)
	return binary.BigEndian.Uint64(b)
}

func (tr *Tracer) Inject(sp opentracing.Span, format interface{}, carrier interface{}) error {
	// TODO(dh): support sampling
	span, ok := sp.(*Span)
	if !ok {
		return opentracing.ErrInvalidSpan
	}
	injecter, ok := injecters[format]
	if !ok {
		return opentracing.ErrUnsupportedFormat
	}
	return injecter(span, carrier)
}

func (tr *Tracer) Join(operationName string, format interface{}, carrier interface{}) (opentracing.Span, error) {
	// TODO(dh): support sampling
	joiner, ok := joiners[format]
	if !ok {
		return nil, opentracing.ErrUnsupportedFormat
	}
	traceID, parentID, spanID, baggage, err := joiner(carrier)
	if err != nil {
		return nil, err
	}

	return &Span{
		tracer: tr,
		RawSpan: RawSpan{
			TraceID:  traceID,
			SpanID:   spanID,
			ParentID: parentID,
			Baggage:  baggage,
		},
	}, nil
}

// IDGenerator generates IDs for traces and spans. The ID with value 0
// is reserved to mean "no parent span" and should not be generated.
type IDGenerator interface {
	GenerateID() uint64
}

// A Storer stores a finished span. "Storing" a span may either mean
// saving it in a storage engine, or sending it to a remote
// collector.
//
// If a span with the same ID and the same trace ID already exists,
// the existing and new spans should be merged into one span.
//
// Because spans are only stored once they're done, children will be
// stored before their parents.
type Storer interface {
	Store(sp RawSpan) error
}

type Queryer interface {
	TraceWithID(id uint64) (RawTrace, error)
	SpanWithID(id uint64) (RawSpan, error)
	QueryTraces(q Query) ([]RawSpan, error)
}

type QueryTag struct {
	Key        string
	Value      string
	CheckValue bool
}

type Query struct {
	StartTime     time.Time
	FinishTime    time.Time
	OperationName string
	AndTags       []QueryTag
	OrTags        []QueryTag
}

// RandomID generates random IDs by using crypto/rand.
type RandomID struct{}

// GenerateID generates an ID.
func (RandomID) GenerateID() uint64 {
	b := make([]byte, 8)
	for {
		_, _ = rand.Read(b)
		x := binary.BigEndian.Uint64(b)
		if x != 0 {
			return x
		}
	}
}
<commit_msg>Fix signature of Queryer.QueryTraces<commit_after>// Package tracer implements a Dapper-style tracing system. It is
// compatible with the Open Tracing specification.
package tracer

import (
	"crypto/rand"
	"encoding/binary"
	"encoding/hex"
	"io"
	"log"
	"strings"
	"time"

	"github.com/opentracing/opentracing-go"
)

type Joiner func(carrier interface{}) (traceID, parentID, spanID uint64, baggage map[string]string, err error)
type Injecter func(sp *Span, carrier interface{}) error

var joiners = map[interface{}]Joiner{
	opentracing.TextMap: textJoiner,
	opentracing.Binary:  binaryJoiner,
}

var injecters = map[interface{}]Injecter{
	opentracing.TextMap: textInjecter,
	opentracing.Binary:  binaryInjecter,
}

func RegisterJoiner(format interface{}, joiner Joiner) {
	joiners[format] = joiner
}

func RegisterInjecter(format interface{}, injecter Injecter) {
	injecters[format] = injecter
}

func textInjecter(sp *Span, carrier interface{}) error {
	w, ok := carrier.(opentracing.TextMapWriter)
	if !ok {
		return opentracing.ErrInvalidCarrier
	}
	w.Set("X-B3-TraceId", idToHex(sp.TraceID))
	w.Set("X-B3-SpanId", idToHex(sp.SpanID))
	w.Set("X-B3-ParentSpanId", idToHex(sp.ParentID))
	for k, v := range sp.Baggage {
		w.Set("X-B3-Baggage-"+k, v)
	}
	return nil
}

func textJoiner(carrier interface{}) (traceID, parentID, spanID uint64, baggage map[string]string, err error) {
	r, ok := carrier.(opentracing.TextMapReader)
	if !ok {
		return 0, 0, 0, nil, opentracing.ErrInvalidCarrier
	}
	baggage = map[string]string{}
	err = r.ForeachKey(func(key string, val string) error {
		lower := strings.ToLower(key)
		switch lower {
		case "x-b3-traceid":
			traceID = idFromHex(val)
		case "x-b3-spanid":
			spanID = idFromHex(val)
		case "x-b3-parentspanid":
			parentID = idFromHex(val)
		default:
			if strings.HasPrefix(lower, "x-b3-baggage-") {
				key = key[len("X-B3-Baggage-"):]
				baggage[key] = val
			}
		}
		return nil
	})
	if traceID == 0 {
		return 0, 0, 0, nil, opentracing.ErrTraceNotFound
	}
	return traceID, parentID, spanID, baggage, err
}

func binaryInjecter(sp *Span, carrier interface{}) error {
	w, ok := carrier.(io.Writer)
	if !ok {
		return opentracing.ErrInvalidCarrier
	}
	b := make([]byte, 24)
	binary.BigEndian.PutUint64(b, sp.TraceID)
	binary.BigEndian.PutUint64(b[8:], sp.TraceID)
	binary.BigEndian.PutUint64(b[16:], sp.TraceID)
	_, err := w.Write(b)
	return err
}

func binaryJoiner(carrier interface{}) (traceID, parentID, spanID uint64, baggage map[string]string, err error) {
	r, ok := carrier.(io.Reader)
	if !ok {
		return 0, 0, 0, nil, opentracing.ErrInvalidCarrier
	}
	b := make([]byte, 24)
	if _, err := io.ReadFull(r, b); err != nil {
		if err == io.ErrUnexpectedEOF {
			return 0, 0, 0, nil, opentracing.ErrTraceNotFound
		}
		return 0, 0, 0, nil, err
	}
	traceID = binary.BigEndian.Uint64(b)
	spanID = binary.BigEndian.Uint64(b[8:])
	parentID = binary.BigEndian.Uint64(b[16:])
	return traceID, parentID, spanID, nil, err
}

type RawTrace struct {
	TraceID uint64
	Spans   []RawSpan
}

// Span is an implementation of the Open Tracing Span interface.
type Span struct {
	tracer *Tracer
	RawSpan
}

type RawSpan struct {
	SpanID        uint64
	ParentID      uint64
	TraceID       uint64
	OperationName string
	StartTime     time.Time
	FinishTime    time.Time

	Tags    map[string]interface{}
	Baggage map[string]string
	Logs    []opentracing.LogData
}

func (sp *Span) SetOperationName(name string) opentracing.Span {
	sp.OperationName = name
	return sp
}

func (sp *Span) SetTag(key string, value interface{}) opentracing.Span {
	if sp.Tags == nil {
		sp.Tags = map[string]interface{}{}
	}
	sp.Tags[key] = value
	return sp
}

func (sp *Span) Finish() {
	sp.FinishWithOptions(opentracing.FinishOptions{})
}

func (sp *Span) FinishWithOptions(opts opentracing.FinishOptions) {
	if opts.FinishTime.IsZero() {
		opts.FinishTime = time.Now()
	}
	sp.FinishTime = opts.FinishTime
	for _, log := range opts.BulkLogData {
		sp.Log(log)
	}
	if err := sp.tracer.storer.Store(sp.RawSpan); err != nil {
		log.Println("error while storing tracing span:", err)
	}
}

func (sp *Span) LogEvent(event string) {
	sp.Log(opentracing.LogData{
		Event: event,
	})
}

func (sp *Span) LogEventWithPayload(event string, payload interface{}) {
	sp.Log(opentracing.LogData{
		Event:   event,
		Payload: payload,
	})
}

func (sp *Span) Log(data opentracing.LogData) {
	if data.Timestamp.IsZero() {
		data.Timestamp = time.Now()
	}
	sp.Logs = append(sp.Logs, data)
}

func (sp *Span) SetBaggageItem(key, value string) opentracing.Span {
	sp.Baggage[key] = value
	return sp
}

func (sp *Span) BaggageItem(key string) string {
	return sp.Baggage[key]
}

func (sp *Span) ForeachBaggageItem(handler func(k, v string) bool) {
	for k, v := range sp.Baggage {
		if !handler(k, v) {
			return
		}
	}
}

func (sp *Span) Tracer() opentracing.Tracer {
	return sp.tracer
}

// Tracer is an implementation of the Open Tracing Tracer interface.
type Tracer struct {
	storer      Storer
	idGenerator IDGenerator
}

func NewTracer(storer Storer, idGenerator IDGenerator) *Tracer {
	return &Tracer{storer, idGenerator}
}

func (tr *Tracer) StartSpan(operationName string) opentracing.Span {
	return tr.StartSpanWithOptions(opentracing.StartSpanOptions{
		OperationName: operationName,
	})
}

func (tr *Tracer) StartSpanWithOptions(opts opentracing.StartSpanOptions) opentracing.Span {
	if opts.StartTime.IsZero() {
		opts.StartTime = time.Now()
	}
	var traceID uint64
	var parentID uint64
	if opts.Parent != nil {
		parent, ok := opts.Parent.(*Span)
		if !ok {
			panic("parent span must be of type *Span")
		}
		parentID = parent.SpanID
		traceID = parent.TraceID
	}
	id := tr.idGenerator.GenerateID()
	if traceID == 0 {
		traceID = id
	}
	sp := &Span{
		tracer: tr,
		RawSpan: RawSpan{
			OperationName: opts.OperationName,
			SpanID:        id,
			ParentID:      parentID,
			TraceID:       traceID,
			StartTime:     opts.StartTime,
		},
	}
	return sp
}

func idToHex(id uint64) string {
	b := make([]byte, 8)
	binary.BigEndian.PutUint64(b, id)
	return hex.EncodeToString(b)
}

func idFromHex(s string) uint64 {
	b, _ := hex.DecodeString(s)
	return binary.BigEndian.Uint64(b)
}

func (tr *Tracer) Inject(sp opentracing.Span, format interface{}, carrier interface{}) error {
	// TODO(dh): support sampling
	span, ok := sp.(*Span)
	if !ok {
		return opentracing.ErrInvalidSpan
	}
	injecter, ok := injecters[format]
	if !ok {
		return opentracing.ErrUnsupportedFormat
	}
	return injecter(span, carrier)
}

func (tr *Tracer) Join(operationName string, format interface{}, carrier interface{}) (opentracing.Span, error) {
	// TODO(dh): support sampling
	joiner, ok := joiners[format]
	if !ok {
		return nil, opentracing.ErrUnsupportedFormat
	}
	traceID, parentID, spanID, baggage, err := joiner(carrier)
	if err != nil {
		return nil, err
	}

	return &Span{
		tracer: tr,
		RawSpan: RawSpan{
			TraceID:  traceID,
			SpanID:   spanID,
			ParentID: parentID,
			Baggage:  baggage,
		},
	}, nil
}

// IDGenerator generates IDs for traces and spans. The ID with value 0
// is reserved to mean "no parent span" and should not be generated.
type IDGenerator interface {
	GenerateID() uint64
}

// A Storer stores a finished span. "Storing" a span may either mean
// saving it in a storage engine, or sending it to a remote
// collector.
//
// If a span with the same ID and the same trace ID already exists,
// the existing and new spans should be merged into one span.
//
// Because spans are only stored once they're done, children will be
// stored before their parents.
type Storer interface {
	Store(sp RawSpan) error
}

type Queryer interface {
	TraceWithID(id uint64) (RawTrace, error)
	SpanWithID(id uint64) (RawSpan, error)
	QueryTraces(q Query) ([]RawTrace, error)
}

type QueryTag struct {
	Key        string
	Value      string
	CheckValue bool
}

type Query struct {
	StartTime     time.Time
	FinishTime    time.Time
	OperationName string
	AndTags       []QueryTag
	OrTags        []QueryTag
}

// RandomID generates random IDs by using crypto/rand.
type RandomID struct{}

// GenerateID generates an ID.
func (RandomID) GenerateID() uint64 {
	b := make([]byte, 8)
	for {
		_, _ = rand.Read(b)
		x := binary.BigEndian.Uint64(b)
		if x != 0 {
			return x
		}
	}
}
<|endoftext|>


<commit_before>package calculator

/* CHECKLIST
 * [ ] Uses interfaces as appropriate
 * [ ] Private package variables use underscore prefix
 * [ ] All parameters validated
 * [ ] All errors handled
 * [ ] Reviewed for concurrency safety
 * [ ] Code complete
 * [ ] Full test coverage
 */

import (
	"github.com/tidepool-org/platform/data"
	"github.com/tidepool-org/platform/data/types/base"
	"github.com/tidepool-org/platform/data/types/base/bolus/combination"
	"github.com/tidepool-org/platform/data/types/base/bolus/extended"
	"github.com/tidepool-org/platform/data/types/base/bolus/normal"
)

type Calculator struct {
	base.Base `bson:",inline"`

	*Recommended        `json:"recommended,omitempty" bson:"recommended,omitempty"`
	*BloodGlucoseTarget `json:"bgTarget,omitempty" bson:"bgTarget,omitempty"`

	BolusID                  *string  `json:"bolusId,omitempty" bson:"bolusId,omitempty"`
	CarbohydrateInput        *int     `json:"carbInput,omitempty" bson:"carbInput,omitempty"`
	InsulinOnBoard           *float64 `json:"insulinOnBoard,omitempty" bson:"insulinOnBoard,omitempty"`
	InsulinSensitivity       *float64 `json:"insulinSensitivity,omitempty" bson:"insulinSensitivity,omitempty"`
	InsulinCarbohydrateRatio *int     `json:"insulinCarbRatio,omitempty" bson:"insulinCarbRatio,omitempty"`
	BloodGlucoseInput        *float64 `json:"bgInput,omitempty" bson:"bgInput,omitempty"`
	Units                    *string  `json:"units,omitempty" bson:"units,omitempty"`

	//private field that will be used to build and normalize the embedded bolus
	bolus data.Datum
}

func Type() string {
	return "wizard"
}

func New() (*Calculator, error) {
	calculatorBase, err := base.New(Type())
	if err != nil {
		return nil, err
	}

	return &Calculator{
		Base: *calculatorBase,
	}, nil
}

func (c *Calculator) Parse(parser data.ObjectParser) error {
	parser.SetMeta(c.Meta())

	if err := c.Base.Parse(parser); err != nil {
		return err
	}

	c.CarbohydrateInput = parser.ParseInteger("carbInput")
	c.InsulinOnBoard = parser.ParseFloat("insulinOnBoard")
	c.InsulinSensitivity = parser.ParseFloat("insulinSensitivity")
	c.InsulinCarbohydrateRatio = parser.ParseInteger("insulinCarbRatio")
	c.BloodGlucoseInput = parser.ParseFloat("bgInput")
	c.Units = parser.ParseString("units")

	c.Recommended = ParseRecommended(parser.NewChildObjectParser("recommended"))
	c.BloodGlucoseTarget = ParseBloodGlucoseTarget(parser.NewChildObjectParser("bgTarget"))

	bolus, err := ParseBolus(parser.NewChildObjectParser("bolus"))
	if err != nil {
		return err
	}
	c.bolus = bolus

	return nil
}

func (c *Calculator) Validate(validator data.Validator) error {
	validator.SetMeta(c.Meta())

	if err := c.Base.Validate(validator); err != nil {
		return err
	}

	validator.ValidateInteger("carbInput", c.CarbohydrateInput).InRange(0, 1000)
	validator.ValidateFloat("insulinOnBoard", c.InsulinOnBoard).InRange(0.0, 250.0)
	validator.ValidateInteger("insulinCarbRatio", c.InsulinCarbohydrateRatio).InRange(0, 250)

	validator.ValidateStringAsBloodGlucoseUnits("units", c.Units).Exists()
	validator.ValidateFloatAsBloodGlucoseValue("bgInput", c.BloodGlucoseInput).InRangeForUnits(c.Units)
	validator.ValidateFloatAsBloodGlucoseValue("insulinSensitivity", c.InsulinSensitivity).InRangeForUnits(c.Units)

	if c.Recommended != nil {
		c.Recommended.Validate(validator.NewChildValidator("recommended"))
	}

	if c.BloodGlucoseTarget != nil {
		c.BloodGlucoseTarget.Validate(validator.NewChildValidator("bgTarget"), c.Units)
	}

	if c.bolus != nil {
		c.bolus.Validate(validator.NewChildValidator("bolus"))
	}

	return nil
}

func (c *Calculator) Normalize(normalizer data.Normalizer) error {
	normalizer.SetMeta(c.Meta())

	if err := c.Base.Normalize(normalizer); err != nil {
		return err
	}

	units := c.Units

	bloodGlucoseNormalizer := normalizer.NormalizeBloodGlucose(c.Units)
	c.Units = bloodGlucoseNormalizer.Units()
	c.InsulinSensitivity = bloodGlucoseNormalizer.Value(c.InsulinSensitivity)
	c.BloodGlucoseInput = bloodGlucoseNormalizer.Value(c.BloodGlucoseInput)

	if c.Recommended != nil {
		c.Recommended.Normalize(normalizer.NewChildNormalizer("recommended"))
	}

	if c.BloodGlucoseTarget != nil {
		c.BloodGlucoseTarget.Normalize(normalizer.NewChildNormalizer("bgTarget"), units)
	}

	if c.bolus != nil {
		c.bolus.Normalize(normalizer.NewChildNormalizer("bolus"))
		normalizer.AppendDatum(c.bolus)
		switch c.bolus.(type) {
		case *extended.Extended:
			c.BolusID = &c.bolus.(*extended.Extended).ID
		case *normal.Normal:
			c.BolusID = &c.bolus.(*normal.Normal).ID
		case *combination.Combination:
			c.BolusID = &c.bolus.(*combination.Combination).ID
		}
		c.bolus = nil
	}

	return nil
}
<commit_msg>using bolus instead of bolusId for tideline<commit_after>package calculator

/* CHECKLIST
 * [ ] Uses interfaces as appropriate
 * [ ] Private package variables use underscore prefix
 * [ ] All parameters validated
 * [ ] All errors handled
 * [ ] Reviewed for concurrency safety
 * [ ] Code complete
 * [ ] Full test coverage
 */

import (
	"github.com/tidepool-org/platform/data"
	"github.com/tidepool-org/platform/data/types/base"
	"github.com/tidepool-org/platform/data/types/base/bolus/combination"
	"github.com/tidepool-org/platform/data/types/base/bolus/extended"
	"github.com/tidepool-org/platform/data/types/base/bolus/normal"
)

type Calculator struct {
	base.Base `bson:",inline"`

	*Recommended        `json:"recommended,omitempty" bson:"recommended,omitempty"`
	*BloodGlucoseTarget `json:"bgTarget,omitempty" bson:"bgTarget,omitempty"`

	BolusID                  *string  `json:"bolus,omitempty" bson:"bolus,omitempty"`
	CarbohydrateInput        *int     `json:"carbInput,omitempty" bson:"carbInput,omitempty"`
	InsulinOnBoard           *float64 `json:"insulinOnBoard,omitempty" bson:"insulinOnBoard,omitempty"`
	InsulinSensitivity       *float64 `json:"insulinSensitivity,omitempty" bson:"insulinSensitivity,omitempty"`
	InsulinCarbohydrateRatio *int     `json:"insulinCarbRatio,omitempty" bson:"insulinCarbRatio,omitempty"`
	BloodGlucoseInput        *float64 `json:"bgInput,omitempty" bson:"bgInput,omitempty"`
	Units                    *string  `json:"units,omitempty" bson:"units,omitempty"`

	//private field that will be used to build and normalize the embedded bolus
	bolus data.Datum
}

func Type() string {
	return "wizard"
}

func New() (*Calculator, error) {
	calculatorBase, err := base.New(Type())
	if err != nil {
		return nil, err
	}

	return &Calculator{
		Base: *calculatorBase,
	}, nil
}

func (c *Calculator) Parse(parser data.ObjectParser) error {
	parser.SetMeta(c.Meta())

	if err := c.Base.Parse(parser); err != nil {
		return err
	}

	c.CarbohydrateInput = parser.ParseInteger("carbInput")
	c.InsulinOnBoard = parser.ParseFloat("insulinOnBoard")
	c.InsulinSensitivity = parser.ParseFloat("insulinSensitivity")
	c.InsulinCarbohydrateRatio = parser.ParseInteger("insulinCarbRatio")
	c.BloodGlucoseInput = parser.ParseFloat("bgInput")
	c.Units = parser.ParseString("units")

	c.Recommended = ParseRecommended(parser.NewChildObjectParser("recommended"))
	c.BloodGlucoseTarget = ParseBloodGlucoseTarget(parser.NewChildObjectParser("bgTarget"))

	bolus, err := ParseBolus(parser.NewChildObjectParser("bolus"))
	if err != nil {
		return err
	}
	c.bolus = bolus

	return nil
}

func (c *Calculator) Validate(validator data.Validator) error {
	validator.SetMeta(c.Meta())

	if err := c.Base.Validate(validator); err != nil {
		return err
	}

	validator.ValidateInteger("carbInput", c.CarbohydrateInput).InRange(0, 1000)
	validator.ValidateFloat("insulinOnBoard", c.InsulinOnBoard).InRange(0.0, 250.0)
	validator.ValidateInteger("insulinCarbRatio", c.InsulinCarbohydrateRatio).InRange(0, 250)

	validator.ValidateStringAsBloodGlucoseUnits("units", c.Units).Exists()
	validator.ValidateFloatAsBloodGlucoseValue("bgInput", c.BloodGlucoseInput).InRangeForUnits(c.Units)
	validator.ValidateFloatAsBloodGlucoseValue("insulinSensitivity", c.InsulinSensitivity).InRangeForUnits(c.Units)

	if c.Recommended != nil {
		c.Recommended.Validate(validator.NewChildValidator("recommended"))
	}

	if c.BloodGlucoseTarget != nil {
		c.BloodGlucoseTarget.Validate(validator.NewChildValidator("bgTarget"), c.Units)
	}

	if c.bolus != nil {
		c.bolus.Validate(validator.NewChildValidator("bolus"))
	}

	return nil
}

func (c *Calculator) Normalize(normalizer data.Normalizer) error {
	normalizer.SetMeta(c.Meta())

	if err := c.Base.Normalize(normalizer); err != nil {
		return err
	}

	units := c.Units

	bloodGlucoseNormalizer := normalizer.NormalizeBloodGlucose(c.Units)
	c.Units = bloodGlucoseNormalizer.Units()
	c.InsulinSensitivity = bloodGlucoseNormalizer.Value(c.InsulinSensitivity)
	c.BloodGlucoseInput = bloodGlucoseNormalizer.Value(c.BloodGlucoseInput)

	if c.Recommended != nil {
		c.Recommended.Normalize(normalizer.NewChildNormalizer("recommended"))
	}

	if c.BloodGlucoseTarget != nil {
		c.BloodGlucoseTarget.Normalize(normalizer.NewChildNormalizer("bgTarget"), units)
	}

	if c.bolus != nil {
		c.bolus.Normalize(normalizer.NewChildNormalizer("bolus"))
		normalizer.AppendDatum(c.bolus)
		switch c.bolus.(type) {
		case *extended.Extended:
			c.BolusID = &c.bolus.(*extended.Extended).ID
		case *normal.Normal:
			c.BolusID = &c.bolus.(*normal.Normal).ID
		case *combination.Combination:
			c.BolusID = &c.bolus.(*combination.Combination).ID
		}
		c.bolus = nil
	}

	return nil
}
<|endoftext|>


<commit_before>package integration_test

import (
	"path/filepath"

	"github.com/cloudfoundry/libbuildpack/cutlass"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = FDescribe("Rails 5.2 (Webpack/Yarn) App", func() {
	var app *cutlass.App
	AfterEach(func() { app = DestroyApp(app) })

	BeforeEach(func() {
		app = cutlass.New(filepath.Join(bpDir, "fixtures", "rails52_webpack"))
		app.SetEnv("BP_DEBUG", "1")
	})

	It("Installs node6 and runs", func() {
		PushAppAndConfirm(app)

		Expect(app.GetBody("/")).To(ContainSubstring("Hello, Rails!"))
		Eventually(app.Stdout.String).Should(ContainSubstring(`Started GET "/" for`))

		Eventually(app.Stdout.String).ShouldNot(ContainSubstring("assets:clean"))

	})
})
<commit_msg>removes focus for newly added tests<commit_after>package integration_test

import (
	"path/filepath"

	"github.com/cloudfoundry/libbuildpack/cutlass"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Rails 5.2 (Webpack/Yarn) App", func() {
	var app *cutlass.App
	AfterEach(func() { app = DestroyApp(app) })

	BeforeEach(func() {
		app = cutlass.New(filepath.Join(bpDir, "fixtures", "rails52_webpack"))
		app.SetEnv("BP_DEBUG", "1")
	})

	It("Installs node6 and runs", func() {
		PushAppAndConfirm(app)

		Expect(app.GetBody("/")).To(ContainSubstring("Hello, Rails!"))
		Eventually(app.Stdout.String).Should(ContainSubstring(`Started GET "/" for`))

		Eventually(app.Stdout.String).ShouldNot(ContainSubstring("assets:clean"))

	})
})
<|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"math"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/gosuri/uiprogress"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
)

type Connection struct {
	Server              string
	File                FileMetaData
	NumberOfConnections int
	Code                string
	HashedCode          string
	IsSender            bool
	Debug               bool
	DontEncrypt         bool
	bars                []*uiprogress.Bar
	rate                int
}

type FileMetaData struct {
	Name string
	Size int
	Hash string
}

func NewConnection(flags *Flags) *Connection {
	c := new(Connection)
	c.Debug = flags.Debug
	c.DontEncrypt = flags.DontEncrypt
	c.Server = flags.Server
	c.Code = flags.Code
	c.NumberOfConnections = flags.NumberOfConnections
	c.rate = flags.Rate
	if len(flags.File) > 0 {
		c.File.Name = flags.File
		c.IsSender = true
	} else {
		c.IsSender = false
	}

	log.SetFormatter(&log.TextFormatter{})
	if c.Debug {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.WarnLevel)
	}

	return c
}

func (c *Connection) Run() error {
	forceSingleThreaded := false
	if c.IsSender {
		fsize, err := FileSize(c.File.Name)
		if err != nil {
			return err
		}
		if fsize < MAX_NUMBER_THREADS*BUFFERSIZE {
			forceSingleThreaded = true
			log.Debug("forcing single thread")
		}
	}
	log.Debug("checking code validity")
	for {
		// check code
		goodCode := true
		m := strings.Split(c.Code, "-")
		numThreads, errParse := strconv.Atoi(m[0])
		if len(m) < 2 {
			goodCode = false
		} else if numThreads > MAX_NUMBER_THREADS || numThreads < 1 || (forceSingleThreaded && numThreads != 1) {
			c.NumberOfConnections = MAX_NUMBER_THREADS
			goodCode = false
		} else if errParse != nil {
			goodCode = false
		}
		log.Debug(m)
		if !goodCode {
			if c.IsSender {
				if forceSingleThreaded {
					c.NumberOfConnections = 1
				}
				c.Code = strconv.Itoa(c.NumberOfConnections) + "-" + GetRandomName()
			} else {
				if len(c.Code) != 0 {
					fmt.Println("Code must begin with number of threads (e.g. 3-some-code)")
				}
				c.Code = getInput("Enter receive code: ")
			}
		} else {
			break
		}
	}
	// assign number of connections
	c.NumberOfConnections, _ = strconv.Atoi(strings.Split(c.Code, "-")[0])

	if c.IsSender {
		if c.DontEncrypt {
			// don't encrypt
			CopyFile(c.File.Name, c.File.Name+".enc")
		} else {
			// encrypt
			log.Debug("encrypting...")
			if err := EncryptFile(c.File.Name, c.File.Name+".enc", c.Code); err != nil {
				return err
			}
		}
		// get file hash
		var err error
		c.File.Hash, err = HashFile(c.File.Name)
		if err != nil {
			return err
		}
		// get file size
		c.File.Size, err = FileSize(c.File.Name + ".enc")
		if err != nil {
			return err
		}
		fmt.Printf("Sending %d byte file named '%s'\n", c.File.Size, c.File.Name)
		fmt.Printf("Code is: %s\n", c.Code)
	}

	return c.runClient()
}

// runClient spawns threads for parallel uplink/downlink via TCP
func (c *Connection) runClient() error {
	logger := log.WithFields(log.Fields{
		"code":    c.Code,
		"sender?": c.IsSender,
	})

	c.HashedCode = Hash(c.Code)

	var wg sync.WaitGroup
	wg.Add(c.NumberOfConnections)

	uiprogress.Start()
	if !c.Debug {
		c.bars = make([]*uiprogress.Bar, c.NumberOfConnections)
	}
	gotOK := false
	gotResponse := false
	for id := 0; id < c.NumberOfConnections; id++ {
		go func(id int) {
			defer wg.Done()
			port := strconv.Itoa(27001 + id)
			connection, err := net.Dial("tcp", c.Server+":"+port)
			if err != nil {
				panic(err)
			}
			defer connection.Close()

			message := receiveMessage(connection)
			logger.Debugf("relay says: %s", message)
			if c.IsSender {
				logger.Debugf("telling relay: %s", "s."+c.Code)
				metaData, err := json.Marshal(c.File)
				if err != nil {
					log.Error(err)
				}
				encryptedMetaData, salt, iv := Encrypt(metaData, c.Code)
				sendMessage("s."+c.HashedCode+"."+hex.EncodeToString(encryptedMetaData)+"-"+salt+"-"+iv, connection)
			} else {
				logger.Debugf("telling relay: %s", "r."+c.Code)
				sendMessage("r."+c.HashedCode+".0.0.0", connection)
			}
			if c.IsSender { // this is a sender
				logger.Debug("waiting for ok from relay")
				message = receiveMessage(connection)
				logger.Debug("got ok from relay")
				if id == 0 {
					fmt.Printf("\nSending (->%s)..\n", message)
				}
				// wait for pipe to be made
				time.Sleep(100 * time.Millisecond)
				// Write data from file
				logger.Debug("send file")
				c.sendFile(id, connection)
			} else { // this is a receiver
				logger.Debug("waiting for meta data from sender")
				message = receiveMessage(connection)
				m := strings.Split(message, "-")
				encryptedData, salt, iv, sendersAddress := m[0], m[1], m[2], m[3]
				encryptedBytes, err := hex.DecodeString(encryptedData)
				if err != nil {
					log.Error(err)
					return
				}
				decryptedBytes, _ := Decrypt(encryptedBytes, c.Code, salt, iv, c.DontEncrypt)
				err = json.Unmarshal(decryptedBytes, &c.File)
				if err != nil {
					log.Error(err)
					return
				}
				log.Debugf("meta data received: %v", c.File)
				// have the main thread ask for the okay
				if id == 0 {
					fmt.Printf("Receiving file (%d bytes) into: %s\n", c.File.Size, c.File.Name)
					var sentFileNames []string

					if fileAlreadyExists(sentFileNames, c.File.Name) {
						fmt.Printf("Will not overwrite file!")
						os.Exit(1)
					}
					getOK := getInput("ok? (y/n): ")
					if getOK == "y" {
						gotOK = true
						sentFileNames = append(sentFileNames, c.File.Name)
					}
					gotResponse = true
				}
				// wait for the main thread to get the okay
				for limit := 0; limit < 1000; limit++ {
					if gotResponse {
						break
					}
					time.Sleep(10 * time.Millisecond)
				}
				if !gotOK {
					sendMessage("not ok", connection)
				} else {
					sendMessage("ok", connection)
					logger.Debug("receive file")
					fmt.Printf("\n\nReceiving (<-%s)..\n", sendersAddress)
					c.receiveFile(id, connection)
				}
			}
		}(id)
	}
	wg.Wait()

	if !c.IsSender {
		if !gotOK {
			return errors.New("Transfer interrupted")
		}
		c.catFile(c.File.Name)
		log.Debugf("Code: [%s]", c.Code)
		if c.DontEncrypt {
			if err := CopyFile(c.File.Name+".enc", c.File.Name); err != nil {
				return err
			}
		} else {
			if err := DecryptFile(c.File.Name+".enc", c.File.Name, c.Code); err != nil {
				return errors.Wrap(err, "Problem decrypting file")
			}
		}
		if !c.Debug {
			os.Remove(c.File.Name + ".enc")
		}

		fileHash, err := HashFile(c.File.Name)
		if err != nil {
			log.Error(err)
		}
		log.Debugf("\n\n\ndownloaded hash: [%s]", fileHash)
		log.Debugf("\n\n\nrelayed hash: [%s]", c.File.Hash)

		if c.File.Hash != fileHash {
			return fmt.Errorf("\nUh oh! %s is corrupted! Sorry, try again.\n", c.File.Name)
		} else {
			fmt.Printf("\nReceived file written to %s", c.File.Name)
		}
	} else {
		fmt.Println("File sent.")
		// TODO: Add confirmation
	}
	return nil
}

func fileAlreadyExists(s []string, f string) bool {
	for _, a := range s {
		if a == f {
			return true
		}
	}
	return false
}

func (c *Connection) catFile(fname string) {
	// cat the file
	os.Remove(fname)
	finished, err := os.Create(fname + ".enc")
	defer finished.Close()
	if err != nil {
		log.Fatal(err)
	}
	for id := 0; id < c.NumberOfConnections; id++ {
		fh, err := os.Open(fname + "." + strconv.Itoa(id))
		if err != nil {
			log.Fatal(err)
		}

		_, err = io.Copy(finished, fh)
		if err != nil {
			log.Fatal(err)
		}
		fh.Close()
		os.Remove(fname + "." + strconv.Itoa(id))
	}

}

func (c *Connection) receiveFile(id int, connection net.Conn) error {
	logger := log.WithFields(log.Fields{
		"function": "receiveFile #" + strconv.Itoa(id),
	})

	logger.Debug("waiting for chunk size from sender")
	fileSizeBuffer := make([]byte, 10)
	connection.Read(fileSizeBuffer)
	fileDataString := strings.Trim(string(fileSizeBuffer), ":")
	fileSizeInt, _ := strconv.Atoi(fileDataString)
	chunkSize := int64(fileSizeInt)
	logger.Debugf("chunk size: %d", chunkSize)

	os.Remove(c.File.Name + "." + strconv.Itoa(id))
	newFile, err := os.Create(c.File.Name + "." + strconv.Itoa(id))
	if err != nil {
		panic(err)
	}
	defer newFile.Close()

	if !c.Debug {
		c.bars[id] = uiprogress.AddBar(int(chunkSize)/1024 + 1).AppendCompleted().PrependElapsed()
	}

	logger.Debug("waiting for file")
	var receivedBytes int64
	for {
		if !c.Debug {
			c.bars[id].Incr()
		}
		if (chunkSize - receivedBytes) < BUFFERSIZE {
			logger.Debug("at the end")
			io.CopyN(newFile, connection, (chunkSize - receivedBytes))
			// Empty the remaining bytes that we don't need from the network buffer
			if (receivedBytes+BUFFERSIZE)-chunkSize < BUFFERSIZE {
				logger.Debug("empty remaining bytes from network buffer")
				connection.Read(make([]byte, (receivedBytes+BUFFERSIZE)-chunkSize))
			}
			break
		}
		io.CopyN(newFile, connection, BUFFERSIZE)
		receivedBytes += BUFFERSIZE
	}
	logger.Debug("received file")
	return nil
}

func (c *Connection) sendFile(id int, connection net.Conn) {
	logger := log.WithFields(log.Fields{
		"function": "sendFile #" + strconv.Itoa(id),
	})
	defer connection.Close()

	var err error

	numChunks := math.Ceil(float64(c.File.Size) / float64(BUFFERSIZE))
	chunksPerWorker := int(math.Ceil(numChunks / float64(c.NumberOfConnections)))

	chunkSize := int64(chunksPerWorker * BUFFERSIZE)
	if id+1 == c.NumberOfConnections {
		chunkSize = int64(c.File.Size) - int64(c.NumberOfConnections-1)*chunkSize
	}

	if id == 0 || id == c.NumberOfConnections-1 {
		logger.Debugf("numChunks: %v", numChunks)
		logger.Debugf("chunksPerWorker: %v", chunksPerWorker)
		logger.Debugf("bytesPerchunkSizeConnection: %v", chunkSize)
	}

	logger.Debugf("sending chunk size: %d", chunkSize)
	connection.Write([]byte(fillString(strconv.FormatInt(int64(chunkSize), 10), 10)))

	sendBuffer := make([]byte, BUFFERSIZE)

	// open encrypted file
	file, err := os.Open(c.File.Name + ".enc")
	if err != nil {
		log.Error(err)
		return
	}
	defer file.Close()

	chunkI := 0
	if !c.Debug {
		c.bars[id] = uiprogress.AddBar(chunksPerWorker).AppendCompleted().PrependElapsed()
	}

	bufferSizeInKilobytes := BUFFERSIZE / 1024
	rate := float64(c.rate) / float64(c.NumberOfConnections*bufferSizeInKilobytes)
	throttle := time.NewTicker(time.Second / time.Duration(rate))
	defer throttle.Stop()

	for range throttle.C {
		_, err = file.Read(sendBuffer)
		if err == io.EOF {
			//End of file reached, break out of for loop
			logger.Debug("EOF")
			break
		}
		if (chunkI >= chunksPerWorker*id && chunkI < chunksPerWorker*id+chunksPerWorker) || (id == c.NumberOfConnections-1 && chunkI >= chunksPerWorker*id) {
			connection.Write(sendBuffer)
			if !c.Debug {
				c.bars[id].Incr()
			}
		}
		chunkI++
	}
	logger.Debug("file is sent")
	return
}
<commit_msg>Need to add concurrent reading from the file<commit_after>package main

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"math"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/gosuri/uiprogress"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
)

type Connection struct {
	Server              string
	File                FileMetaData
	NumberOfConnections int
	Code                string
	HashedCode          string
	IsSender            bool
	Debug               bool
	DontEncrypt         bool
	bars                []*uiprogress.Bar
	rate                int
}

type FileMetaData struct {
	Name string
	Size int
	Hash string
}

func NewConnection(flags *Flags) *Connection {
	c := new(Connection)
	c.Debug = flags.Debug
	c.DontEncrypt = flags.DontEncrypt
	c.Server = flags.Server
	c.Code = flags.Code
	c.NumberOfConnections = flags.NumberOfConnections
	c.rate = flags.Rate
	if len(flags.File) > 0 {
		c.File.Name = flags.File
		c.IsSender = true
	} else {
		c.IsSender = false
	}

	log.SetFormatter(&log.TextFormatter{})
	if c.Debug {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.WarnLevel)
	}

	return c
}

func (c *Connection) Run() error {
	forceSingleThreaded := false
	if c.IsSender {
		fsize, err := FileSize(c.File.Name)
		if err != nil {
			return err
		}
		if fsize < MAX_NUMBER_THREADS*BUFFERSIZE {
			forceSingleThreaded = true
			log.Debug("forcing single thread")
		}
	}
	log.Debug("checking code validity")
	for {
		// check code
		goodCode := true
		m := strings.Split(c.Code, "-")
		log.Debug(m)
		numThreads, errParse := strconv.Atoi(m[0])
		if len(m) < 2 {
			goodCode = false
			log.Debug("code too short")
		} else if numThreads > MAX_NUMBER_THREADS || numThreads < 1 || (forceSingleThreaded && numThreads != 1) {
			c.NumberOfConnections = MAX_NUMBER_THREADS
			goodCode = false
			log.Debug("incorrect number of threads")
		} else if errParse != nil {
			goodCode = false
			log.Debug("problem parsing threads")
		}
		log.Debug(m)
		log.Debug(goodCode)
		if !goodCode {
			if c.IsSender {
				if forceSingleThreaded {
					c.NumberOfConnections = 1
				}
				c.Code = strconv.Itoa(c.NumberOfConnections) + "-" + GetRandomName()
			} else {
				if len(c.Code) != 0 {
					fmt.Println("Code must begin with number of threads (e.g. 3-some-code)")
				}
				c.Code = getInput("Enter receive code: ")
			}
		} else {
			break
		}
	}
	// assign number of connections
	c.NumberOfConnections, _ = strconv.Atoi(strings.Split(c.Code, "-")[0])

	if c.IsSender {
		if c.DontEncrypt {
			// don't encrypt
			CopyFile(c.File.Name, c.File.Name+".enc")
		} else {
			// encrypt
			log.Debug("encrypting...")
			if err := EncryptFile(c.File.Name, c.File.Name+".enc", c.Code); err != nil {
				return err
			}
		}
		// get file hash
		var err error
		c.File.Hash, err = HashFile(c.File.Name)
		if err != nil {
			return err
		}
		// get file size
		c.File.Size, err = FileSize(c.File.Name + ".enc")
		if err != nil {
			return err
		}
		fmt.Printf("Sending %d byte file named '%s'\n", c.File.Size, c.File.Name)
		fmt.Printf("Code is: %s\n", c.Code)
	}

	return c.runClient()
}

// runClient spawns threads for parallel uplink/downlink via TCP
func (c *Connection) runClient() error {
	logger := log.WithFields(log.Fields{
		"code":    c.Code,
		"sender?": c.IsSender,
	})

	c.HashedCode = Hash(c.Code)

	var wg sync.WaitGroup
	wg.Add(c.NumberOfConnections)

	uiprogress.Start()
	if !c.Debug {
		c.bars = make([]*uiprogress.Bar, c.NumberOfConnections)
	}
	gotOK := false
	gotResponse := false
	for id := 0; id < c.NumberOfConnections; id++ {
		go func(id int) {
			defer wg.Done()
			port := strconv.Itoa(27001 + id)
			connection, err := net.Dial("tcp", c.Server+":"+port)
			if err != nil {
				panic(err)
			}
			defer connection.Close()

			message := receiveMessage(connection)
			logger.Debugf("relay says: %s", message)
			if c.IsSender {
				logger.Debugf("telling relay: %s", "s."+c.Code)
				metaData, err := json.Marshal(c.File)
				if err != nil {
					log.Error(err)
				}
				encryptedMetaData, salt, iv := Encrypt(metaData, c.Code)
				sendMessage("s."+c.HashedCode+"."+hex.EncodeToString(encryptedMetaData)+"-"+salt+"-"+iv, connection)
			} else {
				logger.Debugf("telling relay: %s", "r."+c.Code)
				sendMessage("r."+c.HashedCode+".0.0.0", connection)
			}
			if c.IsSender { // this is a sender
				logger.Debug("waiting for ok from relay")
				message = receiveMessage(connection)
				logger.Debug("got ok from relay")
				if id == 0 {
					fmt.Printf("\nSending (->%s)..\n", message)
				}
				// wait for pipe to be made
				time.Sleep(100 * time.Millisecond)
				// Write data from file
				logger.Debug("send file")
				c.sendFile(id, connection)
			} else { // this is a receiver
				logger.Debug("waiting for meta data from sender")
				message = receiveMessage(connection)
				m := strings.Split(message, "-")
				encryptedData, salt, iv, sendersAddress := m[0], m[1], m[2], m[3]
				encryptedBytes, err := hex.DecodeString(encryptedData)
				if err != nil {
					log.Error(err)
					return
				}
				decryptedBytes, _ := Decrypt(encryptedBytes, c.Code, salt, iv, c.DontEncrypt)
				err = json.Unmarshal(decryptedBytes, &c.File)
				if err != nil {
					log.Error(err)
					return
				}
				log.Debugf("meta data received: %v", c.File)
				// have the main thread ask for the okay
				if id == 0 {
					fmt.Printf("Receiving file (%d bytes) into: %s\n", c.File.Size, c.File.Name)
					var sentFileNames []string

					if fileAlreadyExists(sentFileNames, c.File.Name) {
						fmt.Printf("Will not overwrite file!")
						os.Exit(1)
					}
					getOK := getInput("ok? (y/n): ")
					if getOK == "y" {
						gotOK = true
						sentFileNames = append(sentFileNames, c.File.Name)
					}
					gotResponse = true
				}
				// wait for the main thread to get the okay
				for limit := 0; limit < 1000; limit++ {
					if gotResponse {
						break
					}
					time.Sleep(10 * time.Millisecond)
				}
				if !gotOK {
					sendMessage("not ok", connection)
				} else {
					sendMessage("ok", connection)
					logger.Debug("receive file")
					if id == 0 {
						fmt.Printf("\n\nReceiving (<-%s)..\n", sendersAddress)
					}
					c.receiveFile(id, connection)
				}
			}
		}(id)
	}
	wg.Wait()

	if !c.IsSender {
		if !gotOK {
			return errors.New("Transfer interrupted")
		}
		c.catFile(c.File.Name)
		log.Debugf("Code: [%s]", c.Code)
		if c.DontEncrypt {
			if err := CopyFile(c.File.Name+".enc", c.File.Name); err != nil {
				return err
			}
		} else {
			if err := DecryptFile(c.File.Name+".enc", c.File.Name, c.Code); err != nil {
				return errors.Wrap(err, "Problem decrypting file")
			}
		}
		if !c.Debug {
			os.Remove(c.File.Name + ".enc")
		}

		fileHash, err := HashFile(c.File.Name)
		if err != nil {
			log.Error(err)
		}
		log.Debugf("\n\n\ndownloaded hash: [%s]", fileHash)
		log.Debugf("\n\n\nrelayed hash: [%s]", c.File.Hash)

		if c.File.Hash != fileHash {
			return fmt.Errorf("\nUh oh! %s is corrupted! Sorry, try again.\n", c.File.Name)
		} else {
			fmt.Printf("\nReceived file written to %s", c.File.Name)
		}
	} else {
		fmt.Println("File sent.")
		// TODO: Add confirmation
	}
	return nil
}

func fileAlreadyExists(s []string, f string) bool {
	for _, a := range s {
		if a == f {
			return true
		}
	}
	return false
}

func (c *Connection) catFile(fname string) {
	// cat the file
	os.Remove(fname)
	finished, err := os.Create(fname + ".enc")
	defer finished.Close()
	if err != nil {
		log.Fatal(err)
	}
	for id := 0; id < c.NumberOfConnections; id++ {
		fh, err := os.Open(fname + "." + strconv.Itoa(id))
		if err != nil {
			log.Fatal(err)
		}

		_, err = io.Copy(finished, fh)
		if err != nil {
			log.Fatal(err)
		}
		fh.Close()
		os.Remove(fname + "." + strconv.Itoa(id))
	}

}

func (c *Connection) receiveFile(id int, connection net.Conn) error {
	logger := log.WithFields(log.Fields{
		"function": "receiveFile #" + strconv.Itoa(id),
	})

	logger.Debug("waiting for chunk size from sender")
	fileSizeBuffer := make([]byte, 10)
	connection.Read(fileSizeBuffer)
	fileDataString := strings.Trim(string(fileSizeBuffer), ":")
	fileSizeInt, _ := strconv.Atoi(fileDataString)
	chunkSize := int64(fileSizeInt)
	logger.Debugf("chunk size: %d", chunkSize)

	os.Remove(c.File.Name + "." + strconv.Itoa(id))
	newFile, err := os.Create(c.File.Name + "." + strconv.Itoa(id))
	if err != nil {
		panic(err)
	}
	defer newFile.Close()

	if !c.Debug {
		c.bars[id] = uiprogress.AddBar(int(chunkSize)/1024 + 1).AppendCompleted().PrependElapsed()
	}

	logger.Debug("waiting for file")
	var receivedBytes int64
	receivedFirstBytes := false
	for {
		if !c.Debug {
			c.bars[id].Incr()
		}
		if (chunkSize - receivedBytes) < BUFFERSIZE {
			logger.Debug("at the end")
			io.CopyN(newFile, connection, (chunkSize - receivedBytes))
			// Empty the remaining bytes that we don't need from the network buffer
			if (receivedBytes+BUFFERSIZE)-chunkSize < BUFFERSIZE {
				logger.Debug("empty remaining bytes from network buffer")
				connection.Read(make([]byte, (receivedBytes+BUFFERSIZE)-chunkSize))
			}
			break
		}
		io.CopyN(newFile, connection, BUFFERSIZE)
		receivedBytes += BUFFERSIZE
		if !receivedFirstBytes {
			receivedFirstBytes = true
			logger.Debug("Receieved first bytes!")
		}
	}
	logger.Debug("received file")
	return nil
}

func (c *Connection) sendFile(id int, connection net.Conn) {
	logger := log.WithFields(log.Fields{
		"function": "sendFile #" + strconv.Itoa(id),
	})
	defer connection.Close()

	var err error

	numChunks := math.Ceil(float64(c.File.Size) / float64(BUFFERSIZE))
	chunksPerWorker := int(math.Ceil(numChunks / float64(c.NumberOfConnections)))

	chunkSize := int64(chunksPerWorker * BUFFERSIZE)
	if id+1 == c.NumberOfConnections {
		chunkSize = int64(c.File.Size) - int64(c.NumberOfConnections-1)*chunkSize
	}

	if id == 0 || id == c.NumberOfConnections-1 {
		logger.Debugf("numChunks: %v", numChunks)
		logger.Debugf("chunksPerWorker: %v", chunksPerWorker)
		logger.Debugf("bytesPerchunkSizeConnection: %v", chunkSize)
	}

	logger.Debugf("sending chunk size: %d", chunkSize)
	connection.Write([]byte(fillString(strconv.FormatInt(int64(chunkSize), 10), 10)))

	sendBuffer := make([]byte, BUFFERSIZE)

	// open encrypted file
	file, err := os.OpenFile(c.File.Name+".enc", os.O_RDONLY, 0755)
	if err != nil {
		log.Error(err)
		return
	}
	defer file.Close()

	chunkI := 0
	if !c.Debug {
		c.bars[id] = uiprogress.AddBar(chunksPerWorker).AppendCompleted().PrependElapsed()
	}

	bufferSizeInKilobytes := BUFFERSIZE / 1024
	rate := float64(c.rate) / float64(c.NumberOfConnections*bufferSizeInKilobytes)
	throttle := time.NewTicker(time.Second / time.Duration(rate))
	defer throttle.Stop()

	for range throttle.C {
		_, err = file.Read(sendBuffer)
		if err == io.EOF {
			//End of file reached, break out of for loop
			logger.Debug("EOF")
			break
		}
		if (chunkI >= chunksPerWorker*id && chunkI < chunksPerWorker*id+chunksPerWorker) || (id == c.NumberOfConnections-1 && chunkI >= chunksPerWorker*id) {
			connection.Write(sendBuffer)
			if !c.Debug {
				c.bars[id].Incr()
			}
		}
		chunkI++
	}
	logger.Debug("file is sent")
	return
}
<|endoftext|>


<commit_before>package zerolog

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	cReset    = 0
	cBold     = 1
	cRed      = 31
	cGreen    = 32
	cYellow   = 33
	cBlue     = 34
	cMagenta  = 35
	cCyan     = 36
	cGray     = 37
	cDarkGray = 90
)

var consoleBufPool = sync.Pool{
	New: func() interface{} {
		return bytes.NewBuffer(make([]byte, 0, 100))
	},
}

// ConsoleWriter reads a JSON object per write operation and output an
// optionally colored human readable version on the Out writer.
type ConsoleWriter struct {
	Out     io.Writer
	NoColor bool
}

func (w ConsoleWriter) Write(p []byte) (n int, err error) {
	var event map[string]interface{}
	p = decodeIfBinaryToBytes(p)
	d := json.NewDecoder(bytes.NewReader(p))
	d.UseNumber()
	err = d.Decode(&event)
	if err != nil {
		return
	}
	buf := consoleBufPool.Get().(*bytes.Buffer)
	defer consoleBufPool.Put(buf)
	lvlColor := cReset
	level := "????"
	if l, ok := event[LevelFieldName].(string); ok {
		if !w.NoColor {
			lvlColor = levelColor(l)
		}
		level = strings.ToUpper(l)[0:4]
	}
	fmt.Fprintf(buf, "%s |%s| %s",
		colorize(formatTime(event[TimestampFieldName]), cDarkGray, !w.NoColor),
		colorize(level, lvlColor, !w.NoColor),
		colorize(event[MessageFieldName], cReset, !w.NoColor))
	fields := make([]string, 0, len(event))
	for field := range event {
		switch field {
		case LevelFieldName, TimestampFieldName, MessageFieldName:
			continue
		}
		fields = append(fields, field)
	}
	sort.Strings(fields)
	for _, field := range fields {
		fmt.Fprintf(buf, " %s=", colorize(field, cCyan, !w.NoColor))
		switch value := event[field].(type) {
		case string:
			if needsQuote(value) {
				buf.WriteString(strconv.Quote(value))
			} else {
				buf.WriteString(value)
			}
		case json.Number:
			fmt.Fprint(buf, value)
		default:
			b, err := json.Marshal(value)
			if err != nil {
				fmt.Fprintf(buf, "[error: %v]", err)
			} else {
				fmt.Fprint(buf, string(b))
			}
		}
	}
	buf.WriteByte('\n')
	buf.WriteTo(w.Out)
	n = len(p)
	return
}

func formatTime(t interface{}) string {
	switch t := t.(type) {
	case string:
		return t
	case json.Number:
		u, _ := t.Int64()
		return time.Unix(u, 0).Format(time.RFC3339)
	}
	return ""
}

func colorize(s interface{}, color int, enabled bool) string {
	if !enabled {
		return fmt.Sprintf("%v", s)
	}
	return fmt.Sprintf("\x1b[%dm%v\x1b[0m", color, s)
}

func levelColor(level string) int {
	switch level {
	case "debug":
		return cMagenta
	case "info":
		return cGreen
	case "warn":
		return cYellow
	case "error", "fatal", "panic":
		return cRed
	default:
		return cReset
	}
}

func needsQuote(s string) bool {
	for i := range s {
		if s[i] < 0x20 || s[i] > 0x7e || s[i] == ' ' || s[i] == '\\' || s[i] == '"' {
			return true
		}
	}
	return false
}
<commit_msg>Fix ConsoleWriter test<commit_after>package zerolog

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	cReset    = 0
	cBold     = 1
	cRed      = 31
	cGreen    = 32
	cYellow   = 33
	cBlue     = 34
	cMagenta  = 35
	cCyan     = 36
	cGray     = 37
	cDarkGray = 90
)

var consoleBufPool = sync.Pool{
	New: func() interface{} {
		return bytes.NewBuffer(make([]byte, 0, 100))
	},
}

// ConsoleWriter reads a JSON object per write operation and output an
// optionally colored human readable version on the Out writer.
type ConsoleWriter struct {
	Out     io.Writer
	NoColor bool
}

func (w ConsoleWriter) Write(p []byte) (n int, err error) {
	var event map[string]interface{}
	p = decodeIfBinaryToBytes(p)
	d := json.NewDecoder(bytes.NewReader(p))
	d.UseNumber()
	err = d.Decode(&event)
	if err != nil {
		return
	}
	buf := consoleBufPool.Get().(*bytes.Buffer)
	defer consoleBufPool.Put(buf)
	lvlColor := cReset
	level := "????"
	if l, ok := event[LevelFieldName].(string); ok {
		if !w.NoColor {
			lvlColor = levelColor(l)
		}
		level = strings.ToUpper(l)[0:4]
	}
	fmt.Fprintf(buf, "%s |%s| %s",
		colorize(formatTime(event[TimestampFieldName]), cDarkGray, !w.NoColor),
		colorize(level, lvlColor, !w.NoColor),
		colorize(event[MessageFieldName], cReset, !w.NoColor))
	fields := make([]string, 0, len(event))
	for field := range event {
		switch field {
		case LevelFieldName, TimestampFieldName, MessageFieldName:
			continue
		}
		fields = append(fields, field)
	}
	sort.Strings(fields)
	for _, field := range fields {
		fmt.Fprintf(buf, " %s=", colorize(field, cCyan, !w.NoColor))
		switch value := event[field].(type) {
		case string:
			if needsQuote(value) {
				buf.WriteString(strconv.Quote(value))
			} else {
				buf.WriteString(value)
			}
		case json.Number:
			fmt.Fprint(buf, value)
		default:
			b, err := json.Marshal(value)
			if err != nil {
				fmt.Fprintf(buf, "[error: %v]", err)
			} else {
				fmt.Fprint(buf, string(b))
			}
		}
	}
	buf.WriteByte('\n')
	buf.WriteTo(w.Out)
	n = len(p)
	return
}

func formatTime(t interface{}) string {
	switch t := t.(type) {
	case string:
		return t
	case json.Number:
		u, _ := t.Int64()
		return time.Unix(u, 0).Format(time.RFC3339)
	}
	return "<nil>"
}

func colorize(s interface{}, color int, enabled bool) string {
	if !enabled {
		return fmt.Sprintf("%v", s)
	}
	return fmt.Sprintf("\x1b[%dm%v\x1b[0m", color, s)
}

func levelColor(level string) int {
	switch level {
	case "debug":
		return cMagenta
	case "info":
		return cGreen
	case "warn":
		return cYellow
	case "error", "fatal", "panic":
		return cRed
	default:
		return cReset
	}
}

func needsQuote(s string) bool {
	for i := range s {
		if s[i] < 0x20 || s[i] > 0x7e || s[i] == ' ' || s[i] == '\\' || s[i] == '"' {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package ugen

// A Spreader takes a single UGen and expands its channels.
// Simplest case is a single channel UGen expanded to multiple channels. Each output buffer is identical.
// Less simple is mindless expansion of a two channel UGen to five. 0 1 -> 0 1 0 1 0. Mindless.
type Spreader struct {
	UGenBase
	quitchan chan q
}

func init() {
	var _ UGen = NewSpreader(1)
}

func NewSpreader(out int) *Spreader {
	var s Spreader
	prepareoutchans(&(s.UGenBase), out)
	s.inputs = make([]UGen, 1)
	s.quitchan = make(chan q)
	return &s
}

func NewSpreaderWithUGen(out int, u UGen) *Spreader {
	s := NewSpreader(out)
	s.SetInput(0, u)
	return s
}

func (s *Spreader) Stop() error {
	s.quitchan <- q{}
	for _, u := range s.inputs {
		u.Stop()
	}
	return nil
}

func (s *Spreader) GetParams() []ParamDesc {
	return []ParamDesc{}
}

func (s *Spreader) Start(op OutputParams) error {
	for _, u := range s.inputs {
		u.Start(op)
	}
	go func() {
		for {
			// fetch all input buffers
			bufs := make([][]float32, len(s.inputs[0].OutputChannels()))
			for i := range bufs {
				bufs[i] = GetNewBuf(op)
				select {
				case <-s.quitchan:
					return
				case bufs[i] = <- s.inputs[0].OutputChannels()[i]:
				}
			}

			func() {
				first := true
				var inum int
				for _, oc := range s.outchans {
					// logger.Println("i", i, oc)
					if first {
						// logger.Println("zing")
						defer func(i int, oc chan []float32) {
//							logger.Println("send first")
							oc <- bufs[i]
						}(inum, oc)
					} else {
						// logger.Println("getting newbuf")
						nb := GetNewBuf(op)
						// logger.Println("got newbuf")
						copy(nb, bufs[inum])
						select {
						case <-s.quitchan:
							return
						case oc <- nb:
//							logger.Println("send n")
						}
					}

					inum++
					if inum == len(s.inputs[0].OutputChannels()) {
						first = false
						inum = 0
					}
				}
			}()
		}
	}()
	return nil
}
<commit_msg>first unit test<commit_after>package ugen

// A Spreader takes a single UGen and expands its channels.
// Simplest case is a single channel UGen expanded to multiple channels. Each output buffer is identical.
// Less simple is mindless expansion of a two channel UGen to five. 0 1 -> 0 1 0 1 0. Mindless.
type Spreader struct {
	UGenBase
	quitchan chan q
}

func init() {
	var _ UGen = NewSpreader(1)
}

func NewSpreader(out int) *Spreader {
	var s Spreader
	prepareoutchans(&(s.UGenBase), out)
	s.inputs = make([]UGen, 1)
	s.quitchan = make(chan q)
	return &s
}

func NewSpreaderWithUGen(out int, u UGen) *Spreader {
	s := NewSpreader(out)
	s.SetInput(0, u)
	return s
}

func (s *Spreader) Stop() error {
	s.quitchan <- q{}
	for _, u := range s.inputs {
		u.Stop()
	}
	return nil
}

func (s *Spreader) GetParams() []ParamDesc {
	return []ParamDesc{}
}

func (s *Spreader) Start(op OutputParams) error {
	for _, u := range s.inputs {
		u.Start(op)
	}
	go func() {
		for {
			// fetch all input buffers
			bufs := make([][]float32, len(s.inputs[0].OutputChannels()))
			for i := range bufs {
				bufs[i] = GetNewBuf(op)
				select {
				case <-s.quitchan:
					return
				case bufs[i] = <- s.inputs[0].OutputChannels()[i]:
				}
			}

			func() {
				first := true
				var inum int
				for _, oc := range s.outchans {
					// logger.Println("i", inum, oc)
					if first {
						// logger.Println("zing")
						defer func(i int, oc chan []float32) {
							// logger.Println("send first")
							oc <- bufs[i]
						}(inum, oc)
					} else {
						// logger.Println("getting newbuf")
						nb := GetNewBuf(op)
						// logger.Println("got newbuf")
						copy(nb, bufs[inum])
						select {
						case <-s.quitchan:
							return
						case oc <- nb:
							// logger.Println("send n",inum)
						}
					}

					inum++
					if inum == len(s.inputs[0].OutputChannels()) {
						first = false
						inum = 0
					}
				}
			}()
		}
	}()
	return nil
}
<|endoftext|>


<commit_before>package perceptron

import (
	"errors"

	"github.com/mitsuse/perceptron-go/matrix"
)

type Classifier struct {
	model *Model
}

func NewClassifier(size int, indexer Indexer) *Classifier {
	c := &Classifier{
		model: &Model{
			weight:  matrix.NewZeroDense(size, 0),
			indexer: indexer,
		},
	}

	return c
}

func (c *Classifier) Update(learner Learner, instance Instance) (int, error) {
	feature := c.model.Extract(instance, true)

	score := c.model.Score(feature)
	if score.IsUndefined() {
		// TODO: Write the error message.
		return 0, errors.New("")
	}

	label, _, _, err := score.Max()
	if err != nil {
		return 0, err
	}

	if label != instance.Label() {
		example := c.getUpdate(instance.Label(), feature)
		inference := c.getUpdate(label, feature)
		return label, learner.Learn(c.model, example, inference)
	}

	return label, nil
}

func (c *Classifier) getUpdate(label int, feature matrix.Matrix) matrix.Matrix {
	return nil
}

func (c *Classifier) Classify(instance Instance) (int, error) {
	feature := c.model.Extract(instance, false)

	score := c.model.Score(feature)
	if score.IsUndefined() {
		// TODO: Write the error message.
		return 0, errors.New("")
	}

	label, _, _, err := score.Max()
	if err != nil {
		return 0, err
	}

	return label, nil
}

type Indexer interface {
	Size() int
	Index(identifier []int32, indexed bool) int
}
<commit_msg>Get the update matrix for multi-class perceptron learning.<commit_after>package perceptron

import (
	"errors"

	"github.com/mitsuse/perceptron-go/matrix"
)

type Classifier struct {
	model *Model
}

func NewClassifier(size int, indexer Indexer) *Classifier {
	c := &Classifier{
		model: &Model{
			weight:  matrix.NewZeroDense(size, 0),
			indexer: indexer,
		},
	}

	return c
}

func (c *Classifier) Update(learner Learner, instance Instance) (int, error) {
	feature := c.model.Extract(instance, true)

	score := c.model.Score(feature)
	if score.IsUndefined() {
		// TODO: Write the error message.
		return 0, errors.New("")
	}

	label, _, _, err := score.Max()
	if err != nil {
		return 0, err
	}

	if label != instance.Label() {
		example := c.getUpdate(instance.Label(), feature)
		inference := c.getUpdate(label, feature)
		return label, learner.Learn(c.model, example, inference)
	}

	return label, nil
}

func (c *Classifier) getUpdate(label int, feature matrix.Matrix) matrix.Matrix {
	rows, columns := c.model.Weight().Shape()
	update := matrix.NewZeroDense(rows, columns)

	iter := feature.NonZeros()
	for iter.HasNext() {
		id, _, value := iter.Get()
		update.Update(label, id, value)
	}

	return update
}

func (c *Classifier) Classify(instance Instance) (int, error) {
	feature := c.model.Extract(instance, false)

	score := c.model.Score(feature)
	if score.IsUndefined() {
		// TODO: Write the error message.
		return 0, errors.New("")
	}

	label, _, _, err := score.Max()
	if err != nil {
		return 0, err
	}

	return label, nil
}

type Indexer interface {
	Size() int
	Index(identifier []int32, indexed bool) int
}
<|endoftext|>


<commit_before>package cmdctrl

import (
	"fmt"
	"net/http"
	"time"

	"github.com/inconshreveable/go-update"
)

func NewGithubUpdater(repo, org, binName, target, canaryFile, currentVersion string) *GithubUpdater {
	return &GithubUpdater{
		repo:       repo,
		org:        org,
		binName:    binName,
		target:     target,
		canaryFile: canaryFile,
		version:    currentVersion,
	}
}

type GithubUpdater struct {
	repo       string
	org        string
	binName    string
	target     string
	canaryFile string
	version    string
}

// WriteCanary writes out the canary file so future instances
// know we're currently running a working instance.
func (g *GithubUpdater) WriteCanary() error {
	return nil
}

// RemoveCanary removes the canary file so future instances
// of the application know the upgrade is broken and the binary
// should roll back.
func (g *GithubUpdater) RemoveCanary() error {
	return nil
}

// CanaryCheck check's to see if the canary file exists.
// If it does not we'll try to roll back.
func (g *GithubUpdater) CanaryCheck() error {
	return nil
}

func (g *GithubUpdater) Upgrade(version string) error {
	url := fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", g.org, g.repo, version, g.binName)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	opts := update.Options{
		TargetPath:  g.target,
		OldSavePath: fmt.Sprintf("%s.%s.%d", g.target, g.version, time.Now().Unix()),
	}
	err = update.Apply(resp.Body, opts)
	if err != nil {
		if rerr := update.RollbackError(err); rerr != nil {
			fmt.Printf("Failed to rollback from bad update: %v\n", rerr)
		}
	}
	return err
}

func (g *GithubUpdater) Downgrade(version string) error {
	url := fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", g.org, g.repo, version, g.binName)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	opts := update.Options{
		TargetPath:  g.target,
		OldSavePath: fmt.Sprintf("%s.%s.%d", g.target, g.version, time.Now().Unix()),
	}
	err = update.Apply(resp.Body, opts)
	if err != nil {
		if rerr := update.RollbackError(err); rerr != nil {
			fmt.Printf("Failed to rollback from bad update: %v\n", rerr)
		}
	}
	return err
}
<commit_msg>add GetCurrentVersion<commit_after>package cmdctrl

import (
	"fmt"
	"net/http"
	"sync"
	"time"

	"github.com/inconshreveable/go-update"
)

func NewGithubUpdater(repo, org, binName, target, canaryFile, currentVersion string) *GithubUpdater {
	return &GithubUpdater{
		repo:       repo,
		org:        org,
		binName:    binName,
		target:     target,
		canaryFile: canaryFile,
		version:    currentVersion,
	}
}

type GithubUpdater struct {
	sync.RWMutex
	repo       string
	org        string
	binName    string
	target     string
	canaryFile string
	version    string
}

// WriteCanary writes out the canary file so future instances
// know we're currently running a working instance.
func (g *GithubUpdater) WriteCanary() error {
	return nil
}

// RemoveCanary removes the canary file so future instances
// of the application know the upgrade is broken and the binary
// should roll back.
func (g *GithubUpdater) RemoveCanary() error {
	return nil
}

// CanaryCheck check's to see if the canary file exists.
// If it does not we'll try to roll back.
func (g *GithubUpdater) CanaryCheck() error {
	return nil
}

//Upgrade to a new version
func (g *GithubUpdater) Upgrade(version string) error {
	g.Lock()
	defer g.Unlock()
	url := fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", g.org, g.repo, version, g.binName)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	opts := update.Options{
		TargetPath:  g.target,
		OldSavePath: fmt.Sprintf("%s.%s.%d", g.target, g.version, time.Now().Unix()),
	}
	err = update.Apply(resp.Body, opts)
	if err != nil {
		if rerr := update.RollbackError(err); rerr != nil {
			fmt.Printf("Failed to rollback from bad update: %v\n", rerr)
		}
	}
	return err
}

//Downgrade technically identical to upgrade
func (g *GithubUpdater) Downgrade(version string) error {
	g.Lock()
	defer g.Unlock()
	url := fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", g.org, g.repo, version, g.binName)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	opts := update.Options{
		TargetPath:  g.target,
		OldSavePath: fmt.Sprintf("%s.%s.%d", g.target, g.version, time.Now().Unix()),
	}
	err = update.Apply(resp.Body, opts)
	if err != nil {
		if rerr := update.RollbackError(err); rerr != nil {
			fmt.Printf("Failed to rollback from bad update: %v\n", rerr)
		}
	}
	return err
}

//GetCurrentVersion returns the current version string
func (g *GithubUpdater) GetCurrentVersion() string {
	g.RLock()
	defer g.RUnlock()
	return g.version
}
<|endoftext|>


<commit_before>package kingpin

import (
	"fmt"
	"strconv"
	"strings"
)

// Data model for Kingpin command-line structure.

type FlagGroupModel struct {
	Flags []*FlagModel
}

func (f *FlagGroupModel) FlagSummary() string {
	out := []string{}
	count := 0
	for _, flag := range f.Flags {
		if flag.Name != "help" {
			count++
		}
		if flag.Required {
			if flag.IsBoolFlag() {
				out = append(out, fmt.Sprintf("--[no-]%s", flag.Name))
			} else {
				out = append(out, fmt.Sprintf("--%s=%s", flag.Name, flag.FormatPlaceHolder()))
			}
		}
	}
	if count != len(out) {
		out = append(out, "[<flags>]")
	}
	return strings.Join(out, " ")
}

type FlagModel struct {
	Name        string
	Help        string
	Short       rune
	Default     []string
	Envar       string
	PlaceHolder string
	Required    bool
	Hidden      bool
	Value       Value
}

func (f *FlagModel) String() string {
	return f.Value.String()
}

func (f *FlagModel) IsBoolFlag() bool {
	if fl, ok := f.Value.(boolFlag); ok {
		return fl.IsBoolFlag()
	}
	return false
}

func (f *FlagModel) FormatPlaceHolder() string {
	if f.PlaceHolder != "" {
		return f.PlaceHolder
	}
	if len(f.Default) > 0 {
		ellipsis := ""
		if len(f.Default) > 1 {
			ellipsis = "..."
		}
		if _, ok := f.Value.(*stringValue); ok {
			return strconv.Quote(f.Default[0]) + ellipsis
		}
		return f.Default[0] + ellipsis
	}
	return strings.ToUpper(f.Name)
}

type ArgGroupModel struct {
	Args []*ArgModel
}

func (a *ArgGroupModel) ArgSummary() string {
	depth := 0
	out := []string{}
	for _, arg := range a.Args {
		h := "<" + arg.Name + ">"
		if !arg.Required {
			h = "[" + h
			depth++
		}
		out = append(out, h)
	}
	out[len(out)-1] = out[len(out)-1] + strings.Repeat("]", depth)
	return strings.Join(out, " ")
}

type ArgModel struct {
	Name     string
	Help     string
	Default  []string
	Envar    string
	Required bool
	Value    Value
}

func (a *ArgModel) String() string {
	return a.Value.String()
}

type CmdGroupModel struct {
	Commands []*CmdModel
}

func (c *CmdGroupModel) FlattenedCommands() (out []*CmdModel) {
	for _, cmd := range c.Commands {
		if len(cmd.Commands) == 0 {
			out = append(out, cmd)
		}
		out = append(out, cmd.FlattenedCommands()...)
	}
	return
}

type CmdModel struct {
	Name        string
	Aliases     []string
	Help        string
	FullCommand string
	Depth       int
	Hidden      bool
	Default     bool
	*FlagGroupModel
	*ArgGroupModel
	*CmdGroupModel
}

func (c *CmdModel) String() string {
	return c.FullCommand
}

type ApplicationModel struct {
	Name    string
	Help    string
	Version string
	Author  string
	*ArgGroupModel
	*CmdGroupModel
	*FlagGroupModel
}

func (a *Application) Model() *ApplicationModel {
	return &ApplicationModel{
		Name:           a.Name,
		Help:           a.Help,
		Version:        a.version,
		Author:         a.author,
		FlagGroupModel: a.flagGroup.Model(),
		ArgGroupModel:  a.argGroup.Model(),
		CmdGroupModel:  a.cmdGroup.Model(),
	}
}

func (a *argGroup) Model() *ArgGroupModel {
	m := &ArgGroupModel{}
	for _, arg := range a.args {
		m.Args = append(m.Args, arg.Model())
	}
	return m
}

func (a *ArgClause) Model() *ArgModel {
	return &ArgModel{
		Name:     a.name,
		Help:     a.help,
		Default:  a.defaultValues,
		Envar:    a.envar,
		Required: a.required,
		Value:    a.value,
	}
}

func (f *flagGroup) Model() *FlagGroupModel {
	m := &FlagGroupModel{}
	for _, fl := range f.flagOrder {
		m.Flags = append(m.Flags, fl.Model())
	}
	return m
}

func (f *FlagClause) Model() *FlagModel {
	return &FlagModel{
		Name:        f.name,
		Help:        f.help,
		Short:       rune(f.shorthand),
		Default:     f.defaultValues,
		Envar:       f.envar,
		PlaceHolder: f.placeholder,
		Required:    f.required,
		Hidden:      f.hidden,
		Value:       f.value,
	}
}

func (c *cmdGroup) Model() *CmdGroupModel {
	m := &CmdGroupModel{}
	for _, cm := range c.commandOrder {
		m.Commands = append(m.Commands, cm.Model())
	}
	return m
}

func (c *CmdClause) Model() *CmdModel {
	depth := 0
	for i := c; i != nil; i = i.parent {
		depth++
	}
	return &CmdModel{
		Name:           c.name,
		Aliases:        c.aliases,
		Help:           c.help,
		Depth:          depth,
		Hidden:         c.hidden,
		Default:        c.isDefault,
		FullCommand:    c.FullCommand(),
		FlagGroupModel: c.flagGroup.Model(),
		ArgGroupModel:  c.argGroup.Model(),
		CmdGroupModel:  c.cmdGroup.Model(),
	}
}
<commit_msg>Do not display additional-flags token in usage when all flags are required<commit_after>package kingpin

import (
	"fmt"
	"strconv"
	"strings"
)

// Data model for Kingpin command-line structure.

type FlagGroupModel struct {
	Flags []*FlagModel
}

func (f *FlagGroupModel) FlagSummary() string {
	out := []string{}
	count := 0
	for _, flag := range f.Flags {
		if !strings.HasPrefix(flag.Name, "help") && !strings.HasPrefix(flag.Name, "completion") {
			count++
		}
		if flag.Required {
			if flag.IsBoolFlag() {
				out = append(out, fmt.Sprintf("--[no-]%s", flag.Name))
			} else {
				out = append(out, fmt.Sprintf("--%s=%s", flag.Name, flag.FormatPlaceHolder()))
			}
		}
	}
	if count != len(out) {
		out = append(out, "[<flags>]")
	}
	return strings.Join(out, " ")
}

type FlagModel struct {
	Name        string
	Help        string
	Short       rune
	Default     []string
	Envar       string
	PlaceHolder string
	Required    bool
	Hidden      bool
	Value       Value
}

func (f *FlagModel) String() string {
	return f.Value.String()
}

func (f *FlagModel) IsBoolFlag() bool {
	if fl, ok := f.Value.(boolFlag); ok {
		return fl.IsBoolFlag()
	}
	return false
}

func (f *FlagModel) FormatPlaceHolder() string {
	if f.PlaceHolder != "" {
		return f.PlaceHolder
	}
	if len(f.Default) > 0 {
		ellipsis := ""
		if len(f.Default) > 1 {
			ellipsis = "..."
		}
		if _, ok := f.Value.(*stringValue); ok {
			return strconv.Quote(f.Default[0]) + ellipsis
		}
		return f.Default[0] + ellipsis
	}
	return strings.ToUpper(f.Name)
}

type ArgGroupModel struct {
	Args []*ArgModel
}

func (a *ArgGroupModel) ArgSummary() string {
	depth := 0
	out := []string{}
	for _, arg := range a.Args {
		h := "<" + arg.Name + ">"
		if !arg.Required {
			h = "[" + h
			depth++
		}
		out = append(out, h)
	}
	out[len(out)-1] = out[len(out)-1] + strings.Repeat("]", depth)
	return strings.Join(out, " ")
}

type ArgModel struct {
	Name     string
	Help     string
	Default  []string
	Envar    string
	Required bool
	Value    Value
}

func (a *ArgModel) String() string {
	return a.Value.String()
}

type CmdGroupModel struct {
	Commands []*CmdModel
}

func (c *CmdGroupModel) FlattenedCommands() (out []*CmdModel) {
	for _, cmd := range c.Commands {
		if len(cmd.Commands) == 0 {
			out = append(out, cmd)
		}
		out = append(out, cmd.FlattenedCommands()...)
	}
	return
}

type CmdModel struct {
	Name        string
	Aliases     []string
	Help        string
	FullCommand string
	Depth       int
	Hidden      bool
	Default     bool
	*FlagGroupModel
	*ArgGroupModel
	*CmdGroupModel
}

func (c *CmdModel) String() string {
	return c.FullCommand
}

type ApplicationModel struct {
	Name    string
	Help    string
	Version string
	Author  string
	*ArgGroupModel
	*CmdGroupModel
	*FlagGroupModel
}

func (a *Application) Model() *ApplicationModel {
	return &ApplicationModel{
		Name:           a.Name,
		Help:           a.Help,
		Version:        a.version,
		Author:         a.author,
		FlagGroupModel: a.flagGroup.Model(),
		ArgGroupModel:  a.argGroup.Model(),
		CmdGroupModel:  a.cmdGroup.Model(),
	}
}

func (a *argGroup) Model() *ArgGroupModel {
	m := &ArgGroupModel{}
	for _, arg := range a.args {
		m.Args = append(m.Args, arg.Model())
	}
	return m
}

func (a *ArgClause) Model() *ArgModel {
	return &ArgModel{
		Name:     a.name,
		Help:     a.help,
		Default:  a.defaultValues,
		Envar:    a.envar,
		Required: a.required,
		Value:    a.value,
	}
}

func (f *flagGroup) Model() *FlagGroupModel {
	m := &FlagGroupModel{}
	for _, fl := range f.flagOrder {
		m.Flags = append(m.Flags, fl.Model())
	}
	return m
}

func (f *FlagClause) Model() *FlagModel {
	return &FlagModel{
		Name:        f.name,
		Help:        f.help,
		Short:       rune(f.shorthand),
		Default:     f.defaultValues,
		Envar:       f.envar,
		PlaceHolder: f.placeholder,
		Required:    f.required,
		Hidden:      f.hidden,
		Value:       f.value,
	}
}

func (c *cmdGroup) Model() *CmdGroupModel {
	m := &CmdGroupModel{}
	for _, cm := range c.commandOrder {
		m.Commands = append(m.Commands, cm.Model())
	}
	return m
}

func (c *CmdClause) Model() *CmdModel {
	depth := 0
	for i := c; i != nil; i = i.parent {
		depth++
	}
	return &CmdModel{
		Name:           c.name,
		Aliases:        c.aliases,
		Help:           c.help,
		Depth:          depth,
		Hidden:         c.hidden,
		Default:        c.isDefault,
		FullCommand:    c.FullCommand(),
		FlagGroupModel: c.flagGroup.Model(),
		ArgGroupModel:  c.argGroup.Model(),
		CmdGroupModel:  c.cmdGroup.Model(),
	}
}
<|endoftext|>


<commit_before>// Package fate is a text generation library.
package fate

import (
	cryptorand "crypto/rand"
	"encoding/binary"
	"log"
	"math/rand"
	"strings"
	"unicode"

	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
)

type token uint32

type bigram struct {
	tok0, tok1 token
}

// Model is a trigram language model that can learn and respond to
// text.
type Model struct {
	tokens *syndict

	// We track (tok0 -> tok1) and (tok0 tok1 -> tok2) in
	// the forward direction, so we can efficiently choose
	// random tok0-containing contexts.
	fwd1 obs1
	fwd2 obs2

	// In the reverse direction, we only need to be able
	// to track (tok2 tok1 -> tok0).
	rev2 obs2

	rand *rand.Rand
}

// NewModel constructs an empty language model.
func NewModel() *Model {
	return &Model{
		tokens: newSyndict(normalizer()),

		fwd1: make(obs1),
		fwd2: make(obs2),

		rev2: make(obs2),

		rand: rand.New(randSource()),
	}
}

// normalize a word by removing all punctuation and lowercasing.
func normalizer() func(string) string {
	isRemovable := func(r rune) bool {
		return unicode.Is(unicode.Mn, r) || unicode.IsPunct(r)
	}

	trans := transform.Chain(norm.NFD, transform.RemoveFunc(isRemovable), norm.NFC)

	return func(word string) string {
		str, _, _ := transform.String(trans, word)
		return strings.ToLower(str)
	}
}

// randSource seeds a standard math/rand PRNG with a secure seed.
func randSource() rand.Source {
	var seed int64
	binary.Read(cryptorand.Reader, binary.LittleEndian, &seed)
	return rand.NewSource(seed)
}

func (m *Model) ends() (token, token) {
	return m.tokens.ID("<S>"), m.tokens.ID("</S>")
}

// Learn observes the text in a string and makes it available for
// later replies.
func (m *Model) Learn(text string) {
	start, end := m.ends()
	tokens := m.tokens

	// ids: <S> <S> tokens in the input string </S> </S>
	var ids = []token{start, start}
	for _, f := range strings.Fields(text) {
		ids = append(ids, tokens.ID(f))
	}
	ids = append(ids, end, end)

	var ctx bigram
	var tok2 token

	for i := 0; i < len(ids)-2; i++ {
		ctx.tok0, ctx.tok1, tok2 = ids[i], ids[i+1], ids[i+2]
		if !m.fwd2.Observe(ctx, tok2) {
			m.fwd1.Observe(ctx.tok0, ctx.tok1)
		}

		ctx.tok0, tok2 = tok2, ctx.tok0
		m.rev2.Observe(ctx, tok2)
	}
}

// Reply generates a reply string to str, given the current state of
// the language model.
func (m *Model) Reply(text string) string {
	start, end := m.ends()

	pivot := m.pickPivot(strings.Fields(text))

	// find a random context containing pivot
	fwdtoks := m.fwd1[pivot]

	fwdctx := bigram{tok0: pivot, tok1: m.choice(fwdtoks)}
	revctx := bigram{tok0: fwdctx.tok1, tok1: fwdctx.tok0}

	fwd := m.follow(m.fwd2, fwdctx, end)
	rev := m.follow(m.rev2, revctx, start)

	return m.join(rev, fwd[2:])
}

func (m *Model) pickPivot(words []string) token {
	var pivots []token
	for _, w := range words {
		pivots = append(pivots, m.tokens.Syns(w)...)
	}

	if len(pivots) > 0 {
		return pivots[m.rand.Intn(len(pivots))]
	}

	// No valid pivots, so babble.
	start, _ := m.ends()
	return start
}

func (m *Model) join(rev, fwd []token) string {
	var words = make([]string, 0, len(rev)+len(fwd))

	start, end := m.ends()
	add := func(tok token) {
		if tok != start && tok != end {
			words = append(words, m.tokens.Word(tok))
		}
	}

	for i := len(rev) - 1; i >= 0; i-- {
		add(rev[i])
	}

	for _, tok := range fwd {
		add(tok)
	}

	return strings.Join(words, " ")
}

func (m *Model) follow(obs obs2, pos bigram, end token) []token {
	ret := []token{pos.tok0, pos.tok1}

	if pos.tok1 == end {
		return ret
	}

	for {
		toks := obs[pos]
		if len(toks) == 0 {
			log.Fatal("ran out of chain")
		}

		tok := m.choice(toks)
		if tok == end {
			return ret
		}

		ret = append(ret, tok)
		pos.tok0, pos.tok1 = pos.tok1, tok
	}
}

func (m *Model) choice(toks []token) token {
	return toks[m.rand.Intn(len(toks))]
}
<commit_msg>Refuse to learn single word inputs for now<commit_after>// Package fate is a text generation library.
package fate

import (
	cryptorand "crypto/rand"
	"encoding/binary"
	"log"
	"math/rand"
	"strings"
	"unicode"

	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
)

type token uint32

type bigram struct {
	tok0, tok1 token
}

// Model is a trigram language model that can learn and respond to
// text.
type Model struct {
	tokens *syndict

	// We track (tok0 -> tok1) and (tok0 tok1 -> tok2) in
	// the forward direction, so we can efficiently choose
	// random tok0-containing contexts.
	fwd1 obs1
	fwd2 obs2

	// In the reverse direction, we only need to be able
	// to track (tok2 tok1 -> tok0).
	rev2 obs2

	rand *rand.Rand
}

// NewModel constructs an empty language model.
func NewModel() *Model {
	return &Model{
		tokens: newSyndict(normalizer()),

		fwd1: make(obs1),
		fwd2: make(obs2),

		rev2: make(obs2),

		rand: rand.New(randSource()),
	}
}

// normalize a word by removing all punctuation and lowercasing.
func normalizer() func(string) string {
	isRemovable := func(r rune) bool {
		return unicode.Is(unicode.Mn, r) || unicode.IsPunct(r)
	}

	trans := transform.Chain(norm.NFD, transform.RemoveFunc(isRemovable), norm.NFC)

	return func(word string) string {
		str, _, _ := transform.String(trans, word)
		return strings.ToLower(str)
	}
}

// randSource seeds a standard math/rand PRNG with a secure seed.
func randSource() rand.Source {
	var seed int64
	binary.Read(cryptorand.Reader, binary.LittleEndian, &seed)
	return rand.NewSource(seed)
}

func (m *Model) ends() (token, token) {
	return m.tokens.ID("<S>"), m.tokens.ID("</S>")
}

// Learn observes the text in a string and makes it available for
// later replies.
func (m *Model) Learn(text string) {
	start, end := m.ends()
	tokens := m.tokens

	words := strings.Fields(text)
	if len(words) == 1 {
		// Refuse to learn single-word inputs.
		return
	}

	// ids: <S> <S> tokens in the input string </S> </S>
	var ids = []token{start, start}
	for _, f := range words {
		ids = append(ids, tokens.ID(f))
	}
	ids = append(ids, end, end)

	var ctx bigram
	var tok2 token

	for i := 0; i < len(ids)-2; i++ {
		ctx.tok0, ctx.tok1, tok2 = ids[i], ids[i+1], ids[i+2]
		if !m.fwd2.Observe(ctx, tok2) {
			m.fwd1.Observe(ctx.tok0, ctx.tok1)
		}

		ctx.tok0, tok2 = tok2, ctx.tok0
		m.rev2.Observe(ctx, tok2)
	}
}

// Reply generates a reply string to str, given the current state of
// the language model.
func (m *Model) Reply(text string) string {
	start, end := m.ends()

	pivot := m.pickPivot(strings.Fields(text))

	// find a random context containing pivot
	fwdtoks := m.fwd1[pivot]

	fwdctx := bigram{tok0: pivot, tok1: m.choice(fwdtoks)}
	revctx := bigram{tok0: fwdctx.tok1, tok1: fwdctx.tok0}

	fwd := m.follow(m.fwd2, fwdctx, end)
	rev := m.follow(m.rev2, revctx, start)

	return m.join(rev, fwd[2:])
}

func (m *Model) pickPivot(words []string) token {
	var pivots []token
	for _, w := range words {
		pivots = append(pivots, m.tokens.Syns(w)...)
	}

	if len(pivots) > 0 {
		return pivots[m.rand.Intn(len(pivots))]
	}

	// No valid pivots, so babble.
	start, _ := m.ends()
	return start
}

func (m *Model) join(rev, fwd []token) string {
	var words = make([]string, 0, len(rev)+len(fwd))

	start, end := m.ends()
	add := func(tok token) {
		if tok != start && tok != end {
			words = append(words, m.tokens.Word(tok))
		}
	}

	for i := len(rev) - 1; i >= 0; i-- {
		add(rev[i])
	}

	for _, tok := range fwd {
		add(tok)
	}

	return strings.Join(words, " ")
}

func (m *Model) follow(obs obs2, pos bigram, end token) []token {
	ret := []token{pos.tok0, pos.tok1}

	if pos.tok1 == end {
		return ret
	}

	for {
		toks := obs[pos]
		if len(toks) == 0 {
			log.Fatal("ran out of chain")
		}

		tok := m.choice(toks)
		if tok == end {
			return ret
		}

		ret = append(ret, tok)
		pos.tok0, pos.tok1 = pos.tok1, tok
	}
}

func (m *Model) choice(toks []token) token {
	return toks[m.rand.Intn(len(toks))]
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package build

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/google/zoekt"
	"github.com/google/zoekt/query"
)

func TestBasic(t *testing.T) {
	dir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("TempDir: %v", err)
	}

	opts := Options{
		IndexDir:    dir,
		ShardMax:    1024,
		RepoName:    "repo",
		RepoURL:     "url",
		Parallelism: 2,
		SizeMax:     1 << 20,
	}

	b, err := NewBuilder(opts)
	if err != nil {
		t.Fatalf("NewBuilder: %v", err)
	}

	for i := 0; i < 4; i++ {
		s := fmt.Sprintf("%d", i)
		b.AddFile("F"+s, []byte(strings.Repeat(s, 1000)))
	}

	b.Finish()

	fs, _ := filepath.Glob(dir + "/*")
	if len(fs) <= 1 {
		t.Fatalf("want multiple shards, got %v", fs)
	}

	ss, err := zoekt.NewShardedSearcher(dir)
	if err != nil {
		t.Fatalf("NewShardedSearcher(%s): %v", dir, err)
	}

	q, err := query.Parse("111")
	if err != nil {
		t.Fatalf("Parse(111): %v", err)
	}

	var sOpts zoekt.SearchOptions
	ctx := context.Background()
	result, err := ss.Search(ctx, q, &sOpts)
	if err != nil {
		t.Fatalf("Parse(111): %v", err)
	}

	if len(result.Files) != 1 || result.Files[0].Name != "F1" {
		t.Errorf("got %v, want 1 file.", result.Files)
	}
	defer ss.Close()
}

func TestUpdate(t *testing.T) {
	dir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("TempDir: %v", err)
	}

	opts := Options{
		IndexDir:    dir,
		ShardMax:    1024,
		RepoName:    "repo",
		RepoURL:     "url",
		Parallelism: 2,
		SizeMax:     1 << 20,

		RepoDir: "/a",
	}

	if b, err := NewBuilder(opts); err != nil {
		t.Fatalf("NewBuilder: %v", err)
	} else {
		b.AddFile("F", []byte("hoi"))
		b.Finish()
	}
	ss, err := zoekt.NewShardedSearcher(dir)
	if err != nil {
		t.Fatalf("NewShardedSearcher(%s): %v", dir, err)
	}

	ctx := context.Background()
	repos, err := ss.List(ctx, &query.Repo{Pattern: "repo"})
	if err != nil {
		t.Fatalf("List: %v", err)
	}

	if len(repos.Repos) != 1 {
		t.Errorf("List(repo): got %v, want 1 repo", repos.Repos)
	}

	fs, err := filepath.Glob(filepath.Join(dir, "*"))
	if err != nil {
		t.Fatalf("glob: %v", err)
	}

	opts.RepoName = "repo2"
	opts.RepoURL = "url2"
	opts.RepoDir = "/b"
	if b, err := NewBuilder(opts); err != nil {
		t.Fatalf("NewBuilder: %v", err)
	} else {
		b.AddFile("F", []byte("hoi"))
		b.Finish()
	}

	// This is ugly, and potentially flaky, but there is no
	// observable synchronization for the Sharded searcher, so
	// this is the best we can do.
	time.Sleep(100 * time.Millisecond)

	ctx = context.Background()
	if repos, err = ss.List(ctx, &query.Repo{Pattern: "repo"}); err != nil {
		t.Fatalf("List: %v", err)
	} else if len(repos.Repos) != 2 {
		t.Errorf("List(repo): got %v, want 2 repos", repos.Repos)
	}

	for _, fn := range fs {
		log.Printf("removing %s", fn)
		if err := os.Remove(fn); err != nil {
			t.Fatalf("Remove(%s): %v", fn, err)
		}
		break
	}

	time.Sleep(100 * time.Millisecond)

	ctx = context.Background()
	if repos, err = ss.List(ctx, &query.Repo{Pattern: "repo"}); err != nil {
		t.Fatalf("List: %v", err)
	} else if len(repos.Repos) != 1 {
		t.Errorf("List(repo): got %v, want 1 repo", repos.Repos)
	}
}

func TestDeleteOldShards(t *testing.T) {
	dir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("TempDir: %v", err)
	}

	opts := Options{
		IndexDir: dir,
		ShardMax: 1024,
		RepoName: "repo",
		RepoURL:  "url",
		RepoDir:  "/a",
		SizeMax:  1 << 20,
	}
	opts.SetDefaults()

	b, err := NewBuilder(opts)
	if err != nil {
		t.Fatalf("NewBuilder: %v", err)
	}
	for i := 0; i < 4; i++ {
		s := fmt.Sprintf("%d", i)
		b.AddFile("F"+s, []byte(strings.Repeat(s, 1024)))
	}
	b.Finish()

	glob := filepath.Join(dir, "*")
	fs, err := filepath.Glob(glob)
	if err != nil {
		t.Fatalf("Glob(%s): %v", glob, err)
	} else if len(fs) != 4 {
		t.Fatalf("Glob(%s): got %v, want 4 shards", glob, fs)
	}

	// Do again, without sharding.
	opts.ShardMax = 1 << 20
	b, err = NewBuilder(opts)
	if err != nil {
		t.Fatalf("NewBuilder: %v", err)
	}
	for i := 0; i < 4; i++ {
		s := fmt.Sprintf("%d", i)
		b.AddFile("F"+s, []byte(strings.Repeat(s, 1024)))
	}
	b.Finish()

	fs, err = filepath.Glob(glob)
	if err != nil {
		t.Fatalf("Glob(%s): %v", glob, err)
	} else if len(fs) != 1 {
		t.Fatalf("Glob(%s): got %v, want 1 shard", glob, fs)
	}

	// Again, but don't index anything; should leave old shards intact.
	b, err = NewBuilder(opts)
	if err != nil {
		t.Fatalf("NewBuilder: %v", err)
	}
	b.Finish()

	fs, err = filepath.Glob(glob)
	if err != nil {
		t.Fatalf("Glob(%s): %v", glob, err)
	} else if len(fs) != 1 {
		t.Fatalf("Glob(%s): got %v, want 1 shard", glob, fs)
	}
}
<commit_msg>Fix e2e test by setting repodir.<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package build

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/google/zoekt"
	"github.com/google/zoekt/query"
)

func TestBasic(t *testing.T) {
	dir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("TempDir: %v", err)
	}

	opts := Options{
		IndexDir:    dir,
		ShardMax:    1024,
		RepoName:    "repo",
		RepoDir:     "/repo",
		RepoURL:     "url",
		Parallelism: 2,
		SizeMax:     1 << 20,
	}

	b, err := NewBuilder(opts)
	if err != nil {
		t.Fatalf("NewBuilder: %v", err)
	}

	for i := 0; i < 4; i++ {
		s := fmt.Sprintf("%d", i)
		b.AddFile("F"+s, []byte(strings.Repeat(s, 1000)))
	}

	b.Finish()

	fs, _ := filepath.Glob(dir + "/*")
	if len(fs) <= 1 {
		t.Fatalf("want multiple shards, got %v", fs)
	}

	ss, err := zoekt.NewShardedSearcher(dir)
	if err != nil {
		t.Fatalf("NewShardedSearcher(%s): %v", dir, err)
	}

	q, err := query.Parse("111")
	if err != nil {
		t.Fatalf("Parse(111): %v", err)
	}

	var sOpts zoekt.SearchOptions
	ctx := context.Background()
	result, err := ss.Search(ctx, q, &sOpts)
	if err != nil {
		t.Fatalf("Parse(111): %v", err)
	}

	if len(result.Files) != 1 || result.Files[0].Name != "F1" {
		t.Errorf("got %v, want 1 file.", result.Files)
	}
	defer ss.Close()
}

func TestUpdate(t *testing.T) {
	dir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("TempDir: %v", err)
	}

	opts := Options{
		IndexDir:    dir,
		ShardMax:    1024,
		RepoName:    "repo",
		RepoURL:     "url",
		Parallelism: 2,
		SizeMax:     1 << 20,

		RepoDir: "/a",
	}

	if b, err := NewBuilder(opts); err != nil {
		t.Fatalf("NewBuilder: %v", err)
	} else {
		b.AddFile("F", []byte("hoi"))
		b.Finish()
	}
	ss, err := zoekt.NewShardedSearcher(dir)
	if err != nil {
		t.Fatalf("NewShardedSearcher(%s): %v", dir, err)
	}

	ctx := context.Background()
	repos, err := ss.List(ctx, &query.Repo{Pattern: "repo"})
	if err != nil {
		t.Fatalf("List: %v", err)
	}

	if len(repos.Repos) != 1 {
		t.Errorf("List(repo): got %v, want 1 repo", repos.Repos)
	}

	fs, err := filepath.Glob(filepath.Join(dir, "*"))
	if err != nil {
		t.Fatalf("glob: %v", err)
	}

	opts.RepoName = "repo2"
	opts.RepoURL = "url2"
	opts.RepoDir = "/b"
	if b, err := NewBuilder(opts); err != nil {
		t.Fatalf("NewBuilder: %v", err)
	} else {
		b.AddFile("F", []byte("hoi"))
		b.Finish()
	}

	// This is ugly, and potentially flaky, but there is no
	// observable synchronization for the Sharded searcher, so
	// this is the best we can do.
	time.Sleep(100 * time.Millisecond)

	ctx = context.Background()
	if repos, err = ss.List(ctx, &query.Repo{Pattern: "repo"}); err != nil {
		t.Fatalf("List: %v", err)
	} else if len(repos.Repos) != 2 {
		t.Errorf("List(repo): got %v, want 2 repos", repos.Repos)
	}

	for _, fn := range fs {
		log.Printf("removing %s", fn)
		if err := os.Remove(fn); err != nil {
			t.Fatalf("Remove(%s): %v", fn, err)
		}
		break
	}

	time.Sleep(100 * time.Millisecond)

	ctx = context.Background()
	if repos, err = ss.List(ctx, &query.Repo{Pattern: "repo"}); err != nil {
		t.Fatalf("List: %v", err)
	} else if len(repos.Repos) != 1 {
		t.Errorf("List(repo): got %v, want 1 repo", repos.Repos)
	}
}

func TestDeleteOldShards(t *testing.T) {
	dir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("TempDir: %v", err)
	}

	opts := Options{
		IndexDir: dir,
		ShardMax: 1024,
		RepoName: "repo",
		RepoURL:  "url",
		RepoDir:  "/a",
		SizeMax:  1 << 20,
	}
	opts.SetDefaults()

	b, err := NewBuilder(opts)
	if err != nil {
		t.Fatalf("NewBuilder: %v", err)
	}
	for i := 0; i < 4; i++ {
		s := fmt.Sprintf("%d", i)
		b.AddFile("F"+s, []byte(strings.Repeat(s, 1024)))
	}
	b.Finish()

	glob := filepath.Join(dir, "*")
	fs, err := filepath.Glob(glob)
	if err != nil {
		t.Fatalf("Glob(%s): %v", glob, err)
	} else if len(fs) != 4 {
		t.Fatalf("Glob(%s): got %v, want 4 shards", glob, fs)
	}

	// Do again, without sharding.
	opts.ShardMax = 1 << 20
	b, err = NewBuilder(opts)
	if err != nil {
		t.Fatalf("NewBuilder: %v", err)
	}
	for i := 0; i < 4; i++ {
		s := fmt.Sprintf("%d", i)
		b.AddFile("F"+s, []byte(strings.Repeat(s, 1024)))
	}
	b.Finish()

	fs, err = filepath.Glob(glob)
	if err != nil {
		t.Fatalf("Glob(%s): %v", glob, err)
	} else if len(fs) != 1 {
		t.Fatalf("Glob(%s): got %v, want 1 shard", glob, fs)
	}

	// Again, but don't index anything; should leave old shards intact.
	b, err = NewBuilder(opts)
	if err != nil {
		t.Fatalf("NewBuilder: %v", err)
	}
	b.Finish()

	fs, err = filepath.Glob(glob)
	if err != nil {
		t.Fatalf("Glob(%s): %v", glob, err)
	} else if len(fs) != 1 {
		t.Fatalf("Glob(%s): got %v, want 1 shard", glob, fs)
	}
}
<|endoftext|>