<commit_before>package datalayer

import (
	"context"
	"fmt"
	"time"

	"github.com/dollarshaveclub/furan/generated/lib"
	"github.com/dollarshaveclub/furan/lib/db"
	"github.com/gocql/gocql"
	"github.com/golang/protobuf/proto"
	gocqltrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/gocql/gocql"
)

// DataLayer describes an object that interacts with the persistent data store
type DataLayer interface {
	CreateBuild(context.Context, *lib.BuildRequest) (gocql.UUID, error)
	GetBuildByID(context.Context, gocql.UUID) (*lib.BuildStatusResponse, error)
	SetBuildFlags(context.Context, gocql.UUID, map[string]bool) error
	SetBuildCompletedTimestamp(context.Context, gocql.UUID) error
	SetBuildState(context.Context, gocql.UUID, lib.BuildStatusResponse_BuildState) error
	DeleteBuild(context.Context, gocql.UUID) error
	SetBuildTimeMetric(context.Context, gocql.UUID, string) error
	SetDockerImageSizesMetric(context.Context, gocql.UUID, int64, int64) error
	SaveBuildOutput(context.Context, gocql.UUID, []lib.BuildEvent, string) error
	GetBuildOutput(context.Context, gocql.UUID, string) ([]lib.BuildEvent, error)
}

// DBLayer is an DataLayer instance that interacts with the Cassandra database
type DBLayer struct {
	s     *gocql.Session
	sname string
}

// NewDBLayer returns a data layer object
func NewDBLayer(s *gocql.Session, sname string) *DBLayer {
	return &DBLayer{s: s, sname: sname}
}

func (dl *DBLayer) wrapQuery(ctx context.Context, query *gocql.Query) *gocqltrace.Query {
	return gocqltrace.WrapQuery(query, gocqltrace.WithServiceName(dl.sname)).WithContext(ctx)
}

// CreateBuild inserts a new build into the DB returning the ID
func (dl *DBLayer) CreateBuild(ctx context.Context, req *lib.BuildRequest) (id gocql.UUID, err error) {
	q := `INSERT INTO builds_by_id (id, request, state, finished, failed, cancelled, started)
        VALUES (?,{github_repo: ?, dockerfile_path: ?, tags: ?, tag_with_commit_sha: ?, ref: ?,
					push_registry_repo: ?, push_s3_region: ?, push_s3_bucket: ?,
					push_s3_key_prefix: ?},?,?,?,?,?);`
	id, err = gocql.RandomUUID()
	if err != nil {
		return id, err
	}
	udt := db.UDTFromBuildRequest(req)
	query := dl.s.Query(q, id, udt.GithubRepo, udt.DockerfilePath, udt.Tags, udt.TagWithCommitSha, udt.Ref,
		udt.PushRegistryRepo, udt.PushS3Region, udt.PushS3Bucket, udt.PushS3KeyPrefix,
		lib.BuildStatusResponse_STARTED.String(), false, false, false, time.Now())
	err = dl.wrapQuery(ctx, query).Exec()
	if err != nil {
		return id, err
	}
	q = `INSERT INTO build_metrics_by_id (id) VALUES (?);`
	err = dl.wrapQuery(ctx, dl.s.Query(q, id)).Exec()
	if err != nil {
		return id, err
	}
	q = `INSERT INTO build_events_by_id (id) VALUES (?);`
	return id, dl.wrapQuery(ctx, dl.s.Query(q, id)).Exec()
}

// GetBuildByID fetches a build object from the DB
func (dl *DBLayer) GetBuildByID(ctx context.Context, id gocql.UUID) (bi *lib.BuildStatusResponse, err error) {
	q := `SELECT request, state, finished, failed, cancelled, started, completed,
	      duration FROM builds_by_id WHERE id = ?;`
	var udt db.BuildRequestUDT
	var state string
	var started, completed time.Time
	bi = &lib.BuildStatusResponse{
		BuildId: id.String(),
	}
	query := dl.s.Query(q, id)
	err = dl.wrapQuery(ctx, query).Scan(&udt.GithubRepo, &udt.DockerfilePath, udt.Tags, udt.TagWithCommitSha, udt.Ref,
		udt.PushRegistryRepo, udt.PushS3Region, udt.PushS3Bucket, udt.PushS3KeyPrefix, &state, &bi.Finished, &bi.Failed,
		&bi.Cancelled, &started, &completed, &bi.Duration)
	if err != nil {
		return bi, err
	}
	bi.State = db.BuildStateFromString(state)
	bi.BuildRequest = db.BuildRequestFromUDT(&udt)
	bi.Started = started.Format(time.RFC3339)
	bi.Completed = completed.Format(time.RFC3339)
	return bi, nil
}

// SetBuildFlags sets the boolean flags on the build object
// Caller must ensure that the flags passed in are valid
func (dl *DBLayer) SetBuildFlags(ctx context.Context, id gocql.UUID, flags map[string]bool) (err error) {
	q := `UPDATE builds_by_id SET %v = ? WHERE id = ?;`
	for k, v := range flags {
		err = dl.wrapQuery(ctx, dl.s.Query(fmt.Sprintf(q, k), v, id)).Exec()
		if err != nil {
			return err
		}
	}
	return nil
}

// SetBuildCompletedTimestamp sets the completed timestamp on a build to time.Now()
func (dl *DBLayer) SetBuildCompletedTimestamp(ctx context.Context, id gocql.UUID) (err error) {
	var started time.Time
	now := time.Now()
	q := `SELECT started FROM builds_by_id WHERE id = ?;`
	err = dl.wrapQuery(ctx, dl.s.Query(q, id)).Scan(&started)
	if err != nil {
		return err
	}
	duration := now.Sub(started).Seconds()
	q = `UPDATE builds_by_id SET completed = ?, duration = ? WHERE id = ?;`
	return dl.s.Query(q, now, duration, id).Exec()
}

// SetBuildState sets the state of a build
func (dl *DBLayer) SetBuildState(ctx context.Context, id gocql.UUID, state lib.BuildStatusResponse_BuildState) (err error) {
	q := `UPDATE builds_by_id SET state = ? WHERE id = ?;`
	return dl.wrapQuery(ctx, dl.s.Query(q, state.String(), id)).Exec()
}

// DeleteBuild removes a build from the DB.
// Only used in case of queue full when we can't actually do a build
func (dl *DBLayer) DeleteBuild(ctx context.Context, id gocql.UUID) (err error) {
	q := `DELETE FROM builds_by_id WHERE id = ?;`
	err = dl.wrapQuery(ctx, dl.s.Query(q, id)).Exec()
	if err != nil {
		return err
	}
	q = `DELETE FROM build_metrics_by_id WHERE id = ?;`
	return dl.s.Query(q, id).Exec()
}

// SetBuildTimeMetric sets a build metric to time.Now()
// metric is the name of the column to update
// if metric is a *_completed column, it will also compute and persist the duration
func (dl *DBLayer) SetBuildTimeMetric(ctx context.Context, id gocql.UUID, metric string) (err error) {
	var started time.Time
	now := time.Now()
	getstarted := true
	var startedcolumn string
	var durationcolumn string
	switch metric {
	case "docker_build_completed":
		startedcolumn = "docker_build_started"
		durationcolumn = "docker_build_duration"
	case "push_completed":
		startedcolumn = "push_started"
		durationcolumn = "push_duration"
	case "clean_completed":
		startedcolumn = "clean_started"
		durationcolumn = "clean_duration"
	default:
		getstarted = false
	}
	q := `UPDATE build_metrics_by_id SET %v = ? WHERE id = ?;`
	err = dl.wrapQuery(ctx, dl.s.Query(fmt.Sprintf(q, metric), now, id)).Exec()
	if err != nil {
		return err
	}
	if getstarted {
		q = `SELECT %v FROM build_metrics_by_id WHERE id = ?;`
		err = dl.s.Query(fmt.Sprintf(q, startedcolumn), id).Scan(&started)
		if err != nil {
			return err
		}
		duration := now.Sub(started).Seconds()

		q = `UPDATE build_metrics_by_id SET %v = ? WHERE id = ?;`
		return dl.s.Query(fmt.Sprintf(q, durationcolumn), duration, id).Exec()
	}
	return nil
}

// SetDockerImageSizesMetric sets the docker image sizes for a build
func (dl *DBLayer) SetDockerImageSizesMetric(ctx context.Context, id gocql.UUID, size int64, vsize int64) (err error) {
	q := `UPDATE build_metrics_by_id SET docker_image_size = ?, docker_image_vsize = ? WHERE id = ?;`
	return dl.wrapQuery(ctx, dl.s.Query(q, size, vsize, id)).Exec()
}

// SaveBuildOutput serializes an array of stream events to the database
func (dl *DBLayer) SaveBuildOutput(ctx context.Context, id gocql.UUID, output []lib.BuildEvent, column string) (err error) {
	serialized := make([][]byte, len(output))
	var b []byte
	for i, e := range output {
		b, err = proto.Marshal(&e)
		if err != nil {
			return err
		}
		serialized[i] = b
	}
	q := `UPDATE build_events_by_id SET %v = ? WHERE id = ?;`
	return dl.wrapQuery(ctx, dl.s.Query(fmt.Sprintf(q, column), serialized, id.String())).Exec()
}

// GetBuildOutput returns an array of stream events from the database
func (dl *DBLayer) GetBuildOutput(ctx context.Context, id gocql.UUID, column string) (output []lib.BuildEvent, err error) {
	var rawoutput [][]byte
	output = []lib.BuildEvent{}
	q := `SELECT %v FROM build_events_by_id WHERE id = ?;`
	err = dl.wrapQuery(ctx, dl.s.Query(fmt.Sprintf(q, column), id)).Scan(&rawoutput)
	if err != nil {
		return output, err
	}
	for _, rawevent := range rawoutput {
		event := lib.BuildEvent{}
		err = proto.Unmarshal(rawevent, &event)
		if err != nil {
			return output, err
		}
		output = append(output, event)
	}
	return output, nil
}
<commit_msg>fix: use pointers for unmarshaling<commit_after>package datalayer

import (
	"context"
	"fmt"
	"time"

	"github.com/dollarshaveclub/furan/generated/lib"
	"github.com/dollarshaveclub/furan/lib/db"
	"github.com/gocql/gocql"
	"github.com/golang/protobuf/proto"
	gocqltrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/gocql/gocql"
)

// DataLayer describes an object that interacts with the persistent data store
type DataLayer interface {
	CreateBuild(context.Context, *lib.BuildRequest) (gocql.UUID, error)
	GetBuildByID(context.Context, gocql.UUID) (*lib.BuildStatusResponse, error)
	SetBuildFlags(context.Context, gocql.UUID, map[string]bool) error
	SetBuildCompletedTimestamp(context.Context, gocql.UUID) error
	SetBuildState(context.Context, gocql.UUID, lib.BuildStatusResponse_BuildState) error
	DeleteBuild(context.Context, gocql.UUID) error
	SetBuildTimeMetric(context.Context, gocql.UUID, string) error
	SetDockerImageSizesMetric(context.Context, gocql.UUID, int64, int64) error
	SaveBuildOutput(context.Context, gocql.UUID, []lib.BuildEvent, string) error
	GetBuildOutput(context.Context, gocql.UUID, string) ([]lib.BuildEvent, error)
}

// DBLayer is an DataLayer instance that interacts with the Cassandra database
type DBLayer struct {
	s     *gocql.Session
	sname string
}

// NewDBLayer returns a data layer object
func NewDBLayer(s *gocql.Session, sname string) *DBLayer {
	return &DBLayer{s: s, sname: sname}
}

func (dl *DBLayer) wrapQuery(ctx context.Context, query *gocql.Query) *gocqltrace.Query {
	return gocqltrace.WrapQuery(query, gocqltrace.WithServiceName(dl.sname)).WithContext(ctx)
}

// CreateBuild inserts a new build into the DB returning the ID
func (dl *DBLayer) CreateBuild(ctx context.Context, req *lib.BuildRequest) (id gocql.UUID, err error) {
	q := `INSERT INTO builds_by_id (id, request, state, finished, failed, cancelled, started)
        VALUES (?,{github_repo: ?, dockerfile_path: ?, tags: ?, tag_with_commit_sha: ?, ref: ?,
					push_registry_repo: ?, push_s3_region: ?, push_s3_bucket: ?,
					push_s3_key_prefix: ?},?,?,?,?,?);`
	id, err = gocql.RandomUUID()
	if err != nil {
		return id, err
	}
	udt := db.UDTFromBuildRequest(req)
	query := dl.s.Query(q, id, udt.GithubRepo, udt.DockerfilePath, udt.Tags, udt.TagWithCommitSha, udt.Ref,
		udt.PushRegistryRepo, udt.PushS3Region, udt.PushS3Bucket, udt.PushS3KeyPrefix,
		lib.BuildStatusResponse_STARTED.String(), false, false, false, time.Now())
	err = dl.wrapQuery(ctx, query).Exec()
	if err != nil {
		return id, err
	}
	q = `INSERT INTO build_metrics_by_id (id) VALUES (?);`
	err = dl.wrapQuery(ctx, dl.s.Query(q, id)).Exec()
	if err != nil {
		return id, err
	}
	q = `INSERT INTO build_events_by_id (id) VALUES (?);`
	return id, dl.wrapQuery(ctx, dl.s.Query(q, id)).Exec()
}

// GetBuildByID fetches a build object from the DB
func (dl *DBLayer) GetBuildByID(ctx context.Context, id gocql.UUID) (bi *lib.BuildStatusResponse, err error) {
	q := `SELECT request, state, finished, failed, cancelled, started, completed,
	      duration FROM builds_by_id WHERE id = ?;`
	var udt db.BuildRequestUDT
	var state string
	var started, completed time.Time
	bi = &lib.BuildStatusResponse{
		BuildId: id.String(),
	}
	query := dl.s.Query(q, id)
	err = dl.wrapQuery(ctx, query).Scan(&udt.GithubRepo, &udt.DockerfilePath, &udt.Tags, &udt.TagWithCommitSha, &udt.Ref,
		&udt.PushRegistryRepo, &udt.PushS3Region, &udt.PushS3Bucket, &udt.PushS3KeyPrefix, &state, &bi.Finished, &bi.Failed,
		&bi.Cancelled, &started, &completed, &bi.Duration)
	if err != nil {
		return bi, err
	}
	bi.State = db.BuildStateFromString(state)
	bi.BuildRequest = db.BuildRequestFromUDT(&udt)
	bi.Started = started.Format(time.RFC3339)
	bi.Completed = completed.Format(time.RFC3339)
	return bi, nil
}

// SetBuildFlags sets the boolean flags on the build object
// Caller must ensure that the flags passed in are valid
func (dl *DBLayer) SetBuildFlags(ctx context.Context, id gocql.UUID, flags map[string]bool) (err error) {
	q := `UPDATE builds_by_id SET %v = ? WHERE id = ?;`
	for k, v := range flags {
		err = dl.wrapQuery(ctx, dl.s.Query(fmt.Sprintf(q, k), v, id)).Exec()
		if err != nil {
			return err
		}
	}
	return nil
}

// SetBuildCompletedTimestamp sets the completed timestamp on a build to time.Now()
func (dl *DBLayer) SetBuildCompletedTimestamp(ctx context.Context, id gocql.UUID) (err error) {
	var started time.Time
	now := time.Now()
	q := `SELECT started FROM builds_by_id WHERE id = ?;`
	err = dl.wrapQuery(ctx, dl.s.Query(q, id)).Scan(&started)
	if err != nil {
		return err
	}
	duration := now.Sub(started).Seconds()
	q = `UPDATE builds_by_id SET completed = ?, duration = ? WHERE id = ?;`
	return dl.s.Query(q, now, duration, id).Exec()
}

// SetBuildState sets the state of a build
func (dl *DBLayer) SetBuildState(ctx context.Context, id gocql.UUID, state lib.BuildStatusResponse_BuildState) (err error) {
	q := `UPDATE builds_by_id SET state = ? WHERE id = ?;`
	return dl.wrapQuery(ctx, dl.s.Query(q, state.String(), id)).Exec()
}

// DeleteBuild removes a build from the DB.
// Only used in case of queue full when we can't actually do a build
func (dl *DBLayer) DeleteBuild(ctx context.Context, id gocql.UUID) (err error) {
	q := `DELETE FROM builds_by_id WHERE id = ?;`
	err = dl.wrapQuery(ctx, dl.s.Query(q, id)).Exec()
	if err != nil {
		return err
	}
	q = `DELETE FROM build_metrics_by_id WHERE id = ?;`
	return dl.s.Query(q, id).Exec()
}

// SetBuildTimeMetric sets a build metric to time.Now()
// metric is the name of the column to update
// if metric is a *_completed column, it will also compute and persist the duration
func (dl *DBLayer) SetBuildTimeMetric(ctx context.Context, id gocql.UUID, metric string) (err error) {
	var started time.Time
	now := time.Now()
	getstarted := true
	var startedcolumn string
	var durationcolumn string
	switch metric {
	case "docker_build_completed":
		startedcolumn = "docker_build_started"
		durationcolumn = "docker_build_duration"
	case "push_completed":
		startedcolumn = "push_started"
		durationcolumn = "push_duration"
	case "clean_completed":
		startedcolumn = "clean_started"
		durationcolumn = "clean_duration"
	default:
		getstarted = false
	}
	q := `UPDATE build_metrics_by_id SET %v = ? WHERE id = ?;`
	err = dl.wrapQuery(ctx, dl.s.Query(fmt.Sprintf(q, metric), now, id)).Exec()
	if err != nil {
		return err
	}
	if getstarted {
		q = `SELECT %v FROM build_metrics_by_id WHERE id = ?;`
		err = dl.s.Query(fmt.Sprintf(q, startedcolumn), id).Scan(&started)
		if err != nil {
			return err
		}
		duration := now.Sub(started).Seconds()

		q = `UPDATE build_metrics_by_id SET %v = ? WHERE id = ?;`
		return dl.s.Query(fmt.Sprintf(q, durationcolumn), duration, id).Exec()
	}
	return nil
}

// SetDockerImageSizesMetric sets the docker image sizes for a build
func (dl *DBLayer) SetDockerImageSizesMetric(ctx context.Context, id gocql.UUID, size int64, vsize int64) (err error) {
	q := `UPDATE build_metrics_by_id SET docker_image_size = ?, docker_image_vsize = ? WHERE id = ?;`
	return dl.wrapQuery(ctx, dl.s.Query(q, size, vsize, id)).Exec()
}

// SaveBuildOutput serializes an array of stream events to the database
func (dl *DBLayer) SaveBuildOutput(ctx context.Context, id gocql.UUID, output []lib.BuildEvent, column string) (err error) {
	serialized := make([][]byte, len(output))
	var b []byte
	for i, e := range output {
		b, err = proto.Marshal(&e)
		if err != nil {
			return err
		}
		serialized[i] = b
	}
	q := `UPDATE build_events_by_id SET %v = ? WHERE id = ?;`
	return dl.wrapQuery(ctx, dl.s.Query(fmt.Sprintf(q, column), serialized, id.String())).Exec()
}

// GetBuildOutput returns an array of stream events from the database
func (dl *DBLayer) GetBuildOutput(ctx context.Context, id gocql.UUID, column string) (output []lib.BuildEvent, err error) {
	var rawoutput [][]byte
	output = []lib.BuildEvent{}
	q := `SELECT %v FROM build_events_by_id WHERE id = ?;`
	err = dl.wrapQuery(ctx, dl.s.Query(fmt.Sprintf(q, column), id)).Scan(&rawoutput)
	if err != nil {
		return output, err
	}
	for _, rawevent := range rawoutput {
		event := lib.BuildEvent{}
		err = proto.Unmarshal(rawevent, &event)
		if err != nil {
			return output, err
		}
		output = append(output, event)
	}
	return output, nil
}
<|endoftext|>


<commit_before>// Package stackerr provides a way to augment errors with one or more stack
// traces to allow for easier debugging.
package stackerr

import (
	"errors"
	"fmt"

	"github.com/facebookgo/stack"
)

// Error provides the wrapper that adds multiple Stacks to an error. Each Stack
// represents a location in code thru which this error was wrapped.
type Error struct {
	multiStack *stack.Multi
	underlying error
}

// Error provides a multi line error string that includes the stack trace.
func (e *Error) Error() string {
	return fmt.Sprintf("%s\n%s", e.underlying, e.multiStack)
}

// MultiStack identifies the locations this error was wrapped at.
func (e *Error) MultiStack() *stack.Multi {
	return e.multiStack
}

// Underlying returns the error that is being wrapped.
func (e *Error) Underlying() error {
	return e.underlying
}

type hasMultiStack interface {
	MultiStack() *stack.Multi
}

// WrapSkip the error and add the current Stack. The argument skip is the
// number of stack frames to ascend, with 0 identifying the caller of Wrap. If
// the error to be wrapped has a MultiStack, the current stack will be added to
// it.  If the error to be wrapped is nil, a nil error is returned.
func WrapSkip(err error, skip int) error {
	// nil errors are returned back as nil.
	if err == nil {
		return nil
	}

	// we're adding another Stack to an already wrapped error.
	if se, ok := err.(hasMultiStack); ok {
		se.MultiStack().AddCallers(skip + 1)
		return err
	}

	// we're create a freshly wrapped error.
	return &Error{
		multiStack: stack.CallersMulti(skip + 1),
		underlying: err,
	}
}

// Wrap provides a convenience function that calls WrapSkip with skip=0. That
// is, the Stack starts with the caller of Wrap.
func Wrap(err error) error {
	return WrapSkip(err, 1)
}

// New returns a new error that includes the Stack.
func New(s string) error {
	return WrapSkip(errors.New(s), 1)
}

// Newf formats and returns a new error that includes the Stack.
func Newf(format string, args ...interface{}) error {
	return WrapSkip(fmt.Errorf(format, args...), 1)
}

type hasUnderlying interface {
	Underlying() error
}

// Underlying returns all the underlying errors by iteratively checking if the
// error has an Underlying error. If e is nil, the returned slice will be nil.
func Underlying(e error) []error {
	if e == nil {
		return nil
	}

	var errs []error
	for {
		if e == nil {
			return errs
		}
		errs = append(errs, e)

		if eh, ok := e.(hasUnderlying); ok {
			e = eh.Underlying()
		} else {
			e = nil
		}
	}
}

// Matcher defines the interface to check if an error matches an expectation.
type Matcher interface {
	Match(e error) bool
}

type equals struct {
	error error
}

func (e *equals) Match(other error) bool {
	return e.error == other
}

// Equals returns a Matcher to check if an error equals the given error.
func Equals(e error) Matcher {
	return &equals{e}
}

// HasUnderlying returns true if any of the underlying errors satisfy the given
// Matcher.
func HasUnderlying(e error, m Matcher) bool {
	for _, o := range Underlying(e) {
		if m.Match(o) {
			return true
		}
	}
	return false
}
<commit_msg>MatcherFunc<commit_after>// Package stackerr provides a way to augment errors with one or more stack
// traces to allow for easier debugging.
package stackerr

import (
	"errors"
	"fmt"

	"github.com/facebookgo/stack"
)

// Error provides the wrapper that adds multiple Stacks to an error. Each Stack
// represents a location in code thru which this error was wrapped.
type Error struct {
	multiStack *stack.Multi
	underlying error
}

// Error provides a multi line error string that includes the stack trace.
func (e *Error) Error() string {
	return fmt.Sprintf("%s\n%s", e.underlying, e.multiStack)
}

// MultiStack identifies the locations this error was wrapped at.
func (e *Error) MultiStack() *stack.Multi {
	return e.multiStack
}

// Underlying returns the error that is being wrapped.
func (e *Error) Underlying() error {
	return e.underlying
}

type hasMultiStack interface {
	MultiStack() *stack.Multi
}

// WrapSkip the error and add the current Stack. The argument skip is the
// number of stack frames to ascend, with 0 identifying the caller of Wrap. If
// the error to be wrapped has a MultiStack, the current stack will be added to
// it.  If the error to be wrapped is nil, a nil error is returned.
func WrapSkip(err error, skip int) error {
	// nil errors are returned back as nil.
	if err == nil {
		return nil
	}

	// we're adding another Stack to an already wrapped error.
	if se, ok := err.(hasMultiStack); ok {
		se.MultiStack().AddCallers(skip + 1)
		return err
	}

	// we're create a freshly wrapped error.
	return &Error{
		multiStack: stack.CallersMulti(skip + 1),
		underlying: err,
	}
}

// Wrap provides a convenience function that calls WrapSkip with skip=0. That
// is, the Stack starts with the caller of Wrap.
func Wrap(err error) error {
	return WrapSkip(err, 1)
}

// New returns a new error that includes the Stack.
func New(s string) error {
	return WrapSkip(errors.New(s), 1)
}

// Newf formats and returns a new error that includes the Stack.
func Newf(format string, args ...interface{}) error {
	return WrapSkip(fmt.Errorf(format, args...), 1)
}

type hasUnderlying interface {
	Underlying() error
}

// Underlying returns all the underlying errors by iteratively checking if the
// error has an Underlying error. If e is nil, the returned slice will be nil.
func Underlying(e error) []error {
	if e == nil {
		return nil
	}

	var errs []error
	for {
		if e == nil {
			return errs
		}
		errs = append(errs, e)

		if eh, ok := e.(hasUnderlying); ok {
			e = eh.Underlying()
		} else {
			e = nil
		}
	}
}

// Matcher defines the interface to check if an error matches an expectation.
type Matcher interface {
	Match(e error) bool
}

type equals struct {
	error error
}

func (e *equals) Match(other error) bool {
	return e.error == other
}

// Equals returns a Matcher to check if an error equals the given error.
func Equals(e error) Matcher {
	return &equals{e}
}

// HasUnderlying returns true if any of the underlying errors satisfy the given
// Matcher.
func HasUnderlying(e error, m Matcher) bool {
	for _, o := range Underlying(e) {
		if m.Match(o) {
			return true
		}
	}
	return false
}

// MatcherFunc allows using a function as a Matcher. For example to use
// os.IsNotExist as a matcher.
type MatcherFunc func(err error) bool

// Match calls the underlying function.
func (f MatcherFunc) Match(err error) bool {
	return f(err)
}
<|endoftext|>


<commit_before><commit_msg>Add more metrics<commit_after><|endoftext|>


<commit_before><commit_msg>Version bump: 0.11.10<commit_after><|endoftext|>


<commit_before><commit_msg>prepend forward slash for tracker.js asset, so we can omit it on cli argument<commit_after><|endoftext|>


<commit_before><commit_msg>Fix private/usage for Golang 1.18 (#786)<commit_after><|endoftext|>


<commit_before>package smi

import (
	"unsafe"

	"github.com/sleepinggenius2/gosmi/smi/internal"
	"github.com/sleepinggenius2/gosmi/types"
)

// SmiElement *smiGetFirstElement(SmiNode *smiNodePtr)
func GetFirstElement(smiNodePtr *types.SmiNode) *types.SmiElement {
	if smiNodePtr == nil {
		return nil
	}
	objPtr := (*internal.Object)(unsafe.Pointer(smiNodePtr))
	return &objPtr.List.SmiElement
}

// SmiElement *smiGetNextElement(SmiElement *smiElementPtr)
func GetNextElement(smiElementPtr *types.SmiElement) *types.SmiElement {
	if smiElementPtr == nil {
		return nil
	}
	listPtr := (*internal.List)(unsafe.Pointer(smiElementPtr))
	if listPtr.Next == nil {
		return nil
	}
	return &listPtr.Next.SmiElement
}

// SmiNode *smiGetElementNode(SmiElement *smiElementPtr)
func GetElementNode(smiElementPtr *types.SmiElement) *types.SmiNode {
	if smiElementPtr == nil {
		return nil
	}
	listPtr := (*internal.List)(unsafe.Pointer(smiElementPtr))
	if listPtr.Ptr == nil {
		return nil
	}
	return listPtr.Ptr.(*internal.Object).GetSmiNode()
}
<commit_msg>Adds List != nil check in GetFirstElement<commit_after>package smi

import (
	"unsafe"

	"github.com/sleepinggenius2/gosmi/smi/internal"
	"github.com/sleepinggenius2/gosmi/types"
)

// SmiElement *smiGetFirstElement(SmiNode *smiNodePtr)
func GetFirstElement(smiNodePtr *types.SmiNode) *types.SmiElement {
	if smiNodePtr == nil {
		return nil
	}
	objPtr := (*internal.Object)(unsafe.Pointer(smiNodePtr))
	if objPtr.List == nil {
		return nil
	}
	return &objPtr.List.SmiElement
}

// SmiElement *smiGetNextElement(SmiElement *smiElementPtr)
func GetNextElement(smiElementPtr *types.SmiElement) *types.SmiElement {
	if smiElementPtr == nil {
		return nil
	}
	listPtr := (*internal.List)(unsafe.Pointer(smiElementPtr))
	if listPtr.Next == nil {
		return nil
	}
	return &listPtr.Next.SmiElement
}

// SmiNode *smiGetElementNode(SmiElement *smiElementPtr)
func GetElementNode(smiElementPtr *types.SmiElement) *types.SmiNode {
	if smiElementPtr == nil {
		return nil
	}
	listPtr := (*internal.List)(unsafe.Pointer(smiElementPtr))
	if listPtr.Ptr == nil {
		return nil
	}
	return listPtr.Ptr.(*internal.Object).GetSmiNode()
}
<|endoftext|>


<commit_before>package sobjects

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"path"
	"regexp"
	"strings"

	"github.com/grokify/gotilla/io/ioutilmore"
)

type ContactSet struct {
	IdSet      IdSet              `xml:"-"`
	Records    []Contact          `json:"records,omitempty" xml:"records"`
	RecordsMap map[string]Contact `xml:"-"`
}

func NewContactSet() ContactSet {
	set := ContactSet{
		IdSet:      NewIdSet(),
		Records:    []Contact{},
		RecordsMap: map[string]Contact{}}
	return set
}

func NewContactSetSetFromXml(bytes []byte) (ContactSet, error) {
	set := ContactSet{IdSet: NewIdSet()}
	err := xml.Unmarshal(bytes, &set)
	set.Inflate()
	return set, err
}

func NewContactSetFromXmlFile(filepath string) (ContactSet, error) {
	bytes, err := ioutil.ReadFile(filepath)
	if err != nil {
		return ContactSet{}, err
	}
	return NewContactSetSetFromXml(bytes)
}

func NewContactSetFromJSONResponse(resp *http.Response) (ContactSet, error) {
	set := NewContactSet()
	bytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return set, err
	}
	err = json.Unmarshal(bytes, &set)
	return set, err
}

func (set *ContactSet) ReadJsonFilesFromDir(dir string) error {
	files, err := ioutilmore.DirEntriesReNotEmpty(dir, regexp.MustCompile(`(?i)\.json$`))
	if err != nil {
		return err
	}
	for _, fi := range files {
		filepath := path.Join(dir, fi.Name())
		contact, err := NewContactFromJsonFile(filepath)
		if err == nil && len(contact.Id) > 0 {
			set.Records = append(set.Records, contact)
		}
	}
	return nil
}

func (set *ContactSet) Inflate() {
	for _, record := range set.Records {
		if len(record.Id) > 0 {
			set.IdSet.AddId(record.Id)
			set.RecordsMap[record.Id] = record
		}
		if len(record.AccountId) > 0 {
			set.IdSet.AddId(record.AccountId)
		}
	}
}

func (set *ContactSet) GetContactByName(name string) (Contact, error) {
	for _, contact := range set.Records {
		if contact.Name == name {
			return contact, nil
		}
	}
	return Contact{}, errors.New(fmt.Sprintf("Could not found Contact by name [%v]", name))
}

func (set *ContactSet) GetContactById(id string) (Contact, error) {
	for _, contact := range set.Records {
		if contact.Id == id {
			return contact, nil
		}
	}
	return Contact{}, errors.New(fmt.Sprintf("Could not found Contact by id [%v]", id))
}

type Contact struct {
	Id         string
	AccountId  string
	Department string
	Email      string
	Fax        string
	FirstName  string
	LastName   string
	Name       string
}

func NewContactFromJson(bytes []byte) (Contact, error) {
	obj := Contact{}
	err := json.Unmarshal(bytes, &obj)
	return obj, err
}

func NewContactFromJsonFile(filepath string) (Contact, error) {
	bytes, err := ioutil.ReadFile(filepath)
	if err != nil {
		return Contact{}, err
	}
	return NewContactFromJson(bytes)
}

func ContactEmailOrId(contact Contact) string {
	emailOrId := ""
	if len(strings.TrimSpace(contact.Email)) > 0 {
		emailOrId = contact.Email
	} else {
		emailOrId = contact.Id
	}
	return strings.TrimSpace(emailOrId)
}

func ContactsEmailOrId(contacts []Contact) []string {
	emailOrIds := []string{}
	for _, contact := range contacts {
		emailOrId := ContactEmailOrId(contact)
		if len(emailOrId) > 0 {
			emailOrIds = append(emailOrIds, emailOrId)
		}
	}
	return emailOrIds
}

func ContactsEmailOrIdString(contacts []Contact, sep string) string {
	return strings.Join(ContactsEmailOrId(contacts), sep)
}

func ContactIdOrEmail(contact Contact) string {
	idOrEmail := ""
	if len(strings.TrimSpace(contact.Id)) > 0 {
		idOrEmail = contact.Id
	} else {
		idOrEmail = contact.Email
	}
	return strings.TrimSpace(idOrEmail)
}

func ContactsIdOrEmail(contacts []Contact) []string {
	idOrEmails := []string{}
	for _, contact := range contacts {
		idOrEmail := ContactIdOrEmail(contact)
		if len(idOrEmail) > 0 {
			idOrEmails = append(idOrEmails, idOrEmail)
		}
	}
	return idOrEmails
}

func ContactsIdOrEmailString(contacts []Contact, sep string) string {
	return strings.Join(ContactsIdOrEmail(contacts), sep)
}
<commit_msg>dependencies: sobjects: update for gotilla/io/ioutilmore<commit_after>package sobjects

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"strings"

	"github.com/grokify/gotilla/io/ioutilmore"
)

type ContactSet struct {
	IdSet      IdSet              `xml:"-"`
	Records    []Contact          `json:"records,omitempty" xml:"records"`
	RecordsMap map[string]Contact `xml:"-"`
}

func NewContactSet() ContactSet {
	set := ContactSet{
		IdSet:      NewIdSet(),
		Records:    []Contact{},
		RecordsMap: map[string]Contact{}}
	return set
}

func NewContactSetSetFromXml(bytes []byte) (ContactSet, error) {
	set := ContactSet{IdSet: NewIdSet()}
	err := xml.Unmarshal(bytes, &set)
	set.Inflate()
	return set, err
}

func NewContactSetFromXmlFile(filepath string) (ContactSet, error) {
	bytes, err := ioutil.ReadFile(filepath)
	if err != nil {
		return ContactSet{}, err
	}
	return NewContactSetSetFromXml(bytes)
}

func NewContactSetFromJSONResponse(resp *http.Response) (ContactSet, error) {
	set := NewContactSet()
	bytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return set, err
	}
	err = json.Unmarshal(bytes, &set)
	return set, err
}

func (set *ContactSet) ReadJsonFilesFromDir(dir string) error {
	_, filepaths, err := ioutilmore.ReadDirRx(dir, regexp.MustCompile(`(?i)\.json$`), true)
	if err != nil {
		return err
	}
	for _, filepath := range filepaths {
		contact, err := NewContactFromJsonFile(filepath)
		if err == nil && len(contact.Id) > 0 {
			set.Records = append(set.Records, contact)
		}
	}
	return nil
}

func (set *ContactSet) Inflate() {
	for _, record := range set.Records {
		if len(record.Id) > 0 {
			set.IdSet.AddId(record.Id)
			set.RecordsMap[record.Id] = record
		}
		if len(record.AccountId) > 0 {
			set.IdSet.AddId(record.AccountId)
		}
	}
}

func (set *ContactSet) GetContactByName(name string) (Contact, error) {
	for _, contact := range set.Records {
		if contact.Name == name {
			return contact, nil
		}
	}
	return Contact{}, errors.New(fmt.Sprintf("Could not found Contact by name [%v]", name))
}

func (set *ContactSet) GetContactById(id string) (Contact, error) {
	for _, contact := range set.Records {
		if contact.Id == id {
			return contact, nil
		}
	}
	return Contact{}, errors.New(fmt.Sprintf("Could not found Contact by id [%v]", id))
}

type Contact struct {
	Id         string
	AccountId  string
	Department string
	Email      string
	Fax        string
	FirstName  string
	LastName   string
	Name       string
}

func NewContactFromJson(bytes []byte) (Contact, error) {
	obj := Contact{}
	err := json.Unmarshal(bytes, &obj)
	return obj, err
}

func NewContactFromJsonFile(filepath string) (Contact, error) {
	bytes, err := ioutil.ReadFile(filepath)
	if err != nil {
		return Contact{}, err
	}
	return NewContactFromJson(bytes)
}

func ContactEmailOrId(contact Contact) string {
	emailOrId := ""
	if len(strings.TrimSpace(contact.Email)) > 0 {
		emailOrId = contact.Email
	} else {
		emailOrId = contact.Id
	}
	return strings.TrimSpace(emailOrId)
}

func ContactsEmailOrId(contacts []Contact) []string {
	emailOrIds := []string{}
	for _, contact := range contacts {
		emailOrId := ContactEmailOrId(contact)
		if len(emailOrId) > 0 {
			emailOrIds = append(emailOrIds, emailOrId)
		}
	}
	return emailOrIds
}

func ContactsEmailOrIdString(contacts []Contact, sep string) string {
	return strings.Join(ContactsEmailOrId(contacts), sep)
}

func ContactIdOrEmail(contact Contact) string {
	idOrEmail := ""
	if len(strings.TrimSpace(contact.Id)) > 0 {
		idOrEmail = contact.Id
	} else {
		idOrEmail = contact.Email
	}
	return strings.TrimSpace(idOrEmail)
}

func ContactsIdOrEmail(contacts []Contact) []string {
	idOrEmails := []string{}
	for _, contact := range contacts {
		idOrEmail := ContactIdOrEmail(contact)
		if len(idOrEmail) > 0 {
			idOrEmails = append(idOrEmails, idOrEmail)
		}
	}
	return idOrEmails
}

func ContactsIdOrEmailString(contacts []Contact, sep string) string {
	return strings.Join(ContactsIdOrEmail(contacts), sep)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin
// +build 386 amd64

package gldriver

// #include "cocoa.h"
import "C"

import (
	"fmt"
	"image"
	"sync"

	"golang.org/x/exp/shiny/screen"
	"golang.org/x/mobile/event/paint"
)

type screenImpl struct {
	mu      sync.Mutex
	windows map[uintptr]*windowImpl
}

func (s *screenImpl) NewBuffer(size image.Point) (retBuf screen.Buffer, retErr error) {
	return &bufferImpl{
		rgba: image.NewRGBA(image.Rectangle{Max: size}),
		size: size,
	}, nil
}

func (s *screenImpl) NewTexture(size image.Point) (screen.Texture, error) {
	return nil, fmt.Errorf("NewTexture not implemented")
}

func (s *screenImpl) NewWindow(opts *screen.NewWindowOptions) (screen.Window, error) {
	// TODO: look at opts.
	const width, height = 512, 384

	id := C.newWindow(width, height)
	w := &windowImpl{
		s:         s,
		id:        uintptr(id),
		eventsIn:  make(chan interface{}),
		eventsOut: make(chan interface{}),
		endPaint:  make(chan paint.Event, 1),
		draw:      make(chan struct{}),
		drawDone:  make(chan struct{}),
	}

	s.mu.Lock()
	s.windows[uintptr(id)] = w
	s.mu.Unlock()

	go w.pump()
	go w.drawLoop(uintptr(C.showWindow(id)))

	return w, nil
}
<commit_msg>shiny/driver/gldriver: make the default window size consistent with x11.<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin
// +build 386 amd64

package gldriver

// #include "cocoa.h"
import "C"

import (
	"fmt"
	"image"
	"sync"

	"golang.org/x/exp/shiny/screen"
	"golang.org/x/mobile/event/paint"
)

type screenImpl struct {
	mu      sync.Mutex
	windows map[uintptr]*windowImpl
}

func (s *screenImpl) NewBuffer(size image.Point) (retBuf screen.Buffer, retErr error) {
	return &bufferImpl{
		rgba: image.NewRGBA(image.Rectangle{Max: size}),
		size: size,
	}, nil
}

func (s *screenImpl) NewTexture(size image.Point) (screen.Texture, error) {
	return nil, fmt.Errorf("NewTexture not implemented")
}

func (s *screenImpl) NewWindow(opts *screen.NewWindowOptions) (screen.Window, error) {
	// TODO: look at opts.
	const width, height = 1024, 768

	id := C.newWindow(width, height)
	w := &windowImpl{
		s:         s,
		id:        uintptr(id),
		eventsIn:  make(chan interface{}),
		eventsOut: make(chan interface{}),
		endPaint:  make(chan paint.Event, 1),
		draw:      make(chan struct{}),
		drawDone:  make(chan struct{}),
	}

	s.mu.Lock()
	s.windows[uintptr(id)] = w
	s.mu.Unlock()

	go w.pump()
	go w.drawLoop(uintptr(C.showWindow(id)))

	return w, nil
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
)

func subscribe(c *cli.Context) {
	if c.Bool("d") {
		log.SetLevel(log.DebugLevel)
	}
	opts, err := NewOption(c)
	if err != nil {
		log.Error(err)
		os.Exit(1)
	}
	if c.Bool("c") {
		opts.SetCleanSession(false)
	}

	client, err := connect(c, opts)
	if err != nil {
		log.Error(err)
		os.Exit(1)
	}

	qos := c.Int("q")
	topic := c.String("t")
	if topic == "" {
		log.Errorf("Please specify topic")
		os.Exit(1)
	}
	log.Infof("Topic: %s", topic)

	err = client.Subscribe(topic, qos)
	if err != nil {
		log.Error(err)
	}
}
<commit_msg>show warning if -c (cleanSession) without -i (clientid).<commit_after>package main

import (
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
)

func subscribe(c *cli.Context) {
	if c.Bool("d") {
		log.SetLevel(log.DebugLevel)
	}
	opts, err := NewOption(c)
	if err != nil {
		log.Error(err)
		os.Exit(1)
	}
	if c.Bool("c") {
		clientId := c.String("i")
		if clientId == "" {
			log.Warn("clean Flag does not work without client id")
		}

		opts.SetCleanSession(false)
	}

	client, err := connect(c, opts)
	if err != nil {
		log.Error(err)
		os.Exit(1)
	}

	qos := c.Int("q")
	topic := c.String("t")
	if topic == "" {
		log.Errorf("Please specify topic")
		os.Exit(1)
	}
	log.Infof("Topic: %s", topic)

	err = client.Subscribe(topic, qos)
	if err != nil {
		log.Error(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/fs"
)

////////////////////////////////////////////////////////////////////////
// Public
////////////////////////////////////////////////////////////////////////

// An object that knows how to restore previously backed up directories.
type DirectoryRestorer interface {
	// Recursively restore a directory based on the listing named by the supplied
	// score. The first call should set basePath to the target directory and
	// relPath to the empty string.
	RestoreDirectory(score blob.Score, basePath, relPath string) (err error)
}

// Create a directory restorer that uses the supplied objects.
func NewDirectoryRestorer(
	blobStore blob.Store,
	fileSystem fs.FileSystem,
	fileRestorer FileRestorer,
) (restorer DirectoryRestorer, err error) {
	createRestorer := func(wrapped DirectoryRestorer) DirectoryRestorer {
		restorer, err := NewNonRecursiveDirectoryRestorer(
			blobStore,
			fileSystem,
			fileRestorer,
			wrapped,
		)

		if err != nil {
			panic(err)
		}

		return restorer
	}

	return &onDemandDirRestorer{createRestorer}, nil
}

////////////////////////////////////////////////////////////////////////
// Implementation details
////////////////////////////////////////////////////////////////////////

// A directory restorer that creates a new directory restorer for each call.
// This breaks a self-dependency that would be needed to make use of
// NewNonRecursiveDirectoryRestorer.
type onDemandDirRestorer struct {
	createRestorer func(wrapped DirectoryRestorer) DirectoryRestorer
}

func (r *onDemandDirRestorer) RestoreDirectory(
	score blob.Score,
	basePath string,
	relPath string,
) (err error) {
	return r.createRestorer(r).RestoreDirectory(score, basePath, relPath)
}

// Split out for testability. You should not use this directly.
func NewNonRecursiveDirectoryRestorer(
	blobStore blob.Store,
	fileSystem fs.FileSystem,
	fileRestorer FileRestorer,
	wrapped DirectoryRestorer,
) (restorer DirectoryRestorer, err error)
<commit_msg>Added a stub for NewNonRecursiveDirectoryRestorer.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"fmt"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/fs"
)

////////////////////////////////////////////////////////////////////////
// Public
////////////////////////////////////////////////////////////////////////

// An object that knows how to restore previously backed up directories.
type DirectoryRestorer interface {
	// Recursively restore a directory based on the listing named by the supplied
	// score. The first call should set basePath to the target directory and
	// relPath to the empty string.
	RestoreDirectory(score blob.Score, basePath, relPath string) (err error)
}

// Create a directory restorer that uses the supplied objects.
func NewDirectoryRestorer(
	blobStore blob.Store,
	fileSystem fs.FileSystem,
	fileRestorer FileRestorer,
) (restorer DirectoryRestorer, err error) {
	createRestorer := func(wrapped DirectoryRestorer) DirectoryRestorer {
		restorer, err := NewNonRecursiveDirectoryRestorer(
			blobStore,
			fileSystem,
			fileRestorer,
			wrapped,
		)

		if err != nil {
			panic(err)
		}

		return restorer
	}

	return &onDemandDirRestorer{createRestorer}, nil
}

////////////////////////////////////////////////////////////////////////
// Implementation details
////////////////////////////////////////////////////////////////////////

// A directory restorer that creates a new directory restorer for each call.
// This breaks a self-dependency that would be needed to make use of
// NewNonRecursiveDirectoryRestorer.
type onDemandDirRestorer struct {
	createRestorer func(wrapped DirectoryRestorer) DirectoryRestorer
}

func (r *onDemandDirRestorer) RestoreDirectory(
	score blob.Score,
	basePath string,
	relPath string,
) (err error) {
	return r.createRestorer(r).RestoreDirectory(score, basePath, relPath)
}

// Split out for testability. You should not use this directly.
func NewNonRecursiveDirectoryRestorer(
	blobStore blob.Store,
	fileSystem fs.FileSystem,
	fileRestorer FileRestorer,
	wrapped DirectoryRestorer,
) (restorer DirectoryRestorer, err error) {
	err = fmt.Errorf("TODO")
	return
}
<|endoftext|>


<commit_before>package test

import (
	"net/http"
	"net/http/httptest"
	"path/filepath"
	"runtime"
	"testing"

	"github.com/JREAMLU/core/inout"
	_ "github.com/JREAMLU/jkernel/base/routers"
	"github.com/JREAMLU/jkernel/base/services"
	"github.com/JREAMLU/jkernel/base/services/atom"
	"github.com/astaxie/beego"
	. "github.com/smartystreets/goconvey/convey"
)

func init() {
	_, file, _, _ := runtime.Caller(1)
	apppath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, ".."+string(filepath.Separator))))
	beego.TestBeegoInit(apppath)
}

func TestUrlGoshorten(t *testing.T) {
	r := TRollingCurl(Requests{
		Method: "POST",
		UrlStr: "/v1/url/goshorten.json?a=1&b=2",
		Header: map[string]string{
			"Content-Type": "application/json;charset=UTF-8;",
			"Accept":       "application/json",
			"Source":       "gotest",
			"ip":           "9.9.9.9",
		},
		Raw: `{"data":{"urls":[{"long_url":"http://o9d.cn","IP":"127.0.0.1"},{"long_url":"http://huiyimei.com","IP":"192.168.1.1"}],"timestamp":1466668134,"sign":"0B490F84305C7CF4D9CDD293B936BE0D"}}`,
	})
	w := httptest.NewRecorder()

	beego.BeeApp.Handlers.ServeHTTP(w, r)
	beego.Trace("testing", "TestUrlGoshorten", "Code[%d]\n%s", w.Code, w.Body.String())

	Convey("func /v1/url/goshorten.json", t, func() {
		Convey("Status Code Should Be 200", func() {
			So(w.Code, ShouldEqual, 200)
		})
		Convey("The Result Should Not Be Empty", func() {
			So(w.Body.Len(), ShouldBeGreaterThan, 0)
		})
	})
}

func TestUrlServiceGoshorten(t *testing.T) {
	httpStatus, shorten := urlServiceGoshorten()

	Convey("func Goshorten()", t, func() {
		Convey("Status Code Should Be 200", func() {
			So(httpStatus, ShouldEqual, 200)
		})
		Convey("result", func() {
			datalist := shorten.Data.(atom.DataList)
			So(datalist.Total, ShouldEqual, 2)
			So(len(datalist.List["http://huiyimei.com"].(string)), ShouldBeGreaterThan, 0)
			So(len(datalist.List["http://o9d.cn"].(string)), ShouldBeGreaterThan, 0)
		})
	})
}

func Benchmark_UrlServiceGoshorten(b *testing.B) {
	for i := 0; i < b.N; i++ {
		urlServiceGoshorten()
	}
}

func urlServiceGoshorten() (int, inout.Output) {
	data := make(map[string]interface{})
	h := `{"Accept":["application/json"],"Content-Type":["application/json;charset=UTF-8;"],"Ip":["9.9.9.9"],"Request-Id":["base-57c930de30e8bd1aac000001"],"Source":["gotest"]}`
	hm := make(http.Header)
	hm["Accept"] = []string{"application/json"}
	hm["Content-Type"] = []string{"application/json;charset=UTF-8;"}
	hm["Ip"] = []string{"9.9.9.9"}
	hm["Request-Id"] = []string{"base-57c930de30e8bd1aac000001"}
	hm["Source"] = []string{"gotest"}
	b := `{"data":{"urls":[{"long_url":"http://o9d.cn","IP":"127.0.0.1"},{"long_url":"http://huiyimei.com","IP":"192.168.1.1"}],"timestamp":1466668134,"sign":"0B490F84305C7CF4D9CDD293B936BE0D"}}`
	c := `[]`
	q := `"a":["1"],"b":["2"]}`
	qm := make(map[string][]string)
	qm["a"] = []string{"1"}
	qm["b"] = []string{"2"}

	data["header"] = []byte(h)
	data["body"] = []byte(b)
	data["cookies"] = []byte(c)
	data["querystr"] = []byte(q)
	data["headermap"] = hm
	data["cookiesslice"] = []string{""}
	data["querystrmap"] = qm

	var service services.Url

	return service.GoShorten(data)
}
<commit_msg>profect unit test<commit_after>package test

import (
	"net/http"
	"net/http/httptest"
	"path/filepath"
	"runtime"
	"testing"

	"github.com/JREAMLU/core/inout"
	_ "github.com/JREAMLU/jkernel/base/routers"
	"github.com/JREAMLU/jkernel/base/services"
	"github.com/JREAMLU/jkernel/base/services/atom"
	"github.com/astaxie/beego"
	. "github.com/smartystreets/goconvey/convey"
)

func init() {
	_, file, _, _ := runtime.Caller(1)
	apppath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, ".."+string(filepath.Separator))))
	beego.TestBeegoInit(apppath)
}

func TestUrlGoshorten(t *testing.T) {
	r := TRollingCurl(Requests{
		Method: "POST",
		UrlStr: "/v1/url/goshorten.json?a=1&b=2",
		Header: map[string]string{
			"Content-Type": "application/json;charset=UTF-8;",
			"Accept":       "application/json",
			"Source":       "gotest",
			"ip":           "9.9.9.9",
		},
		Raw: `{"data":{"urls":[{"long_url":"http://o9d.cn","IP":"127.0.0.1"},{"long_url":"http://huiyimei.com","IP":"192.168.1.1"}],"timestamp":1466668134,"sign":"0B490F84305C7CF4D9CDD293B936BE0D"}}`,
	})
	w := httptest.NewRecorder()

	beego.BeeApp.Handlers.ServeHTTP(w, r)
	beego.Trace("testing", "TestUrlGoshorten", "Code[%d]\n%s", w.Code, w.Body.String())

	Convey("func /v1/url/goshorten.json", t, func() {
		Convey("Status Code Should Be 200", func() {
			So(w.Code, ShouldEqual, 200)
		})
		Convey("The Result Should Not Be Empty", func() {
			So(w.Body.Len(), ShouldBeGreaterThan, 0)
		})
	})
}

func TestUrlServiceGoshorten(t *testing.T) {
	httpStatus, shorten := urlServiceGoshorten()

	Convey("func Goshorten()", t, func() {
		Convey("Status Code Should Be 200", func() {
			So(httpStatus, ShouldEqual, 200)
		})
		Convey("result", func() {
			datalist := shorten.Data.(atom.DataList)
			So(datalist.Total, ShouldEqual, 2)
			So(len(datalist.List["http://huiyimei.com"].(string)), ShouldBeGreaterThan, 0)
			So(len(datalist.List["http://o9d.cn"].(string)), ShouldBeGreaterThan, 0)
		})
	})
}

func BenchmarkUrlServiceGoshorten(b *testing.B) {
	Convey("bench UrlServiceGoshorten \n", b, func() {
		for i := 0; i < b.N; i++ {
			urlServiceGoshorten()
		}
	})
}

func urlServiceGoshorten() (int, inout.Output) {
	data := make(map[string]interface{})
	h := `{"Accept":["application/json"],"Content-Type":["application/json;charset=UTF-8;"],"Ip":["9.9.9.9"],"Request-Id":["base-57c930de30e8bd1aac000001"],"Source":["gotest"]}`
	hm := make(http.Header)
	hm["Accept"] = []string{"application/json"}
	hm["Content-Type"] = []string{"application/json;charset=UTF-8;"}
	hm["Ip"] = []string{"9.9.9.9"}
	hm["Request-Id"] = []string{"base-57c930de30e8bd1aac000001"}
	hm["Source"] = []string{"gotest"}
	b := `{"data":{"urls":[{"long_url":"http://o9d.cn","IP":"127.0.0.1"},{"long_url":"http://huiyimei.com","IP":"192.168.1.1"}],"timestamp":1466668134,"sign":"0B490F84305C7CF4D9CDD293B936BE0D"}}`
	c := `[]`
	q := `"a":["1"],"b":["2"]}`
	qm := make(map[string][]string)
	qm["a"] = []string{"1"}
	qm["b"] = []string{"2"}

	data["header"] = []byte(h)
	data["body"] = []byte(b)
	data["cookies"] = []byte(c)
	data["querystr"] = []byte(q)
	data["headermap"] = hm
	data["cookiesslice"] = []string{""}
	data["querystrmap"] = qm

	var service services.Url

	return service.GoShorten(data)
}
<|endoftext|>


<commit_before>package sorting

type Position struct {
	Position int
}

func (Position) PositionMoveUp(pos int) {
}

func (Position) PositionMoveDown(pos int) {
}

func (Position) PositionMoveTo(pos int) {
}
<commit_msg>Add dummy PositionMoveUp, PositionMoveDown, PositionMoveTo method<commit_after>package sorting

type Position struct {
	Position int
}

func (Position) PositionMoveUp(pos int) {
	// update other's position: (current position + 1 .. current position + pos) - 1
	// update self's position: current position + pos
}

func (Position) PositionMoveDown(pos int) {
	// update other's position: (current position + 1 .. current position + pos) + 1
	// update self's position: current position - pos
}

func (Position) PositionMoveTo(pos int) {
	// if pos < current position
	//   update other's position: (pos .. current position) + 1
	//   update self's position: pos
	// else if pos > current position
	//   update other's position: (current position .. pos) - 1
	//   update self's position: pos
}
<|endoftext|>


<commit_before>package libvirt

import (
	"encoding/json"
	"log"
	"strings"
	"time"

	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
	libvirt "github.com/libvirt/libvirt-go"
)

const qemuGetIfaceWait = "qemu-agent-wait"
const qemuGetIfaceDone = "qemu-agent-done"

// QemuAgentInterfacesResponse type
type QemuAgentInterfacesResponse struct {
	Interfaces []QemuAgentInterface `json:"return"`
}

// QemuAgentInterface type
type QemuAgentInterface struct {
	Name        string                        `json:"name"`
	Hwaddr      string                        `json:"hardware-address"`
	IPAddresses []QemuAgentInterfaceIPAddress `json:"ip-addresses"`
}

// QemuAgentInterfaceIPAddress type
type QemuAgentInterfaceIPAddress struct {
	Type    string `json:"ip-address-type"`
	Address string `json:"ip-address"`
	Prefix  uint   `json:"prefix"`
}

func qemuAgentInterfacesRefreshFunc(domain Domain, wait4ipv4 bool) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {

		var interfaces []libvirt.DomainInterface

		log.Printf("[DEBUG] sending command to qemu-agent")
		result, err := domain.QemuAgentCommand(
			"{\"execute\":\"guest-network-get-interfaces\"}",
			libvirt.DOMAIN_QEMU_AGENT_COMMAND_DEFAULT,
			0)
		if err != nil {
			log.Printf("[DEBUG] command error: %s", err)
			return interfaces, qemuGetIfaceWait, nil
		}

		log.Printf("[DEBUG] qemu-agent response: %s", result)

		response := QemuAgentInterfacesResponse{}
		if err := json.Unmarshal([]byte(result), &response); err != nil {
			log.Printf("[DEBUG] Error converting qemu-agent response about domain interfaces: %s", err)
			log.Printf("[DEBUG] Original message: %+v", response)
			log.Print("[DEBUG] Returning an empty list of interfaces")
			return interfaces, "", nil
		}
		log.Printf("[DEBUG] Parsed response %+v", response)

		for _, iface := range response.Interfaces {
			if iface.Name == "lo" {
				// ignore loopback interface
				continue
			}

			libVirtIface := libvirt.DomainInterface{
				Name:   iface.Name,
				Hwaddr: iface.Hwaddr}

			ipv4Assigned := false
			for _, addr := range iface.IPAddresses {
				if addr.Address == "" {
					// ignore interfaces without an address (eg. waiting for dhcp lease)
					continue
				}

				libVirtAddr := libvirt.DomainIPAddress{
					Addr:   addr.Address,
					Prefix: addr.Prefix,
				}

				switch strings.ToLower(addr.Type) {
				case "ipv4":
					libVirtAddr.Type = int(libvirt.IP_ADDR_TYPE_IPV4)
					ipv4Assigned = true
				case "ipv6":
					libVirtAddr.Type = int(libvirt.IP_ADDR_TYPE_IPV6)
				default:
					log.Printf("[ERROR] Cannot handle unknown address type %s", addr.Type)
					continue
				}
				libVirtIface.Addrs = append(libVirtIface.Addrs, libVirtAddr)
			}
			if len(libVirtIface.Addrs) > 0 && (ipv4Assigned || !wait4ipv4) {
				interfaces = append(interfaces, libVirtIface)
			}
		}

		log.Printf("[DEBUG] Interfaces obtained via qemu-agent: %+v", interfaces)
		return interfaces, qemuGetIfaceDone, nil
	}
}

// Retrieve all the interfaces attached to a domain and their addresses. Only
// the interfaces with at least an IP address are returned.
// When wait4ipv4 is turned on the code will not report interfaces that don't
// have a ipv4 address set. This is useful when a domain gets the ipv6 address
// before the ipv4 one.
func qemuAgentGetInterfacesInfo(domain Domain, wait4ipv4 bool) []libvirt.DomainInterface {

	qemuAgentQuery := &resource.StateChangeConf{
		Pending:    []string{qemuGetIfaceWait},
		Target:     []string{qemuGetIfaceDone},
		Refresh:    qemuAgentInterfacesRefreshFunc(domain, wait4ipv4),
		MinTimeout:  1 * time.Minute,
		Delay:      30 * time.Second, // Wait this time before starting checks
		Timeout:    30 * time.Minute,
	}

	interfaces, err := qemuAgentQuery.WaitForState()
	if err != nil {
		return []libvirt.DomainInterface{}
	}

	return interfaces.([]libvirt.DomainInterface)
}
<commit_msg>reduce timeout from 30 to 5 minutes<commit_after>package libvirt

import (
	"encoding/json"
	"log"
	"strings"
	"time"

	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
	libvirt "github.com/libvirt/libvirt-go"
)

const qemuGetIfaceWait = "qemu-agent-wait"
const qemuGetIfaceDone = "qemu-agent-done"

// QemuAgentInterfacesResponse type
type QemuAgentInterfacesResponse struct {
	Interfaces []QemuAgentInterface `json:"return"`
}

// QemuAgentInterface type
type QemuAgentInterface struct {
	Name        string                        `json:"name"`
	Hwaddr      string                        `json:"hardware-address"`
	IPAddresses []QemuAgentInterfaceIPAddress `json:"ip-addresses"`
}

// QemuAgentInterfaceIPAddress type
type QemuAgentInterfaceIPAddress struct {
	Type    string `json:"ip-address-type"`
	Address string `json:"ip-address"`
	Prefix  uint   `json:"prefix"`
}

func qemuAgentInterfacesRefreshFunc(domain Domain, wait4ipv4 bool) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {

		var interfaces []libvirt.DomainInterface

		log.Printf("[DEBUG] sending command to qemu-agent")
		result, err := domain.QemuAgentCommand(
			"{\"execute\":\"guest-network-get-interfaces\"}",
			libvirt.DOMAIN_QEMU_AGENT_COMMAND_DEFAULT,
			0)
		if err != nil {
			log.Printf("[DEBUG] command error: %s", err)
			return interfaces, qemuGetIfaceWait, nil
		}

		log.Printf("[DEBUG] qemu-agent response: %s", result)

		response := QemuAgentInterfacesResponse{}
		if err := json.Unmarshal([]byte(result), &response); err != nil {
			log.Printf("[DEBUG] Error converting qemu-agent response about domain interfaces: %s", err)
			log.Printf("[DEBUG] Original message: %+v", response)
			log.Print("[DEBUG] Returning an empty list of interfaces")
			return interfaces, "", nil
		}
		log.Printf("[DEBUG] Parsed response %+v", response)

		for _, iface := range response.Interfaces {
			if iface.Name == "lo" {
				// ignore loopback interface
				continue
			}

			libVirtIface := libvirt.DomainInterface{
				Name:   iface.Name,
				Hwaddr: iface.Hwaddr}

			ipv4Assigned := false
			for _, addr := range iface.IPAddresses {
				if addr.Address == "" {
					// ignore interfaces without an address (eg. waiting for dhcp lease)
					continue
				}

				libVirtAddr := libvirt.DomainIPAddress{
					Addr:   addr.Address,
					Prefix: addr.Prefix,
				}

				switch strings.ToLower(addr.Type) {
				case "ipv4":
					libVirtAddr.Type = int(libvirt.IP_ADDR_TYPE_IPV4)
					ipv4Assigned = true
				case "ipv6":
					libVirtAddr.Type = int(libvirt.IP_ADDR_TYPE_IPV6)
				default:
					log.Printf("[ERROR] Cannot handle unknown address type %s", addr.Type)
					continue
				}
				libVirtIface.Addrs = append(libVirtIface.Addrs, libVirtAddr)
			}
			if len(libVirtIface.Addrs) > 0 && (ipv4Assigned || !wait4ipv4) {
				interfaces = append(interfaces, libVirtIface)
			}
		}

		log.Printf("[DEBUG] Interfaces obtained via qemu-agent: %+v", interfaces)
		return interfaces, qemuGetIfaceDone, nil
	}
}

// Retrieve all the interfaces attached to a domain and their addresses. Only
// the interfaces with at least an IP address are returned.
// When wait4ipv4 is turned on the code will not report interfaces that don't
// have a ipv4 address set. This is useful when a domain gets the ipv6 address
// before the ipv4 one.
func qemuAgentGetInterfacesInfo(domain Domain, wait4ipv4 bool) []libvirt.DomainInterface {

	qemuAgentQuery := &resource.StateChangeConf{
		Pending:    []string{qemuGetIfaceWait},
		Target:     []string{qemuGetIfaceDone},
		Refresh:    qemuAgentInterfacesRefreshFunc(domain, wait4ipv4),
		MinTimeout:  1 * time.Minute,
		Delay:      30 * time.Second, // Wait this time before starting checks
		Timeout:    5 * time.Minute,
	}

	interfaces, err := qemuAgentQuery.WaitForState()
	if err != nil {
		return []libvirt.DomainInterface{}
	}

	return interfaces.([]libvirt.DomainInterface)
}
<|endoftext|>


<commit_before>package libvirt

import (
	"encoding/xml"
)

type defBackingStore struct {
	Path   string `xml:"path"`
	Format struct {
		Type string `xml:"type,attr"`
	} `xml:"format"`
}

type defVolume struct {
	XMLName xml.Name `xml:"volume"`
	Name    string   `xml:"name"`
	Target  struct {
		Format struct {
			Type string `xml:"type,attr"`
		} `xml:"format"`
	} `xml:"target"`
	Allocation int `xml:"allocation"`
	Capacity   struct {
		Unit   string `xml:"unit,attr"`
		Amount int    `xml:"chardata"`
	} `xml:"capacity"`
	BackingStore *defBackingStore `xml:"backingStore,omitempty"`
}

func newDefVolume() defVolume {
	volumeDef := defVolume{}
	volumeDef.Target.Format.Type = "qcow2"
	volumeDef.Capacity.Unit = "bytes"
	volumeDef.Capacity.Amount = 1
	return volumeDef
}
<commit_msg>if libvirt will not set permissions correctly, at least allow others read base images<commit_after>package libvirt

import (
	"encoding/xml"
)

type defBackingStore struct {
	Path   string `xml:"path"`
	Format struct {
		Type string `xml:"type,attr"`
	} `xml:"format"`
}

type defVolume struct {
	XMLName xml.Name `xml:"volume"`
	Name    string   `xml:"name"`
	Target  struct {
		Format struct {
			Type string `xml:"type,attr"`
		} `xml:"format"`
		Permissions struct {
			Mode int `xml:"mode,omitempty"`
		} `xml:"permissions,omitempty"`

	} `xml:"target"`
	Allocation int `xml:"allocation"`
	Capacity   struct {
		Unit   string `xml:"unit,attr"`
		Amount int    `xml:"chardata"`
	} `xml:"capacity"`
	BackingStore *defBackingStore `xml:"backingStore,omitempty"`
}

func newDefVolume() defVolume {
	volumeDef := defVolume{}
	volumeDef.Target.Format.Type = "qcow2"
	volumeDef.Target.Permissions.Mode = 644
	volumeDef.Capacity.Unit = "bytes"
	volumeDef.Capacity.Amount = 1
	return volumeDef
}
<|endoftext|>


<commit_before>package goutil

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestSumInt(t *testing.T) {
	var tests = map[string]struct {
		in  []int
		out int
	}{
		"positive": {[]int{1, 2, 3}, 6},
		"mixed":    {[]int{-1, -2, 3}, 0},
		"negative": {[]int{-1, -2, -3}, -6},
	}

	for name, tc := range tests {
		t.Run(name, func(t *testing.T) {
			assert.Equal(t, SumInt(tc.in), tc.out, "Should be equal")
		})
	}
}

func TestSumInt8(t *testing.T) {
	var tests = map[string]struct {
		in  []int8
		out int8
	}{
		"positive": {[]int8{1, 2, 3}, 6},
		"mixed":    {[]int8{-1, -2, 3}, 0},
		"negative": {[]int8{-1, -2, -3}, -6},
	}

	for name, tc := range tests {
		t.Run(name, func(t *testing.T) {
			assert.Equal(t, SumInt8(tc.in), tc.out, "Should be equal")
		})
	}
}

func TestSumInt16(t *testing.T) {
	var tests = map[string]struct {
		in  []int16
		out int16
	}{
		"positive": {[]int16{1, 2, 3}, 6},
		"mixed":    {[]int16{-1, -2, 3}, 0},
		"negative": {[]int16{-1, -2, -3}, -6},
	}

	for name, tc := range tests {
		t.Run(name, func(t *testing.T) {
			assert.Equal(t, SumInt16(tc.in), tc.out, "Should be equal")
		})
	}
}

func TestSumInt32(t *testing.T) {
	var tests = map[string]struct {
		in  []int32
		out int32
	}{
		"positive": {[]int32{1, 2, 3}, 6},
		"mixed":    {[]int32{-1, -2, 3}, 0},
		"negative": {[]int32{-1, -2, -3}, -6},
	}

	for name, tc := range tests {
		t.Run(name, func(t *testing.T) {
			assert.Equal(t, SumInt32(tc.in), tc.out, "Should be equal")
		})
	}
}

func TestSumInt64(t *testing.T) {
	var tests = map[string]struct {
		in  []int64
		out int64
	}{
		"positive": {[]int64{1, 2, 3}, 6},
		"mixed":    {[]int64{-1, -2, 3}, 0},
		"negative": {[]int64{-1, -2, -3}, -6},
	}

	for name, tc := range tests {
		t.Run(name, func(t *testing.T) {
			assert.Equal(t, SumInt64(tc.in), tc.out, "Should be equal")
		})
	}
}
<commit_msg>Remove unnecessary additional arguments to assert.Equal<commit_after>package goutil

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestSumInt(t *testing.T) {
	var tests = map[string]struct {
		in  []int
		out int
	}{
		"positive": {[]int{1, 2, 3}, 6},
		"mixed":    {[]int{-1, -2, 3}, 0},
		"negative": {[]int{-1, -2, -3}, -6},
	}

	for name, tc := range tests {
		t.Run(name, func(t *testing.T) {
			assert.Equal(t, SumInt(tc.in), tc.out)
		})
	}
}

func TestSumInt8(t *testing.T) {
	var tests = map[string]struct {
		in  []int8
		out int8
	}{
		"positive": {[]int8{1, 2, 3}, 6},
		"mixed":    {[]int8{-1, -2, 3}, 0},
		"negative": {[]int8{-1, -2, -3}, -6},
	}

	for name, tc := range tests {
		t.Run(name, func(t *testing.T) {
			assert.Equal(t, SumInt8(tc.in), tc.out)
		})
	}
}

func TestSumInt16(t *testing.T) {
	var tests = map[string]struct {
		in  []int16
		out int16
	}{
		"positive": {[]int16{1, 2, 3}, 6},
		"mixed":    {[]int16{-1, -2, 3}, 0},
		"negative": {[]int16{-1, -2, -3}, -6},
	}

	for name, tc := range tests {
		t.Run(name, func(t *testing.T) {
			assert.Equal(t, SumInt16(tc.in), tc.out)
		})
	}
}

func TestSumInt32(t *testing.T) {
	var tests = map[string]struct {
		in  []int32
		out int32
	}{
		"positive": {[]int32{1, 2, 3}, 6},
		"mixed":    {[]int32{-1, -2, 3}, 0},
		"negative": {[]int32{-1, -2, -3}, -6},
	}

	for name, tc := range tests {
		t.Run(name, func(t *testing.T) {
			assert.Equal(t, SumInt32(tc.in), tc.out)
		})
	}
}

func TestSumInt64(t *testing.T) {
	var tests = map[string]struct {
		in  []int64
		out int64
	}{
		"positive": {[]int64{1, 2, 3}, 6},
		"mixed":    {[]int64{-1, -2, 3}, 0},
		"negative": {[]int64{-1, -2, -3}, -6},
	}

	for name, tc := range tests {
		t.Run(name, func(t *testing.T) {
			assert.Equal(t, SumInt64(tc.in), tc.out)
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	. "github.com/frezadev/hdc/hive"
	// . "github.com/eaciit/hdc/hive"
	"github.com/pkg/profile"
	"log"
)

type Students struct {
	Name    string
	Age     int
	Phone   string
	Address string
}

func fatalCheck(what string, e error) {
	if e != nil {
		log.Fatalf("%s: %s", what, e.Error())
	}
}

func main() {
	defer profile.Start(profile.CPUProfile).Stop()

	h := HiveConfig("192.168.0.223:10000", "default", "hdfs", "", "")
	err := h.Conn.Open()
	fatalCheck("Populate", err)

	var student Students

	totalWorker := 10
	retVal, err := h.LoadFileWithWorker("/home/developer/contoh.txt", "students", "csv", "dd/MM/yyyy", &student, totalWorker)

	if err != nil {
		fatalCheck("Populate", err)
	}

	h.Conn.Close()
	log.Printf("retVal: \n%v\n", retVal)
}
<commit_msg>add profiling for testing purpose<commit_after>package main

import (
	. "github.com/frezadev/hdc/hive"
	// . "github.com/eaciit/hdc/hive"
	"github.com/pkg/profile"
	"log"
)

type Students struct {
	Name    string
	Age     int
	Phone   string
	Address string
}

func fatalCheck(what string, e error) {
	if e != nil {
		log.Fatalf("%s: %s", what, e.Error())
	}
}

func main() {
	defer profile.Start(profile.CPUProfile, profile.MemProfile, profile.BlockProfile).Stop()

	h := HiveConfig("192.168.0.223:10000", "default", "hdfs", "", "")
	err := h.Conn.Open()
	fatalCheck("Populate", err)

	var student Students

	totalWorker := 10
	retVal, err := h.LoadFileWithWorker("/home/developer/contoh.txt", "students", "csv", "dd/MM/yyyy", &student, totalWorker)

	if err != nil {
		fatalCheck("Populate", err)
	}

	h.Conn.Close()
	log.Printf("retVal: \n%v\n", retVal)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/docker/docker/pkg/mflag"
	. "github.com/weaveworks/weave/common"
	"github.com/weaveworks/weave/common/mflagext"
	"github.com/weaveworks/weave/proxy"
)

var (
	version = "(unreleased version)"
)

func main() {
	var (
		justVersion bool
		logLevel    = "info"
		c           = proxy.Config{ListenAddrs: []string{}}
	)

	c.Version = version

	mflag.BoolVar(&justVersion, []string{"#version", "-version"}, false, "print version and exit")
	mflag.StringVar(&logLevel, []string{"-log-level"}, "info", "logging level (debug, info, warning, error)")
	mflagext.ListVar(&c.ListenAddrs, []string{"H"}, nil, "addresses on which to listen")
	mflag.StringVar(&c.HostnameFromLabel, []string{"-hostname-from-label"}, "", "Key of container label from which to obtain the container's hostname")
	mflag.StringVar(&c.HostnameMatch, []string{"-hostname-match"}, "(.*)", "Regexp pattern to apply on container names (e.g. '^aws-[0-9]+-(.*)$')")
	mflag.StringVar(&c.HostnameReplacement, []string{"-hostname-replacement"}, "$1", "Expression to generate hostnames based on matches from --hostname-match (e.g. 'my-app-$1')")
	mflag.BoolVar(&c.RewriteInspect, []string{"-rewrite-inspect"}, false, "Rewrite 'inspect' calls to return the weave network settings (if attached)")
	mflag.BoolVar(&c.NoDefaultIPAM, []string{"#-no-default-ipam", "-no-default-ipalloc"}, false, "do not automatically allocate addresses for containers without a WEAVE_CIDR")
	mflag.BoolVar(&c.NoRewriteHosts, []string{"-no-rewrite-hosts"}, false, "do not automatically rewrite /etc/hosts. Use if you need the docker IP to remain in /etc/hosts")
	mflag.StringVar(&c.TLSConfig.CACert, []string{"#tlscacert", "-tlscacert"}, "", "Trust certs signed only by this CA")
	mflag.StringVar(&c.TLSConfig.Cert, []string{"#tlscert", "-tlscert"}, "", "Path to TLS certificate file")
	mflag.BoolVar(&c.TLSConfig.Enabled, []string{"#tls", "-tls"}, false, "Use TLS; implied by --tls-verify")
	mflag.StringVar(&c.TLSConfig.Key, []string{"#tlskey", "-tlskey"}, "", "Path to TLS key file")
	mflag.BoolVar(&c.TLSConfig.Verify, []string{"#tlsverify", "-tlsverify"}, false, "Use TLS and verify the remote")
	mflag.BoolVar(&c.WithDNS, []string{"-with-dns", "w"}, false, "instruct created containers to always use weaveDNS as their nameserver")
	mflag.BoolVar(&c.WithoutDNS, []string{"-without-dns"}, false, "instruct created containers to never use weaveDNS as their nameserver")
	mflag.Parse()

	if justVersion {
		fmt.Printf("weave proxy %s\n", version)
		os.Exit(0)
	}

	if c.WithDNS && c.WithoutDNS {
		Log.Fatalf("Cannot use both '--with-dns' and '--without-dns' flags")
	}

	SetLogLevel(logLevel)

	Log.Infoln("weave proxy", version)
	Log.Infoln("Command line arguments:", strings.Join(os.Args[1:], " "))

	p, err := proxy.NewProxy(c)
	if err != nil {
		Log.Fatalf("Could not start proxy: %s", err)
	}

	go p.ListenAndServe()
	SignalHandlerLoop()
}
<commit_msg>cosmetic: vertical alignment in version output<commit_after>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/docker/docker/pkg/mflag"
	. "github.com/weaveworks/weave/common"
	"github.com/weaveworks/weave/common/mflagext"
	"github.com/weaveworks/weave/proxy"
)

var (
	version = "(unreleased version)"
)

func main() {
	var (
		justVersion bool
		logLevel    = "info"
		c           = proxy.Config{ListenAddrs: []string{}}
	)

	c.Version = version

	mflag.BoolVar(&justVersion, []string{"#version", "-version"}, false, "print version and exit")
	mflag.StringVar(&logLevel, []string{"-log-level"}, "info", "logging level (debug, info, warning, error)")
	mflagext.ListVar(&c.ListenAddrs, []string{"H"}, nil, "addresses on which to listen")
	mflag.StringVar(&c.HostnameFromLabel, []string{"-hostname-from-label"}, "", "Key of container label from which to obtain the container's hostname")
	mflag.StringVar(&c.HostnameMatch, []string{"-hostname-match"}, "(.*)", "Regexp pattern to apply on container names (e.g. '^aws-[0-9]+-(.*)$')")
	mflag.StringVar(&c.HostnameReplacement, []string{"-hostname-replacement"}, "$1", "Expression to generate hostnames based on matches from --hostname-match (e.g. 'my-app-$1')")
	mflag.BoolVar(&c.RewriteInspect, []string{"-rewrite-inspect"}, false, "Rewrite 'inspect' calls to return the weave network settings (if attached)")
	mflag.BoolVar(&c.NoDefaultIPAM, []string{"#-no-default-ipam", "-no-default-ipalloc"}, false, "do not automatically allocate addresses for containers without a WEAVE_CIDR")
	mflag.BoolVar(&c.NoRewriteHosts, []string{"-no-rewrite-hosts"}, false, "do not automatically rewrite /etc/hosts. Use if you need the docker IP to remain in /etc/hosts")
	mflag.StringVar(&c.TLSConfig.CACert, []string{"#tlscacert", "-tlscacert"}, "", "Trust certs signed only by this CA")
	mflag.StringVar(&c.TLSConfig.Cert, []string{"#tlscert", "-tlscert"}, "", "Path to TLS certificate file")
	mflag.BoolVar(&c.TLSConfig.Enabled, []string{"#tls", "-tls"}, false, "Use TLS; implied by --tls-verify")
	mflag.StringVar(&c.TLSConfig.Key, []string{"#tlskey", "-tlskey"}, "", "Path to TLS key file")
	mflag.BoolVar(&c.TLSConfig.Verify, []string{"#tlsverify", "-tlsverify"}, false, "Use TLS and verify the remote")
	mflag.BoolVar(&c.WithDNS, []string{"-with-dns", "w"}, false, "instruct created containers to always use weaveDNS as their nameserver")
	mflag.BoolVar(&c.WithoutDNS, []string{"-without-dns"}, false, "instruct created containers to never use weaveDNS as their nameserver")
	mflag.Parse()

	if justVersion {
		fmt.Printf("weave proxy  %s\n", version)
		os.Exit(0)
	}

	if c.WithDNS && c.WithoutDNS {
		Log.Fatalf("Cannot use both '--with-dns' and '--without-dns' flags")
	}

	SetLogLevel(logLevel)

	Log.Infoln("weave proxy", version)
	Log.Infoln("Command line arguments:", strings.Join(os.Args[1:], " "))

	p, err := proxy.NewProxy(c)
	if err != nil {
		Log.Fatalf("Could not start proxy: %s", err)
	}

	go p.ListenAndServe()
	SignalHandlerLoop()
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Honnibal
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tag

import (
	"regexp"
	"strconv"
	"strings"

	"github.com/jdkato/prose/internal/model"
	"github.com/jdkato/prose/internal/util"
	"github.com/montanaflynn/stats"
	"github.com/shogo82148/go-shuffle"
)

var none = regexp.MustCompile(`^(?:0|\*[\w?]\*|\*\-\d{1,3}|\*[A-Z]+\*\-\d{1,3}|\*)$`)
var keep = regexp.MustCompile(`^\-[A-Z]{3}\-$`)

// AveragedPerceptron is a Averaged Perceptron classifier.
type AveragedPerceptron struct {
	classes   []string
	instances float64
	stamps    map[string]float64
	tagMap    map[string]string
	totals    map[string]float64
	weights   map[string]map[string]float64
}

// NewAveragedPerceptron creates a new AveragedPerceptron model.
func NewAveragedPerceptron(weights map[string]map[string]float64,
	tags map[string]string, classes []string) *AveragedPerceptron {
	return &AveragedPerceptron{
		totals: make(map[string]float64), stamps: make(map[string]float64),
		classes: classes, tagMap: tags, weights: weights}
}

// PerceptronTagger is a port of Textblob's "fast and accurate" POS tagger.
// See https://github.com/sloria/textblob-aptagger for details.
type PerceptronTagger struct {
	tagMap map[string]string
	model  *AveragedPerceptron
}

// NewPerceptronTagger creates a new PerceptronTagger and loads the built-in
// AveragedPerceptron model.
func NewPerceptronTagger() *PerceptronTagger {
	var wts map[string]map[string]float64
	var tags map[string]string
	var classes []string

	dec := model.GetAsset("classes.gob")
	util.CheckError(dec.Decode(&classes))

	dec = model.GetAsset("tags.gob")
	util.CheckError(dec.Decode(&tags))

	dec = model.GetAsset("weights.gob")
	util.CheckError(dec.Decode(&wts))

	return &PerceptronTagger{model: NewAveragedPerceptron(wts, tags, classes)}
}

// Weights returns the model's weights in the form
//
//    {
//      "i-1 suffix ity": {
//        "MD": -0.816,
//        "VB": -0.695,
//        ...
//       }
//       ...
//     }
func (pt *PerceptronTagger) Weights() map[string]map[string]float64 {
	return pt.model.weights
}

// Classes returns the model's classes in the form
//
//    ["EX", "NNPS", "WP$", ...]
func (pt *PerceptronTagger) Classes() []string {
	return pt.model.classes
}

// TagMap returns the model's classes in the form
//
//    {
//      "four": "CD",
//      "facilities": "NNS",
//      ...
//    }
func (pt *PerceptronTagger) TagMap() map[string]string {
	return pt.model.tagMap
}

// NewTrainedPerceptronTagger creates a new PerceptronTagger using the given
// model.
func NewTrainedPerceptronTagger(model *AveragedPerceptron) *PerceptronTagger {
	return &PerceptronTagger{model: model}
}

// Tag takes a slice of words and returns a slice of tagged tokens.
func (pt *PerceptronTagger) Tag(words []string) []Token {
	var tokens []Token
	var clean []string
	var tag string
	var found bool

	p1, p2 := "-START-", "-START2-"
	context := []string{p1, p2}
	for _, w := range words {
		if w == "" {
			continue
		}
		context = append(context, normalize(w))
		clean = append(clean, w)
	}
	context = append(context, []string{"-END-", "-END2-"}...)
	for i, word := range clean {
		if none.MatchString(word) {
			tag = "-NONE-"
		} else if keep.MatchString(word) {
			tag = word
		} else if tag, found = pt.model.tagMap[word]; !found {
			tag = pt.model.predict(featurize(i, context, word, p1, p2))
		}
		tokens = append(tokens, Token{Tag: tag, Text: word})
		p2 = p1
		p1 = tag
	}

	return tokens
}

// Train an Averaged Perceptron model based on sentences.
func (pt *PerceptronTagger) Train(sentences TupleSlice, iterations int) {
	var guess string
	var found bool

	pt.makeTagMap(sentences)
	for i := 0; i < iterations; i++ {
		for _, tuple := range sentences {
			words, tags := tuple[0], tuple[1]
			p1, p2 := "-START-", "-START2-"
			context := []string{p1, p2}
			for _, w := range words {
				if w == "" {
					continue
				}
				context = append(context, normalize(w))
			}
			context = append(context, []string{"-END-", "-END2-"}...)
			for i, word := range words {
				if guess, found = pt.tagMap[word]; !found {
					feats := featurize(i, context, word, p1, p2)
					guess = pt.model.predict(feats)
					pt.model.update(tags[i], guess, feats)
				}
				p2 = p1
				p1 = guess
			}
		}
		shuffle.Shuffle(sentences)
	}
	pt.model.averageWeights()
}

func (pt *PerceptronTagger) makeTagMap(sentences TupleSlice) {
	counts := make(map[string]map[string]int)
	for _, tuple := range sentences {
		words, tags := tuple[0], tuple[1]
		for i, word := range words {
			tag := tags[i]
			if counts[word] == nil {
				counts[word] = make(map[string]int)
			}
			counts[word][tag]++
			pt.model.addClass(tag)
		}
	}
	for word, tagFreqs := range counts {
		tag, mode := maxValue(tagFreqs)
		n := float64(sumValues(tagFreqs))
		if n >= 20 && (float64(mode)/n) >= 0.97 {
			pt.tagMap[word] = tag
		}
	}
}

func (ap *AveragedPerceptron) predict(features map[string]float64) string {
	var weights map[string]float64
	var found bool

	scores := make(map[string]float64)
	for feat, value := range features {
		if weights, found = ap.weights[feat]; !found || value == 0 {
			continue
		}
		for label, weight := range weights {
			if _, ok := scores[label]; ok {
				scores[label] += value * weight
			} else {
				scores[label] = value * weight
			}
		}
	}
	return max(scores)
}

func (ap *AveragedPerceptron) update(truth, guess string, feats map[string]float64) {
	ap.instances++
	if truth == guess {
		return
	}
	for f := range feats {
		weights := make(map[string]float64)
		if val, ok := ap.weights[f]; ok {
			weights = val
		} else {
			ap.weights[f] = weights
		}
		ap.updateFeat(truth, f, get(truth, weights), 1.0)
		ap.updateFeat(guess, f, get(guess, weights), -1.0)
	}
}

func (ap *AveragedPerceptron) updateFeat(c, f string, v, w float64) {
	key := f + "-" + c
	ap.totals[key] = (ap.instances - ap.stamps[key]) * w
	ap.stamps[key] = ap.instances
	ap.weights[f][c] = w + v
}

func (ap *AveragedPerceptron) addClass(class string) {
	if !util.StringInSlice(class, ap.classes) {
		ap.classes = append(ap.classes, class)
	}
}

func (ap *AveragedPerceptron) averageWeights() {
	for feat, weights := range ap.weights {
		newWeights := make(map[string]float64)
		for class, weight := range weights {
			key := feat + "-" + class
			total := ap.totals[key]
			total += (ap.instances - ap.stamps[key]) * weight
			averaged, _ := stats.Round(total/ap.instances, 3)
			if averaged != 0.0 {
				newWeights[class] = averaged
			}
		}
		ap.weights[feat] = newWeights
	}
}

func max(scores map[string]float64) string {
	var class string
	max := 0.0
	for label, value := range scores {
		if value > max {
			max = value
			class = label
		}
	}
	return class
}

func featurize(i int, ctx []string, w, p1, p2 string) map[string]float64 {
	feats := make(map[string]float64)
	suf := util.Min(len(w), 3)
	i = util.Min(len(ctx)-2, i+2)
	iminus := util.Min(len(ctx[i-1]), 3)
	iplus := util.Min(len(ctx[i+1]), 3)
	feats = add([]string{"bias"}, feats)
	feats = add([]string{"i suffix", w[len(w)-suf:]}, feats)
	feats = add([]string{"i pref1", string(w[0])}, feats)
	feats = add([]string{"i-1 tag", p1}, feats)
	feats = add([]string{"i-2 tag", p2}, feats)
	feats = add([]string{"i tag+i-2 tag", p1, p2}, feats)
	feats = add([]string{"i word", ctx[i]}, feats)
	feats = add([]string{"i-1 tag+i word", p1, ctx[i]}, feats)
	feats = add([]string{"i-1 word", ctx[i-1]}, feats)
	feats = add([]string{"i-1 suffix", ctx[i-1][len(ctx[i-1])-iminus:]}, feats)
	feats = add([]string{"i-2 word", ctx[i-2]}, feats)
	feats = add([]string{"i+1 word", ctx[i+1]}, feats)
	feats = add([]string{"i+1 suffix", ctx[i+1][len(ctx[i+1])-iplus:]}, feats)
	feats = add([]string{"i+2 word", ctx[i+2]}, feats)
	return feats
}

func add(args []string, features map[string]float64) map[string]float64 {
	key := strings.Join(args, " ")
	if _, ok := features[key]; ok {
		features[key]++
	} else {
		features[key] = 1
	}
	return features
}

func normalize(word string) string {
	if word == "" {
		return word
	}
	first := string(word[0])
	if strings.Contains(word, "-") && first != "-" {
		return "!HYPHEN"
	} else if _, err := strconv.Atoi(word); err == nil && len(word) == 4 {
		return "!YEAR"
	} else if _, err := strconv.Atoi(first); err == nil {
		return "!DIGITS"
	}
	return strings.ToLower(word)
}

func sumValues(m map[string]int) int {
	sum := 0
	for _, v := range m {
		sum += v
	}
	return sum
}

func maxValue(m map[string]int) (string, int) {
	maxValue := 0
	key := ""
	for k, v := range m {
		if v >= maxValue {
			maxValue = v
			key = k
		}
	}
	return key, maxValue
}

func get(k string, m map[string]float64) float64 {
	if v, ok := m[k]; ok {
		return v
	}
	return 0.0
}
<commit_msg>Fix GoDoc bracket alignment<commit_after>// Copyright 2013 Matthew Honnibal
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tag

import (
	"regexp"
	"strconv"
	"strings"

	"github.com/jdkato/prose/internal/model"
	"github.com/jdkato/prose/internal/util"
	"github.com/montanaflynn/stats"
	"github.com/shogo82148/go-shuffle"
)

var none = regexp.MustCompile(`^(?:0|\*[\w?]\*|\*\-\d{1,3}|\*[A-Z]+\*\-\d{1,3}|\*)$`)
var keep = regexp.MustCompile(`^\-[A-Z]{3}\-$`)

// AveragedPerceptron is a Averaged Perceptron classifier.
type AveragedPerceptron struct {
	classes   []string
	instances float64
	stamps    map[string]float64
	tagMap    map[string]string
	totals    map[string]float64
	weights   map[string]map[string]float64
}

// NewAveragedPerceptron creates a new AveragedPerceptron model.
func NewAveragedPerceptron(weights map[string]map[string]float64,
	tags map[string]string, classes []string) *AveragedPerceptron {
	return &AveragedPerceptron{
		totals: make(map[string]float64), stamps: make(map[string]float64),
		classes: classes, tagMap: tags, weights: weights}
}

// PerceptronTagger is a port of Textblob's "fast and accurate" POS tagger.
// See https://github.com/sloria/textblob-aptagger for details.
type PerceptronTagger struct {
	tagMap map[string]string
	model  *AveragedPerceptron
}

// NewPerceptronTagger creates a new PerceptronTagger and loads the built-in
// AveragedPerceptron model.
func NewPerceptronTagger() *PerceptronTagger {
	var wts map[string]map[string]float64
	var tags map[string]string
	var classes []string

	dec := model.GetAsset("classes.gob")
	util.CheckError(dec.Decode(&classes))

	dec = model.GetAsset("tags.gob")
	util.CheckError(dec.Decode(&tags))

	dec = model.GetAsset("weights.gob")
	util.CheckError(dec.Decode(&wts))

	return &PerceptronTagger{model: NewAveragedPerceptron(wts, tags, classes)}
}

// Weights returns the model's weights in the form
//
//    {
//      "i-1 suffix ity": {
//        "MD": -0.816,
//        "VB": -0.695,
//        ...
//       }
//       ...
//    }
func (pt *PerceptronTagger) Weights() map[string]map[string]float64 {
	return pt.model.weights
}

// Classes returns the model's classes in the form
//
//    ["EX", "NNPS", "WP$", ...]
func (pt *PerceptronTagger) Classes() []string {
	return pt.model.classes
}

// TagMap returns the model's classes in the form
//
//    {
//      "four": "CD",
//      "facilities": "NNS",
//      ...
//    }
func (pt *PerceptronTagger) TagMap() map[string]string {
	return pt.model.tagMap
}

// NewTrainedPerceptronTagger creates a new PerceptronTagger using the given
// model.
func NewTrainedPerceptronTagger(model *AveragedPerceptron) *PerceptronTagger {
	return &PerceptronTagger{model: model}
}

// Tag takes a slice of words and returns a slice of tagged tokens.
func (pt *PerceptronTagger) Tag(words []string) []Token {
	var tokens []Token
	var clean []string
	var tag string
	var found bool

	p1, p2 := "-START-", "-START2-"
	context := []string{p1, p2}
	for _, w := range words {
		if w == "" {
			continue
		}
		context = append(context, normalize(w))
		clean = append(clean, w)
	}
	context = append(context, []string{"-END-", "-END2-"}...)
	for i, word := range clean {
		if none.MatchString(word) {
			tag = "-NONE-"
		} else if keep.MatchString(word) {
			tag = word
		} else if tag, found = pt.model.tagMap[word]; !found {
			tag = pt.model.predict(featurize(i, context, word, p1, p2))
		}
		tokens = append(tokens, Token{Tag: tag, Text: word})
		p2 = p1
		p1 = tag
	}

	return tokens
}

// Train an Averaged Perceptron model based on sentences.
func (pt *PerceptronTagger) Train(sentences TupleSlice, iterations int) {
	var guess string
	var found bool

	pt.makeTagMap(sentences)
	for i := 0; i < iterations; i++ {
		for _, tuple := range sentences {
			words, tags := tuple[0], tuple[1]
			p1, p2 := "-START-", "-START2-"
			context := []string{p1, p2}
			for _, w := range words {
				if w == "" {
					continue
				}
				context = append(context, normalize(w))
			}
			context = append(context, []string{"-END-", "-END2-"}...)
			for i, word := range words {
				if guess, found = pt.tagMap[word]; !found {
					feats := featurize(i, context, word, p1, p2)
					guess = pt.model.predict(feats)
					pt.model.update(tags[i], guess, feats)
				}
				p2 = p1
				p1 = guess
			}
		}
		shuffle.Shuffle(sentences)
	}
	pt.model.averageWeights()
}

func (pt *PerceptronTagger) makeTagMap(sentences TupleSlice) {
	counts := make(map[string]map[string]int)
	for _, tuple := range sentences {
		words, tags := tuple[0], tuple[1]
		for i, word := range words {
			tag := tags[i]
			if counts[word] == nil {
				counts[word] = make(map[string]int)
			}
			counts[word][tag]++
			pt.model.addClass(tag)
		}
	}
	for word, tagFreqs := range counts {
		tag, mode := maxValue(tagFreqs)
		n := float64(sumValues(tagFreqs))
		if n >= 20 && (float64(mode)/n) >= 0.97 {
			pt.tagMap[word] = tag
		}
	}
}

func (ap *AveragedPerceptron) predict(features map[string]float64) string {
	var weights map[string]float64
	var found bool

	scores := make(map[string]float64)
	for feat, value := range features {
		if weights, found = ap.weights[feat]; !found || value == 0 {
			continue
		}
		for label, weight := range weights {
			if _, ok := scores[label]; ok {
				scores[label] += value * weight
			} else {
				scores[label] = value * weight
			}
		}
	}
	return max(scores)
}

func (ap *AveragedPerceptron) update(truth, guess string, feats map[string]float64) {
	ap.instances++
	if truth == guess {
		return
	}
	for f := range feats {
		weights := make(map[string]float64)
		if val, ok := ap.weights[f]; ok {
			weights = val
		} else {
			ap.weights[f] = weights
		}
		ap.updateFeat(truth, f, get(truth, weights), 1.0)
		ap.updateFeat(guess, f, get(guess, weights), -1.0)
	}
}

func (ap *AveragedPerceptron) updateFeat(c, f string, v, w float64) {
	key := f + "-" + c
	ap.totals[key] = (ap.instances - ap.stamps[key]) * w
	ap.stamps[key] = ap.instances
	ap.weights[f][c] = w + v
}

func (ap *AveragedPerceptron) addClass(class string) {
	if !util.StringInSlice(class, ap.classes) {
		ap.classes = append(ap.classes, class)
	}
}

func (ap *AveragedPerceptron) averageWeights() {
	for feat, weights := range ap.weights {
		newWeights := make(map[string]float64)
		for class, weight := range weights {
			key := feat + "-" + class
			total := ap.totals[key]
			total += (ap.instances - ap.stamps[key]) * weight
			averaged, _ := stats.Round(total/ap.instances, 3)
			if averaged != 0.0 {
				newWeights[class] = averaged
			}
		}
		ap.weights[feat] = newWeights
	}
}

func max(scores map[string]float64) string {
	var class string
	max := 0.0
	for label, value := range scores {
		if value > max {
			max = value
			class = label
		}
	}
	return class
}

func featurize(i int, ctx []string, w, p1, p2 string) map[string]float64 {
	feats := make(map[string]float64)
	suf := util.Min(len(w), 3)
	i = util.Min(len(ctx)-2, i+2)
	iminus := util.Min(len(ctx[i-1]), 3)
	iplus := util.Min(len(ctx[i+1]), 3)
	feats = add([]string{"bias"}, feats)
	feats = add([]string{"i suffix", w[len(w)-suf:]}, feats)
	feats = add([]string{"i pref1", string(w[0])}, feats)
	feats = add([]string{"i-1 tag", p1}, feats)
	feats = add([]string{"i-2 tag", p2}, feats)
	feats = add([]string{"i tag+i-2 tag", p1, p2}, feats)
	feats = add([]string{"i word", ctx[i]}, feats)
	feats = add([]string{"i-1 tag+i word", p1, ctx[i]}, feats)
	feats = add([]string{"i-1 word", ctx[i-1]}, feats)
	feats = add([]string{"i-1 suffix", ctx[i-1][len(ctx[i-1])-iminus:]}, feats)
	feats = add([]string{"i-2 word", ctx[i-2]}, feats)
	feats = add([]string{"i+1 word", ctx[i+1]}, feats)
	feats = add([]string{"i+1 suffix", ctx[i+1][len(ctx[i+1])-iplus:]}, feats)
	feats = add([]string{"i+2 word", ctx[i+2]}, feats)
	return feats
}

func add(args []string, features map[string]float64) map[string]float64 {
	key := strings.Join(args, " ")
	if _, ok := features[key]; ok {
		features[key]++
	} else {
		features[key] = 1
	}
	return features
}

func normalize(word string) string {
	if word == "" {
		return word
	}
	first := string(word[0])
	if strings.Contains(word, "-") && first != "-" {
		return "!HYPHEN"
	} else if _, err := strconv.Atoi(word); err == nil && len(word) == 4 {
		return "!YEAR"
	} else if _, err := strconv.Atoi(first); err == nil {
		return "!DIGITS"
	}
	return strings.ToLower(word)
}

func sumValues(m map[string]int) int {
	sum := 0
	for _, v := range m {
		sum += v
	}
	return sum
}

func maxValue(m map[string]int) (string, int) {
	maxValue := 0
	key := ""
	for k, v := range m {
		if v >= maxValue {
			maxValue = v
			key = k
		}
	}
	return key, maxValue
}

func get(k string, m map[string]float64) float64 {
	if v, ok := m[k]; ok {
		return v
	}
	return 0.0
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"time"
)

var fflag = flag.Bool("f", false, "After printing the tail as usual, follow additions to the file and print them.")
var nflag = flag.Int("n", 10, "Set the number of `lines` at the end of file to be printed.")

func init() {
	log.SetPrefix("tailf: ")
	log.SetFlags(0)
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: tail [options] file")
		flag.PrintDefaults()
	}
	flag.Parse()
	if len(flag.Args()) != 1 {
		flag.Usage()
		os.Exit(1)
	}
}

func main() {
	f, err := os.Open(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}
	seek(f, *nflag)
	buf := make([]byte, 8192)
	tcopy(f, buf)
	if *fflag {
		for range time.NewTicker(500 * time.Millisecond).C {
			tcopy(f, buf)
		}
	}
}

// Count n lines backwards from the end of the file.
func seek(f *os.File, n int) {
	// seek to just before the last byte of the file, so we can compare it to '\n'.
	if _, err := f.Seek(-1, 2); err != nil {
		return
	}
	buf := make([]byte, 1)

	// stop only after encountering the *nflag+1th newline, so that the line
	// containing the 10th encountered newline is also printed.
	remaining := n + 1

	var pos int64
	for {
		n, err := f.Read(buf)
		if err != nil {
			log.Fatal(err)
		}
		pos += int64(n)
		if buf[0] == '\n' {
			remaining--
		}
		if remaining == 0 {
			break
		}
		// seek back 2 bytes: one for what we read, one for backwards progress
		pos, err = f.Seek(-2, 1)
		if err != nil {
			// if that fails, at least try to seek back what we just read
			f.Seek(-1, 1)
			return
		}
		if pos == 0 {
			break
		}
	}
}

func tcopy(f *os.File, buf []byte) {
	_, err := io.CopyBuffer(os.Stdout, f, buf)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>tail: simple handling of stdin<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"time"
)

var fflag = flag.Bool("f", false, "After printing the tail as usual, follow additions to the file and print them.")
var nflag = flag.Int("n", 10, "Set the number of `lines` at the end of file to be printed.")

func main() {
	log.SetPrefix("tailf: ")
	log.SetFlags(0)

	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: tail [options] [file]")
		flag.PrintDefaults()
	}
	flag.Parse()
	if flag.NArg() > 1 {
		flag.Usage()
		os.Exit(1)
	}

	// can't seek stdin; read it all and print a slice of it
	if flag.NArg() == 0 {
		buf, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			log.Fatal(err)
		}
		sep := []byte("\n")
		p := buf
		i := len(p) - 1
		for lines := 0; lines <= *nflag; lines++ {
			i = bytes.LastIndex(p, sep)
			if i < 0 {
				break
			}
			p = p[:i]
		}
		i += 1 // don't print the newline itself
		os.Stdout.Write(buf[i:])
		return
	}

	f, err := os.Open(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}
	seek(f, *nflag)
	buf := make([]byte, 8192)
	tcopy(f, buf)
	if *fflag {
		for range time.NewTicker(500 * time.Millisecond).C {
			tcopy(f, buf)
		}
	}
}

// Count n lines backwards from the end of the file.
func seek(f *os.File, n int) {
	// seek to just before the last byte of the file, so we can compare it to '\n'.
	if _, err := f.Seek(-1, 2); err != nil {
		return
	}
	buf := make([]byte, 1)

	// stop only after encountering the *nflag+1th newline, so that the line
	// containing the 10th encountered newline is also printed.
	remaining := n + 1

	var pos int64
	for {
		n, err := f.Read(buf)
		if err != nil {
			log.Fatal(err)
		}
		pos += int64(n)
		if buf[0] == '\n' {
			remaining--
		}
		if remaining == 0 {
			break
		}
		// seek back 2 bytes: one for what we read, one for backwards progress
		pos, err = f.Seek(-2, 1)
		if err != nil {
			// if that fails, at least try to seek back what we just read
			f.Seek(-1, 1)
			return
		}
		if pos == 0 {
			break
		}
	}
}

func tcopy(f *os.File, buf []byte) {
	_, err := io.CopyBuffer(os.Stdout, f, buf)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package task

import "time"

type Task struct {
	Name       string                 `json:"task"`
	ID         string                 `json:"id"`
	Args       []string               `json:"args"`
	KWArgs     map[string]interface{} `json:"kwargs"`
	Retries    int                    `json:"retries"`
	ETA        *time.Time             `json:"eta"`
	Expires    *time.Time             `json:"expires"`
	UTC        bool                   `json:"utc"`
	Callbacks  []string               `json:"callbacks"`
	Errbacks   []string               `json:"errbacks"`
	TimeLimits [2]*float64            `json:"timelimit"`
	TaskSet    *string                `json:"taskset"`
	Chord      *string                `json:"chord"`
}
<commit_msg>Add omit empty to optional task fields<commit_after>package task

import "time"

type Task struct {
	Name       string                 `json:"task"`
	ID         string                 `json:"id"`
	Args       []string               `json:"args,omitempty"`
	KWArgs     map[string]interface{} `json:"kwargs,omitempty"`
	Retries    int                    `json:"retries,omitempty"`
	ETA        *time.Time             `json:"eta,omitempty"`
	Expires    *time.Time             `json:"expires,omitempty"`
	UTC        bool                   `json:"utc,omitempty"`
	Callbacks  []string               `json:"callbacks,omitempty"`
	Errbacks   []string               `json:"errbacks,omitempty"`
	TimeLimits [2]*float64            `json:"timelimit,omitempty"`
	TaskSet    *string                `json:"taskset,omitempty"`
	Chord      *string                `json:"chord,omitempty"`
}
<|endoftext|>