<commit_before>package RethinkDBStorage

import (
	"github.com/RangelReale/osin"
	"github.com/mitchellh/mapstructure"
	r "gopkg.in/dancannon/gorethink.v1"
)

const (
	clientsTable      = "oauth_clients"
	authorizeTable    = "oauth_authorize_data"
	accessTable       = "oauth_access_data"
	accessTokenField  = "AccessToken"
	refreshTokenField = "RefreshToken"
)

// RethinkDBStorage implements storage for osin
type RethinkDBStorage struct {
	session *r.Session
}

// New initializes and returns a new RethinkDBStorage
func New(session *r.Session) *RethinkDBStorage {
	storage := &RethinkDBStorage{session}
	return storage
}

// Clone the storage if needed.
func (s *RethinkDBStorage) Clone() osin.Storage {
	return s
}

// Close the resources the Storage potentially holds
func (s *RethinkDBStorage) Close() {}

// CreateClient inserts a new client
func (s *RethinkDBStorage) CreateClient(c osin.Client) error {
	_, err := r.Table(clientsTable).Insert(c).RunWrite(s.session)
	return err
}

// GetClient returns client with given ID
func (s *RethinkDBStorage) GetClient(clientID string) (osin.Client, error) {
	result, err := r.Table(clientsTable).Filter(r.Row.Field("Id").Eq(clientID)).Run(s.session)
	if err != nil {
		return nil, err
	}
	defer result.Close()

	var clientMap map[string]interface{}
	err = result.One(&clientMap)
	if err != nil {
		return nil, err
	}

	var clientStruct osin.DefaultClient
	err = mapstructure.Decode(clientMap, &clientStruct)
	if err != nil {
		return nil, err
	}

	return &clientStruct, nil
}

// UpdateClient updates given client
func (s *RethinkDBStorage) UpdateClient(c osin.Client) error {
	result, err := r.Table(clientsTable).Filter(r.Row.Field("Id").Eq(c.GetId())).Run(s.session)
	if err != nil {
		return err
	}
	defer result.Close()

	var clientMap map[string]interface{}
	err = result.One(&clientMap)
	if err != nil {
		return err
	}

	_, err = r.Table(clientsTable).Get(clientMap["id"]).Update(c).RunWrite(s.session)
	return err
}

// DeleteClient deletes given client
func (s *RethinkDBStorage) DeleteClient(c osin.Client) error {
	result, err := r.Table(clientsTable).Filter(r.Row.Field("Id").Eq(c.GetId())).Run(s.session)
	if err != nil {
		return err
	}
	defer result.Close()

	var clientMap map[string]interface{}
	err = result.One(&clientMap)
	if err != nil {
		return err
	}

	_, err = r.Table(clientsTable).Get(clientMap["id"]).Delete().RunWrite(s.session)
	return err
}

// SaveAuthorize creates a new authorization
func (s *RethinkDBStorage) SaveAuthorize(data *osin.AuthorizeData) error {
	_, err := r.Table(authorizeTable).Insert(data).RunWrite(s.session)
	return err
}

// LoadAuthorize gets authorization data with given code
func (s *RethinkDBStorage) LoadAuthorize(code string) (*osin.AuthorizeData, error) {
	result, err := r.Table(authorizeTable).Filter(r.Row.Field("Code").Eq(code)).Run(s.session)
	if err != nil {
		return nil, err
	}
	defer result.Close()

	var dataMap map[string]interface{}
	err = result.One(&dataMap)
	if err != nil {
		return nil, err
	}

	var client osin.Client
	clientID := dataMap["Client"].(map[string]interface{})["Id"].(string)
	client, err = s.GetClient(clientID)
	if err != nil {
		return nil, err
	}
	dataMap["Client"] = client

	var dataStruct osin.AuthorizeData
	err = mapstructure.Decode(dataMap, &dataStruct)
	if err != nil {
		return nil, err
	}

	return &dataStruct, nil
}

// RemoveAuthorize deletes given authorization
func (s *RethinkDBStorage) RemoveAuthorize(code string) error {
	result, err := r.Table(authorizeTable).Filter(r.Row.Field("Code").Eq(code)).Run(s.session)
	if err != nil {
		return err
	}
	defer result.Close()

	var dataMap map[string]interface{}
	err = result.One(&dataMap)
	if err != nil {
		return err
	}

	_, err = r.Table(authorizeTable).Get(dataMap["id"]).Delete().RunWrite(s.session)
	return err
}

// SaveAccess creates a new access data
func (s *RethinkDBStorage) SaveAccess(data *osin.AccessData) error {
	_, err := r.Table(accessTable).Insert(data).RunWrite(s.session)
	return err
}

// LoadAccess gets access data with given access token
func (s *RethinkDBStorage) LoadAccess(accessToken string) (*osin.AccessData, error) {
	return s.getAccessData(accessTokenField, accessToken)
}

// RemoveAccess deletes AccessData with given access token
func (s *RethinkDBStorage) RemoveAccess(accessToken string) error {
	return s.removeAccessData(accessTokenField, accessToken)
}

// LoadRefresh gets access data with given refresh token
func (s *RethinkDBStorage) LoadRefresh(refreshToken string) (*osin.AccessData, error) {
	return s.getAccessData(refreshTokenField, refreshToken)
}

// RemoveRefresh deletes AccessData with given refresh token
func (s *RethinkDBStorage) RemoveRefresh(refreshToken string) error {
	return s.removeAccessData(refreshTokenField, refreshToken)
}

// getAccessData is a common function to get AccessData by field
func (s *RethinkDBStorage) getAccessData(fieldName, token string) (*osin.AccessData, error) {
	result, err := r.Table(accessTable).Filter(r.Row.Field(fieldName).Eq(token)).Run(s.session)
	if err != nil {
		return nil, err
	}
	defer result.Close()

	var dataMap map[string]interface{}
	err = result.One(&dataMap)
	if err != nil {
		return nil, err
	}

	var client osin.Client
	clientID := dataMap["Client"].(map[string]interface{})["Id"].(string)
	client, err = s.GetClient(clientID)
	if err != nil {
		return nil, err
	}
	dataMap["Client"] = client

	if authorizeData := dataMap["AuthorizeData"]; authorizeData != nil {
		if authorizeDataClient := authorizeData.(map[string]interface{})["Client"]; authorizeDataClient != nil {
			var authorizeDataClientStruct osin.Client
			if authorizeDataClientID := authorizeDataClient.(map[string]interface{})["Id"]; authorizeDataClientID != nil {
				authorizeDataClientStruct, err = s.GetClient(authorizeDataClientID.(string))
				if err != nil {
					return nil, err
				}
				dataMap["AuthorizeData"].(map[string]interface{})["Client"] = authorizeDataClientStruct
			}
		}
	}

	var dataStruct osin.AccessData
	err = mapstructure.Decode(dataMap, &dataStruct)
	if err != nil {
		return nil, err
	}

	return &dataStruct, nil
}

// removeAccessData is a common function to remove AccessData by field
func (s *RethinkDBStorage) removeAccessData(fieldName, token string) error {
	result, err := r.Table(accessTable).Filter(r.Row.Field(fieldName).Eq(token)).Run(s.session)
	if err != nil {
		return err
	}
	defer result.Close()

	var dataMap map[string]interface{}
	err = result.One(&dataMap)
	if err != nil {
		return err
	}

	_, err = r.Table(accessTable).Get(dataMap["id"]).Delete().RunWrite(s.session)
	return err
}
<commit_msg>fix issue with nested AccessData when using refresh tokens<commit_after>package RethinkDBStorage

import (
	"github.com/RangelReale/osin"
	"github.com/mitchellh/mapstructure"
	r "gopkg.in/dancannon/gorethink.v1"
)

const (
	clientsTable      = "oauth_clients"
	authorizeTable    = "oauth_authorize_data"
	accessTable       = "oauth_access_data"
	accessTokenField  = "AccessToken"
	refreshTokenField = "RefreshToken"
)

// RethinkDBStorage implements storage for osin
type RethinkDBStorage struct {
	session *r.Session
}

// New initializes and returns a new RethinkDBStorage
func New(session *r.Session) *RethinkDBStorage {
	storage := &RethinkDBStorage{session}
	return storage
}

// Clone the storage if needed.
func (s *RethinkDBStorage) Clone() osin.Storage {
	return s
}

// Close the resources the Storage potentially holds
func (s *RethinkDBStorage) Close() {}

// CreateClient inserts a new client
func (s *RethinkDBStorage) CreateClient(c osin.Client) error {
	_, err := r.Table(clientsTable).Insert(c).RunWrite(s.session)
	return err
}

// GetClient returns client with given ID
func (s *RethinkDBStorage) GetClient(clientID string) (osin.Client, error) {
	result, err := r.Table(clientsTable).Filter(r.Row.Field("Id").Eq(clientID)).Run(s.session)
	if err != nil {
		return nil, err
	}
	defer result.Close()

	var clientMap map[string]interface{}
	err = result.One(&clientMap)
	if err != nil {
		return nil, err
	}

	var clientStruct osin.DefaultClient
	err = mapstructure.Decode(clientMap, &clientStruct)
	if err != nil {
		return nil, err
	}

	return &clientStruct, nil
}

// UpdateClient updates given client
func (s *RethinkDBStorage) UpdateClient(c osin.Client) error {
	result, err := r.Table(clientsTable).Filter(r.Row.Field("Id").Eq(c.GetId())).Run(s.session)
	if err != nil {
		return err
	}
	defer result.Close()

	var clientMap map[string]interface{}
	err = result.One(&clientMap)
	if err != nil {
		return err
	}

	_, err = r.Table(clientsTable).Get(clientMap["id"]).Update(c).RunWrite(s.session)
	return err
}

// DeleteClient deletes given client
func (s *RethinkDBStorage) DeleteClient(c osin.Client) error {
	result, err := r.Table(clientsTable).Filter(r.Row.Field("Id").Eq(c.GetId())).Run(s.session)
	if err != nil {
		return err
	}
	defer result.Close()

	var clientMap map[string]interface{}
	err = result.One(&clientMap)
	if err != nil {
		return err
	}

	_, err = r.Table(clientsTable).Get(clientMap["id"]).Delete().RunWrite(s.session)
	return err
}

// SaveAuthorize creates a new authorization
func (s *RethinkDBStorage) SaveAuthorize(data *osin.AuthorizeData) error {
	_, err := r.Table(authorizeTable).Insert(data).RunWrite(s.session)
	return err
}

// LoadAuthorize gets authorization data with given code
func (s *RethinkDBStorage) LoadAuthorize(code string) (*osin.AuthorizeData, error) {
	result, err := r.Table(authorizeTable).Filter(r.Row.Field("Code").Eq(code)).Run(s.session)
	if err != nil {
		return nil, err
	}
	defer result.Close()

	var dataMap map[string]interface{}
	err = result.One(&dataMap)
	if err != nil {
		return nil, err
	}

	var client osin.Client
	clientID := dataMap["Client"].(map[string]interface{})["Id"].(string)
	client, err = s.GetClient(clientID)
	if err != nil {
		return nil, err
	}
	dataMap["Client"] = client

	var dataStruct osin.AuthorizeData
	err = mapstructure.Decode(dataMap, &dataStruct)
	if err != nil {
		return nil, err
	}

	return &dataStruct, nil
}

// RemoveAuthorize deletes given authorization
func (s *RethinkDBStorage) RemoveAuthorize(code string) error {
	result, err := r.Table(authorizeTable).Filter(r.Row.Field("Code").Eq(code)).Run(s.session)
	if err != nil {
		return err
	}
	defer result.Close()

	var dataMap map[string]interface{}
	err = result.One(&dataMap)
	if err != nil {
		return err
	}

	_, err = r.Table(authorizeTable).Get(dataMap["id"]).Delete().RunWrite(s.session)
	return err
}

// SaveAccess creates a new access data
func (s *RethinkDBStorage) SaveAccess(data *osin.AccessData) error {
	// limit nested AccessData to one level
	if data.AccessData != nil {
		data.AccessData.AccessData = nil
	}

	_, err := r.Table(accessTable).Insert(data).RunWrite(s.session)
	return err
}

// LoadAccess gets access data with given access token
func (s *RethinkDBStorage) LoadAccess(accessToken string) (*osin.AccessData, error) {
	return s.getAccessData(accessTokenField, accessToken)
}

// RemoveAccess deletes AccessData with given access token
func (s *RethinkDBStorage) RemoveAccess(accessToken string) error {
	return s.removeAccessData(accessTokenField, accessToken)
}

// LoadRefresh gets access data with given refresh token
func (s *RethinkDBStorage) LoadRefresh(refreshToken string) (*osin.AccessData, error) {
	return s.getAccessData(refreshTokenField, refreshToken)
}

// RemoveRefresh deletes AccessData with given refresh token
func (s *RethinkDBStorage) RemoveRefresh(refreshToken string) error {
	return s.removeAccessData(refreshTokenField, refreshToken)
}

// getAccessData is a common function to get AccessData by field
func (s *RethinkDBStorage) getAccessData(fieldName, token string) (*osin.AccessData, error) {
	result, err := r.Table(accessTable).Filter(r.Row.Field(fieldName).Eq(token)).Run(s.session)
	if err != nil {
		return nil, err
	}
	defer result.Close()

	var dataMap map[string]interface{}
	err = result.One(&dataMap)
	if err != nil {
		return nil, err
	}

	var client osin.Client
	clientID := dataMap["Client"].(map[string]interface{})["Id"].(string)
	client, err = s.GetClient(clientID)
	if err != nil {
		return nil, err
	}
	dataMap["Client"] = client

	if authorizeData := dataMap["AuthorizeData"]; authorizeData != nil {
		if authorizeDataClient := authorizeData.(map[string]interface{})["Client"]; authorizeDataClient != nil {
			var authorizeDataClientStruct osin.Client
			if authorizeDataClientID := authorizeDataClient.(map[string]interface{})["Id"]; authorizeDataClientID != nil {
				authorizeDataClientStruct, err = s.GetClient(authorizeDataClientID.(string))
				if err != nil {
					return nil, err
				}
				dataMap["AuthorizeData"].(map[string]interface{})["Client"] = authorizeDataClientStruct
			}
		}
	}

	if accessData := dataMap["AccessData"]; accessData != nil {

		if accessDataClient := accessData.(map[string]interface{})["Client"]; accessDataClient != nil {
			var accessDataClientStruct osin.Client
			if accessDataClientID := accessDataClient.(map[string]interface{})["Id"]; accessDataClientID != nil {
				accessDataClientStruct, err = s.GetClient(accessDataClientID.(string))
				if err != nil {
					return nil, err
				}
				dataMap["AccessData"].(map[string]interface{})["Client"] = accessDataClientStruct
			}
		}
	}

	var dataStruct osin.AccessData
	err = mapstructure.Decode(dataMap, &dataStruct)
	if err != nil {
		return nil, err
	}

	return &dataStruct, nil
}

// removeAccessData is a common function to remove AccessData by field
func (s *RethinkDBStorage) removeAccessData(fieldName, token string) error {
	result, err := r.Table(accessTable).Filter(r.Row.Field(fieldName).Eq(token)).Run(s.session)
	if err != nil {
		return err
	}
	defer result.Close()

	var dataMap map[string]interface{}
	err = result.One(&dataMap)
	if err != nil {
		return err
	}

	_, err = r.Table(accessTable).Get(dataMap["id"]).Delete().RunWrite(s.session)
	return err
}
<|endoftext|>


<commit_before>package eval

import (
	"fmt"

	"github.com/cirbo-lang/cirbo/cty"
	"github.com/cirbo-lang/cirbo/source"
)

type Expr interface {
	// value evaluates the expression in the given context.
	//
	// If the result is being used directly as the definition of a symbol
	// then the that symbol is provided in "targetSym"; otherwise,
	// targetSym is nil. Some expression types are valid only when being
	// assigned directly to a symbol.
	value(ctx *Context, targetSym *Symbol) (cty.Value, source.Diags)

	// eachChild should pass each of its child expressions (e.g. operands)
	// to the given callback in some reasonable order.
	eachChild(cb walkCb)

	// sourceRange returns the source code range that represents the receiving
	// expression.
	sourceRange() source.Range
}

type symbolExpr struct {
	sym *Symbol
	rng
	leafExpr
}

func SymbolExpr(sym *Symbol, rng source.Range) Expr {
	return &symbolExpr{
		sym: sym,
		rng: srcRange(rng),
	}
}

func (e *symbolExpr) value(ctx *Context, targetSym *Symbol) (cty.Value, source.Diags) {
	val := ctx.Value(e.sym)
	if val == cty.NilValue {
		// This is actually an implementation error in Cirbo rather than a
		// user error, but we'll return it as a diagnostic anyway since that's
		// more graceful.
		return cty.PlaceholderVal, source.Diags{
			{
				Level:   source.Error,
				Summary: "Symbol not yet defined",
				Detail:  fmt.Sprintf("The symbol %q has not yet been defined. This is a bug in Cirbo that should be reported!", e.sym.name),
				Ranges:  e.sourceRange().List(),
			},
		}
	}
	return val, nil
}

func (e *symbolExpr) GoString() string {
	return fmt.Sprintf("eval.SymbolExpr(%#v, %#v)", e.sym, e.rng.sourceRange())
}

type literalExpr struct {
	val cty.Value
	rng
	leafExpr
}

func LiteralExpr(val cty.Value, rng source.Range) Expr {
	return &literalExpr{
		val: val,
		rng: srcRange(rng),
	}
}

func (e *literalExpr) value(ctx *Context, targetSym *Symbol) (cty.Value, source.Diags) {
	return e.val, nil
}

func (e *literalExpr) GoString() string {
	return fmt.Sprintf("eval.LiteralExpr(%#v, %#v)", e.val, e.rng.sourceRange())
}

type binaryOpExpr struct {
	lhs Expr
	rhs Expr
	op  operator
	rng
}

func makeBinaryOpExpr(lhs, rhs Expr, op operator, rng source.Range) Expr {
	return &binaryOpExpr{
		lhs: lhs,
		rhs: rhs,
		op:  op,
		rng: srcRange(rng),
	}
}

func AddExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opAdd, rng)
}

func SubtractExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opSubtract, rng)
}

func MultiplyExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opMultiply, rng)
}

func DivideExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opDivide, rng)
}

func ModuloExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opModulo, rng)
}

func ExponentExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opExponent, rng)
}

func ConcatExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opConcat, rng)
}

func EqualExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opEqual, rng)
}

func NotEqualExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opNotEqual, rng)
}

func LessThanExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opLessThan, rng)
}

func LessThanOrEqualExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opLessThanOrEqual, rng)
}

func GreaterThanExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opGreaterThan, rng)
}

func GreaterThanOrEqualExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opGreaterThanOrEqual, rng)
}

func AndExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opAnd, rng)
}

func OrExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opOr, rng)
}

func (e *binaryOpExpr) value(ctx *Context, targetSym *Symbol) (cty.Value, source.Diags) {
	return e.op.evalBinary(ctx, e.lhs, e.rhs, e.sourceRange())
}

func (e *binaryOpExpr) eachChild(cb walkCb) {
	cb(e.lhs)
	cb(e.rhs)
}

func (e *binaryOpExpr) GoString() string {
	name := e.op.String()[2:]
	return fmt.Sprintf("eval.%sExpr(%#v, %#v, %#v)", name, e.lhs, e.rhs, e.sourceRange())
}

type unaryOpExpr struct {
	val Expr
	op  operator
	rng
}

func makeUnaryOpExpr(val Expr, op operator, rng source.Range) Expr {
	return &unaryOpExpr{
		val: val,
		op:  op,
		rng: srcRange(rng),
	}
}

func NegateExpr(val Expr, rng source.Range) Expr {
	return makeUnaryOpExpr(val, opNegate, rng)
}

func NotExpr(val Expr, rng source.Range) Expr {
	return makeUnaryOpExpr(val, opNot, rng)
}

func (e *unaryOpExpr) value(ctx *Context, targetSym *Symbol) (cty.Value, source.Diags) {
	return e.op.evalUnary(ctx, e.val, e.sourceRange())
}

func (e *unaryOpExpr) eachChild(cb walkCb) {
	cb(e.val)
}

type callExpr struct {
	callee    Expr
	posArgs   []Expr
	namedArgs map[string]Expr
	rng
}

func CallExpr(callee Expr, posArgs []Expr, namedArgs map[string]Expr, rng source.Range) Expr {
	return &callExpr{
		callee:    callee,
		posArgs:   posArgs,
		namedArgs: namedArgs,
		rng:       srcRange(rng),
	}
}

func (e *callExpr) value(ctx *Context, targetSym *Symbol) (cty.Value, source.Diags) {
	panic("callExpr.value not yet implemented")
}

func (e *callExpr) eachChild(cb walkCb) {
	cb(e.callee)
	for _, expr := range e.posArgs {
		cb(expr)
	}
	for _, expr := range e.namedArgs {
		cb(expr)
	}
}
<commit_msg>eval: stub of AttrExpr (not yet evaluable)<commit_after>package eval

import (
	"fmt"

	"github.com/cirbo-lang/cirbo/cty"
	"github.com/cirbo-lang/cirbo/source"
)

type Expr interface {
	// value evaluates the expression in the given context.
	//
	// If the result is being used directly as the definition of a symbol
	// then the that symbol is provided in "targetSym"; otherwise,
	// targetSym is nil. Some expression types are valid only when being
	// assigned directly to a symbol.
	value(ctx *Context, targetSym *Symbol) (cty.Value, source.Diags)

	// eachChild should pass each of its child expressions (e.g. operands)
	// to the given callback in some reasonable order.
	eachChild(cb walkCb)

	// sourceRange returns the source code range that represents the receiving
	// expression.
	sourceRange() source.Range
}

type symbolExpr struct {
	sym *Symbol
	rng
	leafExpr
}

func SymbolExpr(sym *Symbol, rng source.Range) Expr {
	return &symbolExpr{
		sym: sym,
		rng: srcRange(rng),
	}
}

func (e *symbolExpr) value(ctx *Context, targetSym *Symbol) (cty.Value, source.Diags) {
	val := ctx.Value(e.sym)
	if val == cty.NilValue {
		// This is actually an implementation error in Cirbo rather than a
		// user error, but we'll return it as a diagnostic anyway since that's
		// more graceful.
		return cty.PlaceholderVal, source.Diags{
			{
				Level:   source.Error,
				Summary: "Symbol not yet defined",
				Detail:  fmt.Sprintf("The symbol %q has not yet been defined. This is a bug in Cirbo that should be reported!", e.sym.name),
				Ranges:  e.sourceRange().List(),
			},
		}
	}
	return val, nil
}

func (e *symbolExpr) GoString() string {
	return fmt.Sprintf("eval.SymbolExpr(%#v, %#v)", e.sym, e.rng.sourceRange())
}

type literalExpr struct {
	val cty.Value
	rng
	leafExpr
}

func LiteralExpr(val cty.Value, rng source.Range) Expr {
	return &literalExpr{
		val: val,
		rng: srcRange(rng),
	}
}

func (e *literalExpr) value(ctx *Context, targetSym *Symbol) (cty.Value, source.Diags) {
	return e.val, nil
}

func (e *literalExpr) GoString() string {
	return fmt.Sprintf("eval.LiteralExpr(%#v, %#v)", e.val, e.rng.sourceRange())
}

type binaryOpExpr struct {
	lhs Expr
	rhs Expr
	op  operator
	rng
}

func makeBinaryOpExpr(lhs, rhs Expr, op operator, rng source.Range) Expr {
	return &binaryOpExpr{
		lhs: lhs,
		rhs: rhs,
		op:  op,
		rng: srcRange(rng),
	}
}

func AddExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opAdd, rng)
}

func SubtractExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opSubtract, rng)
}

func MultiplyExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opMultiply, rng)
}

func DivideExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opDivide, rng)
}

func ModuloExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opModulo, rng)
}

func ExponentExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opExponent, rng)
}

func ConcatExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opConcat, rng)
}

func EqualExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opEqual, rng)
}

func NotEqualExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opNotEqual, rng)
}

func LessThanExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opLessThan, rng)
}

func LessThanOrEqualExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opLessThanOrEqual, rng)
}

func GreaterThanExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opGreaterThan, rng)
}

func GreaterThanOrEqualExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opGreaterThanOrEqual, rng)
}

func AndExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opAnd, rng)
}

func OrExpr(lhs, rhs Expr, rng source.Range) Expr {
	return makeBinaryOpExpr(lhs, rhs, opOr, rng)
}

func (e *binaryOpExpr) value(ctx *Context, targetSym *Symbol) (cty.Value, source.Diags) {
	return e.op.evalBinary(ctx, e.lhs, e.rhs, e.sourceRange())
}

func (e *binaryOpExpr) eachChild(cb walkCb) {
	cb(e.lhs)
	cb(e.rhs)
}

func (e *binaryOpExpr) GoString() string {
	name := e.op.String()[2:]
	return fmt.Sprintf("eval.%sExpr(%#v, %#v, %#v)", name, e.lhs, e.rhs, e.sourceRange())
}

type unaryOpExpr struct {
	val Expr
	op  operator
	rng
}

func makeUnaryOpExpr(val Expr, op operator, rng source.Range) Expr {
	return &unaryOpExpr{
		val: val,
		op:  op,
		rng: srcRange(rng),
	}
}

func NegateExpr(val Expr, rng source.Range) Expr {
	return makeUnaryOpExpr(val, opNegate, rng)
}

func NotExpr(val Expr, rng source.Range) Expr {
	return makeUnaryOpExpr(val, opNot, rng)
}

func (e *unaryOpExpr) value(ctx *Context, targetSym *Symbol) (cty.Value, source.Diags) {
	return e.op.evalUnary(ctx, e.val, e.sourceRange())
}

func (e *unaryOpExpr) eachChild(cb walkCb) {
	cb(e.val)
}

type callExpr struct {
	callee    Expr
	posArgs   []Expr
	namedArgs map[string]Expr
	rng
}

func CallExpr(callee Expr, posArgs []Expr, namedArgs map[string]Expr, rng source.Range) Expr {
	return &callExpr{
		callee:    callee,
		posArgs:   posArgs,
		namedArgs: namedArgs,
		rng:       srcRange(rng),
	}
}

func (e *callExpr) value(ctx *Context, targetSym *Symbol) (cty.Value, source.Diags) {
	panic("callExpr.value not yet implemented")
}

func (e *callExpr) eachChild(cb walkCb) {
	cb(e.callee)
	for _, expr := range e.posArgs {
		cb(expr)
	}
	for _, expr := range e.namedArgs {
		cb(expr)
	}
}

type attrExpr struct {
	obj  Expr
	name string
	rng
}

func AttrExpr(obj Expr, name string, rng source.Range) Expr {
	return &attrExpr{
		obj:  obj,
		name: name,
		rng:  srcRange(rng),
	}
}

func (e *attrExpr) value(ctx *Context, targetSym *Symbol) (cty.Value, source.Diags) {
	panic("attrExpr.value not yet implemented")
}

func (e *attrExpr) eachChild(cb walkCb) {
	cb(e.obj)
}
<|endoftext|>


<commit_before>package eval

import (
	"errors"
	"fmt"
	"reflect"
	"strings"
	"unicode"

	"github.com/elves/elvish/eval/vals"
	"github.com/elves/elvish/glob"
	"github.com/elves/elvish/parse"
)

// GlobPattern is en ephemeral Value generated when evaluating tilde and
// wildcards.
type GlobPattern struct {
	glob.Pattern
	Flags GlobFlag
	Buts  []string
}

type GlobFlag uint

const (
	NoMatchOK GlobFlag = 1 << iota
)

func (f GlobFlag) Has(g GlobFlag) bool {
	return (f & g) == g
}

var (
	_ interface{}     = GlobPattern{}
	_ vals.ErrIndexer = GlobPattern{}
)

var (
	ErrMustFollowWildcard   = errors.New("must follow wildcard")
	ErrModifierMustBeString = errors.New("modifier must be string")
	ErrWildcardNoMatch      = errors.New("wildcard has no match")
)

var runeMatchers = map[string]func(rune) bool{
	"control": unicode.IsControl,
	"digit":   unicode.IsDigit,
	"graphic": unicode.IsGraphic,
	"letter":  unicode.IsLetter,
	"lower":   unicode.IsDigit,
	"mark":    unicode.IsMark,
	"number":  unicode.IsNumber,
	"print":   unicode.IsPrint,
	"punct":   unicode.IsPunct,
	"space":   unicode.IsSpace,
	"symbol":  unicode.IsSymbol,
	"title":   unicode.IsTitle,
	"upper":   unicode.IsUpper,
}

func (GlobPattern) Kind() string {
	return "glob-pattern"
}

func (gp GlobPattern) Equal(a interface{}) bool {
	return reflect.DeepEqual(gp, a)
}

func (gp GlobPattern) Hash() uint32 {
	// GlobPattern is not a first-class value.
	return 0
}

func (gp GlobPattern) Repr(int) string {
	return fmt.Sprintf("<GlobPattern%v>", gp)
}

func (gp GlobPattern) Index(k interface{}) (interface{}, error) {
	modifierv, ok := k.(string)
	if !ok {
		return nil, ErrModifierMustBeString
	}
	modifier := modifierv
	switch {
	case modifier == "nomatch-ok":
		gp.Flags |= NoMatchOK
	case strings.HasPrefix(modifier, "but:"):
		gp.Buts = append(gp.Buts, modifier[len("but:"):])
	case modifier == "match-hidden":
		lastSeg := gp.mustGetLastWildSeg()
		gp.Segments[len(gp.Segments)-1] = glob.Wild{
			lastSeg.Type, true, lastSeg.Matchers,
		}
	default:
		if matcher, ok := runeMatchers[modifier]; ok {
			gp.addMatcher(matcher)
		} else if strings.HasPrefix(modifier, "set:") {
			set := modifier[len("set:"):]
			gp.addMatcher(func(r rune) bool {
				return strings.ContainsRune(set, r)
			})
		} else if strings.HasPrefix(modifier, "range:") {
			rangeExpr := modifier[len("range:"):]
			badRangeExpr := fmt.Errorf("bad range modifier: %s", parse.Quote(rangeExpr))
			runes := []rune(rangeExpr)
			if len(runes) != 3 {
				return nil, badRangeExpr
			}
			from, sep, to := runes[0], runes[1], runes[2]
			switch sep {
			case '-':
				gp.addMatcher(func(r rune) bool {
					return from <= r && r <= to
				})
			case '~':
				gp.addMatcher(func(r rune) bool {
					return from <= r && r < to
				})
			default:
				return nil, badRangeExpr
			}
		} else {
			return nil, fmt.Errorf("unknown modifier %s", vals.Repr(modifierv, vals.NoPretty))
		}
	}
	return gp, nil
}

func (gp GlobPattern) Concat(v interface{}) (interface{}, error) {
	switch rhs := v.(type) {
	case string:
		gp.append(stringToSegments(rhs)...)
		return gp, nil
	case GlobPattern:
		// We know rhs contains exactly one segment.
		gp.append(rhs.Segments[0])
		gp.Flags |= rhs.Flags
		gp.Buts = append(gp.Buts, rhs.Buts...)
		return gp, nil
	}

	return nil, vals.ErrConcatNotImplemented
}

func (gp GlobPattern) RConcat(v interface{}) (interface{}, error) {
	switch lhs := v.(type) {
	case string:
		segs := stringToSegments(lhs)
		// We know gp contains exactly one segment.
		segs = append(segs, gp.Segments[0])
		return GlobPattern{glob.Pattern{Segments: segs}, gp.Flags, gp.Buts}, nil
	}

	return nil, vals.ErrConcatNotImplemented
}

func (gp *GlobPattern) mustGetLastWildSeg() glob.Wild {
	if len(gp.Segments) == 0 {
		throw(ErrBadGlobPattern)
	}
	if !glob.IsWild(gp.Segments[len(gp.Segments)-1]) {
		throw(ErrMustFollowWildcard)
	}
	return gp.Segments[len(gp.Segments)-1].(glob.Wild)
}

func (gp *GlobPattern) addMatcher(matcher func(rune) bool) {
	lastSeg := gp.mustGetLastWildSeg()
	gp.Segments[len(gp.Segments)-1] = glob.Wild{
		lastSeg.Type, lastSeg.MatchHidden,
		append(lastSeg.Matchers, matcher),
	}
}

func (gp *GlobPattern) append(segs ...glob.Segment) {
	gp.Segments = append(gp.Segments, segs...)
}

func wildcardToSegment(s string) (glob.Segment, error) {
	switch s {
	case "*":
		return glob.Wild{glob.Star, false, nil}, nil
	case "**":
		return glob.Wild{glob.StarStar, false, nil}, nil
	case "?":
		return glob.Wild{glob.Question, false, nil}, nil
	default:
		return nil, fmt.Errorf("bad wildcard: %q", s)
	}
}

func stringToSegments(s string) []glob.Segment {
	segs := []glob.Segment{}
	for i := 0; i < len(s); {
		j := i
		for ; j < len(s) && s[j] != '/'; j++ {
		}
		if j > i {
			segs = append(segs, glob.Literal{s[i:j]})
		}
		if j < len(s) {
			for ; j < len(s) && s[j] == '/'; j++ {
			}
			segs = append(segs, glob.Slash{})
			i = j
		} else {
			break
		}
	}
	return segs
}

func doGlob(gp GlobPattern, abort <-chan struct{}) ([]interface{}, error) {
	but := make(map[string]struct{})
	for _, s := range gp.Buts {
		but[s] = struct{}{}
	}

	vs := make([]interface{}, 0)
	if !gp.Glob(func(name string) bool {
		select {
		case <-abort:
			logger.Println("glob aborted")
			return false
		default:
		}
		if _, b := but[name]; !b {
			vs = append(vs, name)
		}
		return true
	}) {
		return nil, ErrInterrupted
	}
	if len(vs) == 0 && !gp.Flags.Has(NoMatchOK) {
		return nil, ErrWildcardNoMatch
	}
	return vs, nil
}
<commit_msg>eval: Remove use of throws from globbing code.<commit_after>package eval

import (
	"errors"
	"fmt"
	"reflect"
	"strings"
	"unicode"

	"github.com/elves/elvish/eval/vals"
	"github.com/elves/elvish/glob"
	"github.com/elves/elvish/parse"
)

// GlobPattern is en ephemeral Value generated when evaluating tilde and
// wildcards.
type GlobPattern struct {
	glob.Pattern
	Flags GlobFlag
	Buts  []string
}

type GlobFlag uint

const (
	NoMatchOK GlobFlag = 1 << iota
)

func (f GlobFlag) Has(g GlobFlag) bool {
	return (f & g) == g
}

var (
	_ interface{}     = GlobPattern{}
	_ vals.ErrIndexer = GlobPattern{}
)

var (
	ErrMustFollowWildcard   = errors.New("must follow wildcard")
	ErrModifierMustBeString = errors.New("modifier must be string")
	ErrWildcardNoMatch      = errors.New("wildcard has no match")
)

var runeMatchers = map[string]func(rune) bool{
	"control": unicode.IsControl,
	"digit":   unicode.IsDigit,
	"graphic": unicode.IsGraphic,
	"letter":  unicode.IsLetter,
	"lower":   unicode.IsDigit,
	"mark":    unicode.IsMark,
	"number":  unicode.IsNumber,
	"print":   unicode.IsPrint,
	"punct":   unicode.IsPunct,
	"space":   unicode.IsSpace,
	"symbol":  unicode.IsSymbol,
	"title":   unicode.IsTitle,
	"upper":   unicode.IsUpper,
}

func (GlobPattern) Kind() string {
	return "glob-pattern"
}

func (gp GlobPattern) Equal(a interface{}) bool {
	return reflect.DeepEqual(gp, a)
}

func (gp GlobPattern) Hash() uint32 {
	// GlobPattern is not a first-class value.
	return 0
}

func (gp GlobPattern) Repr(int) string {
	return fmt.Sprintf("<GlobPattern%v>", gp)
}

func (gp GlobPattern) Index(k interface{}) (interface{}, error) {
	modifierv, ok := k.(string)
	if !ok {
		return nil, ErrModifierMustBeString
	}
	modifier := modifierv
	switch {
	case modifier == "nomatch-ok":
		gp.Flags |= NoMatchOK
	case strings.HasPrefix(modifier, "but:"):
		gp.Buts = append(gp.Buts, modifier[len("but:"):])
	case modifier == "match-hidden":
		lastSeg, err := gp.lastWildSeg()
		if err != nil {
			return nil, err
		}
		gp.Segments[len(gp.Segments)-1] = glob.Wild{
			lastSeg.Type, true, lastSeg.Matchers,
		}
	default:
		var matcher func(rune) bool
		if m, ok := runeMatchers[modifier]; ok {
			matcher = m
		} else if strings.HasPrefix(modifier, "set:") {
			set := modifier[len("set:"):]
			matcher = func(r rune) bool {
				return strings.ContainsRune(set, r)
			}
		} else if strings.HasPrefix(modifier, "range:") {
			rangeExpr := modifier[len("range:"):]
			badRangeExpr := fmt.Errorf("bad range modifier: %s", parse.Quote(rangeExpr))
			runes := []rune(rangeExpr)
			if len(runes) != 3 {
				return nil, badRangeExpr
			}
			from, sep, to := runes[0], runes[1], runes[2]
			switch sep {
			case '-':
				matcher = func(r rune) bool {
					return from <= r && r <= to
				}
			case '~':
				matcher = func(r rune) bool {
					return from <= r && r < to
				}
			default:
				return nil, badRangeExpr
			}
		} else {
			return nil, fmt.Errorf("unknown modifier %s", vals.Repr(modifierv, vals.NoPretty))
		}
		err := gp.addMatcher(matcher)
		return gp, err
	}
	return gp, nil
}

func (gp GlobPattern) Concat(v interface{}) (interface{}, error) {
	switch rhs := v.(type) {
	case string:
		gp.append(stringToSegments(rhs)...)
		return gp, nil
	case GlobPattern:
		// We know rhs contains exactly one segment.
		gp.append(rhs.Segments[0])
		gp.Flags |= rhs.Flags
		gp.Buts = append(gp.Buts, rhs.Buts...)
		return gp, nil
	}

	return nil, vals.ErrConcatNotImplemented
}

func (gp GlobPattern) RConcat(v interface{}) (interface{}, error) {
	switch lhs := v.(type) {
	case string:
		segs := stringToSegments(lhs)
		// We know gp contains exactly one segment.
		segs = append(segs, gp.Segments[0])
		return GlobPattern{glob.Pattern{Segments: segs}, gp.Flags, gp.Buts}, nil
	}

	return nil, vals.ErrConcatNotImplemented
}

func (gp *GlobPattern) lastWildSeg() (glob.Wild, error) {
	if len(gp.Segments) == 0 {
		return glob.Wild{}, ErrBadGlobPattern
	}
	if !glob.IsWild(gp.Segments[len(gp.Segments)-1]) {
		return glob.Wild{}, ErrMustFollowWildcard
	}
	return gp.Segments[len(gp.Segments)-1].(glob.Wild), nil
}

func (gp *GlobPattern) addMatcher(matcher func(rune) bool) error {
	lastSeg, err := gp.lastWildSeg()
	if err != nil {
		return err
	}
	gp.Segments[len(gp.Segments)-1] = glob.Wild{
		lastSeg.Type, lastSeg.MatchHidden,
		append(lastSeg.Matchers, matcher),
	}
	return nil
}

func (gp *GlobPattern) append(segs ...glob.Segment) {
	gp.Segments = append(gp.Segments, segs...)
}

func wildcardToSegment(s string) (glob.Segment, error) {
	switch s {
	case "*":
		return glob.Wild{glob.Star, false, nil}, nil
	case "**":
		return glob.Wild{glob.StarStar, false, nil}, nil
	case "?":
		return glob.Wild{glob.Question, false, nil}, nil
	default:
		return nil, fmt.Errorf("bad wildcard: %q", s)
	}
}

func stringToSegments(s string) []glob.Segment {
	segs := []glob.Segment{}
	for i := 0; i < len(s); {
		j := i
		for ; j < len(s) && s[j] != '/'; j++ {
		}
		if j > i {
			segs = append(segs, glob.Literal{s[i:j]})
		}
		if j < len(s) {
			for ; j < len(s) && s[j] == '/'; j++ {
			}
			segs = append(segs, glob.Slash{})
			i = j
		} else {
			break
		}
	}
	return segs
}

func doGlob(gp GlobPattern, abort <-chan struct{}) ([]interface{}, error) {
	but := make(map[string]struct{})
	for _, s := range gp.Buts {
		but[s] = struct{}{}
	}

	vs := make([]interface{}, 0)
	if !gp.Glob(func(name string) bool {
		select {
		case <-abort:
			logger.Println("glob aborted")
			return false
		default:
		}
		if _, b := but[name]; !b {
			vs = append(vs, name)
		}
		return true
	}) {
		return nil, ErrInterrupted
	}
	if len(vs) == 0 && !gp.Flags.Has(NoMatchOK) {
		return nil, ErrWildcardNoMatch
	}
	return vs, nil
}
<|endoftext|>


<commit_before>package main

import "testing"

func TestInsufficientMaterial(t *testing.T) {
	for _, fen := range []string{
		"K6k/8/8/8/8/8/8/8 w - - 0 1",
		"KN5k/8/8/8/8/8/8/8 w - - 0 1",
		"KB5k/8/8/8/8/8/8/8 w - - 0 1",
		"KB1b3k/8/8/8/8/8/8/8 w - - 0 1",
		"K6k/8/8/8/8/8/8/1B1B1B1B w - - 0 1",
		"K1b1b1bk/8/8/8/8/8/8/1B1B1B1B w - - 0 1",
	} {
		pos, err := ParseFEN(fen)
		if err != nil {
			panic(err)
		}
		if got := Eval(pos); got != 0 {
			t.Errorf("Insufficient Material: Eval(%v)==%v, want 0", fen, got)
		}
	}
}
<commit_msg>benchmark Eval<commit_after>package main

import "testing"

func TestInsufficientMaterial(t *testing.T) {
	for _, fen := range []string{
		"K6k/8/8/8/8/8/8/8 w - - 0 1",
		"KN5k/8/8/8/8/8/8/8 w - - 0 1",
		"KB5k/8/8/8/8/8/8/8 w - - 0 1",
		"KB1b3k/8/8/8/8/8/8/8 w - - 0 1",
		"K6k/8/8/8/8/8/8/1B1B1B1B w - - 0 1",
		"K1b1b1bk/8/8/8/8/8/8/1B1B1B1B w - - 0 1",
	} {
		pos, err := ParseFEN(fen)
		if err != nil {
			panic(err)
		}
		if got := Eval(pos); got != 0 {
			t.Errorf("Insufficient Material: Eval(%v)==%v, want 0", fen, got)
		}
	}
}

func BenchmarkEval(b *testing.B) {
	pos, err := ParseFEN("r1b2rkB/1pp1ppbp/2n3p1/8/PpP3nP/8/3KPP2/1N3BNR w - - 0 13")
	if err != nil {
		b.Fatal(err)
	}
	for i := 0; i < b.N; i++ {
		Eval(pos)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/APTrust/bagman"
	"github.com/APTrust/bagman/processutil"
	"github.com/APTrust/bagman/ingesthelper"
	"github.com/bitly/go-nsq"
	"time"
)

type Channels struct {
	FetchChannel   chan *ingesthelper.IngestHelper
	UnpackChannel  chan *ingesthelper.IngestHelper
	StorageChannel chan *ingesthelper.IngestHelper
	CleanUpChannel chan *ingesthelper.IngestHelper
	ResultsChannel chan *ingesthelper.IngestHelper
}

// Global vars.
var procUtil *processutil.ProcessUtil
var channels *Channels

// bag_processor receives messages from nsqd describing
// items in the S3 receiving buckets. Each item/message
// follows this flow:
//
// 1. Fetch channel: fetches the file from S3.
// 2. Unpack channel: untars the bag files, parses and validates
//    the bag, reads tags, generates checksums and generic file
//    UUIDs.
// 3. Storage channel: copies files to S3 permanent storage.
// 4. Results channel: tells the queue whether processing
//    succeeded, and if not, whether the item should be requeued.
//    Also logs results to json and message logs.
// 5. Cleanup channel: cleans up the files after processing
//    completes.
//
// If a failure occurs anywhere in the first three steps,
// processing goes directly to the Results Channel, which
// records the error and the disposition (retry/give up).
//
// As long as the message from nsq contains valid JSON,
// steps 5 and 6 ALWAYS run.
//
// The bag processor has so many responsibilities because
// downloading, untarring and running checksums on
// multi-gigabyte files takes a lot of time. We want to
// avoid having separate processes repeatedly download and
// untar the files, so bag_processor performs all operations
// that require local access to the raw contents of the bags.
func main() {
	requestedConfig := flag.String("config", "", "configuration to run")
	flag.Parse()
	procUtil = processutil.NewProcessUtil(requestedConfig)

	procUtil.MessageLog.Info("Bag Processor started")
	err := procUtil.Config.EnsureFluctusConfig()
	if err != nil {
		procUtil.MessageLog.Fatalf("Required Fluctus config vars are missing: %v", err)
	}

	initChannels()
	initGoRoutines()

	nsqConfig := nsq.NewConfig()
	nsqConfig.Set("max_in_flight", 20)
	nsqConfig.Set("heartbeat_interval", "10s")
	nsqConfig.Set("max_attempts", uint16(procUtil.Config.MaxBagAttempts))
	nsqConfig.Set("read_timeout", "60s")
	nsqConfig.Set("write_timeout", "10s")
	nsqConfig.Set("msg_timeout", "60m")
	consumer, err := nsq.NewConsumer(procUtil.Config.BagProcessorTopic, procUtil.Config.BagProcessorChannel, nsqConfig)
	if err != nil {
		procUtil.MessageLog.Fatalf(err.Error())
	}

	handler := &BagProcessor{}
	consumer.SetHandler(handler)
	consumer.ConnectToNSQLookupd(procUtil.Config.NsqLookupd)

	// This reader blocks until we get an interrupt, so our program does not exit.
	<-consumer.StopChan
}

// Set up the channels. It's essential that that the fetchChannel
// be limited to a relatively low number. If we are downloading
// 1GB tar files, we need space to store the tar file AND the
// untarred version. That's about 2 x 1GB. We do not want to pull
// down 1000 files at once, or we'll run out of disk space!
// If config sets fetchers to 10, we can pull down 10 files at a
// time. The fetch queue could hold 10 * 4 = 40 items, so we'd
// have max 40 tar files + untarred directories on disk at once.
// The number of workers should be close to the number of CPU
// cores.
func initChannels() {
	fetcherBufferSize := procUtil.Config.Fetchers * 4
	workerBufferSize := procUtil.Config.Workers * 10

	channels = &Channels{}
	channels.FetchChannel = make(chan *ingesthelper.IngestHelper, fetcherBufferSize)
	channels.UnpackChannel = make(chan *ingesthelper.IngestHelper, workerBufferSize)
	channels.StorageChannel = make(chan *ingesthelper.IngestHelper, workerBufferSize)
	channels.CleanUpChannel = make(chan *ingesthelper.IngestHelper, workerBufferSize)
	channels.ResultsChannel = make(chan *ingesthelper.IngestHelper, workerBufferSize)
}

// Set up our go routines. We do NOT want one go routine per
// S3 file. If we do that, the system will run out of file handles,
// as we'll have tens of thousands of open connections to S3
// trying to write data into tens of thousands of local files.
func initGoRoutines() {
	for i := 0; i < procUtil.Config.Fetchers; i++ {
		go doFetch()
	}

	for i := 0; i < procUtil.Config.Workers; i++ {
		go doUnpack()
		go saveToStorage()
		go logResult()
		go doCleanUp()
	}
}

type BagProcessor struct {
}

// MessageHandler handles messages from the queue, putting each
// item into the pipleline.
func (*BagProcessor) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	var s3File bagman.S3File
	err := json.Unmarshal(message.Body, &s3File)
	if err != nil {
		procUtil.MessageLog.Error("Could not unmarshal JSON data from nsq:",
			string(message.Body))
		message.Finish()
		return nil
	}

	// If we're not reprocessing on purpose, and this item has already
	// been successfully processed, skip it. There are certain timing
	// conditions that can cause the bucket reader to add items to the
	// queue twice. If we get rid of NSQ, we can get rid of this check.
	if procUtil.Config.SkipAlreadyProcessed == true && ingesthelper.BagNeedsProcessing(&s3File, procUtil) == false {
		procUtil.MessageLog.Info("Marking %s as complete, without processing because "+
			"this bag was successfully processed previously and Config.SkipAlreadyProcessed "+
			"= true", s3File.Key.Key)
		message.Finish()
		return nil
	}

	// Don't start working on a message that we're already working on.
	// Note that the key we include in the syncMap includes multipart
	// bag endings, so we can be working on ncsu.edu/obj.b1of2.tar and
	// ncsu.edu/obj.b2of2.tar at the same time. This is what we want.
	mapErr := procUtil.RegisterItem(s3File.BagName(), message.ID)
	if mapErr != nil {
		procUtil.MessageLog.Info("Marking %s as complete because the file is already "+
			"being processed under another message id.\n", s3File.Key.Key)
		message.Finish()
		return nil
	}

	// Create the result struct and pass it down the pipeline
	helper := ingesthelper.NewIngestHelper(procUtil, message, &s3File)
	channels.FetchChannel <- helper
	procUtil.MessageLog.Debug("Put %s into fetch queue", s3File.Key.Key)
	return nil
}

// -- Step 1 of 5 --
// This runs as a go routine to fetch files from S3.
func doFetch() {
	for helper := range channels.FetchChannel {
		result := helper.Result
		s3Key := result.S3File.Key
		// Disk needs filesize * 2 disk space to accomodate tar file & untarred files
		err := procUtil.Volume.Reserve(uint64(s3Key.Size * 2))
		if err != nil {
			// Not enough room on disk
			procUtil.MessageLog.Warning("Requeueing %s - not enough disk space", s3Key.Key)
			result.ErrorMessage = err.Error()
			result.Retry = true
			channels.ResultsChannel <- helper
		} else {
			procUtil.MessageLog.Info("Fetching %s", s3Key.Key)
			helper.FetchTarFile()
			if result.ErrorMessage != "" {
				// Fetch from S3 failed. Requeue.
				channels.ResultsChannel <- helper
			} else {
				// Got S3 file. Untar it.
				// And touch the message, so nsqd knows we're making progress.
				result.NsqMessage.Touch()
				channels.UnpackChannel <- helper
			}
		}
	}
}

// -- Step 2 of 5 --
// This runs as a go routine to untar files downloaded from S3.
// We calculate checksums and create generic files during the unpack
// stage to avoid having to reprocess large streams of data several times.
func doUnpack() {
	for helper := range channels.UnpackChannel {
		result := helper.Result
		if result.ErrorMessage != "" {
			// Unpack failed. Go to end.
			procUtil.MessageLog.Warning("Nothing to unpack for %s",
				result.S3File.Key.Key)
			channels.ResultsChannel <- helper
		} else {
			// Unpacked! Now process the bag and touch message
			// so nsqd knows we're making progress.
			procUtil.MessageLog.Info("Unpacking %s", result.S3File.Key.Key)
			result.NsqMessage.Touch()
			helper.ProcessBagFile()
			if result.ErrorMessage == "" {
				// Move to permanent storage if bag processing succeeded
				channels.StorageChannel <- helper
			} else {
				channels.ResultsChannel <- helper
			}
		}
	}
}

// -- Step 3 of 5 --
// This runs as a go routine to save generic files to the permanent
// S3 storage bucket. Unfortunately, there is no concept of transaction
// here. Ideally, either all GenericFiles make to S3 or none do. In
// cases where we're updating an existing bag (i.e. user uploaded a new
// version of it), we may wind up in a state where half of the new files
// make it successfully to S3 and half do not. That would leave us with
// an inconsistent bag, containing half new files and half old files.
// In addition, the failure to copy all files to S3 would result in no
// metadata going to Fedora. So Fedora would not show that any of the
// generic files were overwritten, even though some were. We should alert
// an admin in these cases. The JSON log will have full information about
// the state of all of the files.
func saveToStorage() {
	for helper := range channels.StorageChannel {
		helper.Result.NsqMessage.Touch()
		err := helper.SaveGenericFiles()
		if err != nil {
			channels.ResultsChannel <- helper
			continue
		}
		// If there were no errors, put this into the metadata
		// queue, so we can record the events in Fluctus.
		if helper.Result.ErrorMessage == "" {
			SendToMetadataQueue(helper)
		}

		// Pass problem cases off to the trouble queue
		if helper.IncompleteCopyToS3() || helper.FailedAndNoMoreRetries() {
			SendToTroubleQueue(helper)
		}

		// Record the results.
		channels.ResultsChannel <- helper
	}
}

// -- Step 4 of 5 --
// TODO: This code is duplicated in metarecord.go
// This prints to the log the result of the program's attempt to fetch,
// untar, unbag and verify an individual S3 tar file.
// THIS STEP ALWAYS RUNS, EVEN IF PRIOR STEPS FAILED.
func logResult() {
	for helper := range channels.ResultsChannel {
		helper.LogResult()
		channels.CleanUpChannel <- helper
	}
}

// -- Step 5 of 5 --
// This runs as a go routine to remove the files we downloaded
// and untarred.
// THIS STEP ALWAYS RUNS, EVEN IF PRIOR STEPS FAILED.
func doCleanUp() {
	for helper := range channels.CleanUpChannel {
		result := helper.Result
		procUtil.MessageLog.Debug("Cleaning up %s", result.S3File.Key.Key)
		if (result.S3File.Key.Key != "" && result.FetchResult != nil &&
			result.FetchResult.LocalTarFile != "") {
			// Clean up any files we downloaded and unpacked
			errors := helper.DeleteLocalFiles()
			if errors != nil && len(errors) > 0 {
				procUtil.MessageLog.Warning("Errors cleaning up %s",
					result.FetchResult.LocalTarFile)
				for _, e := range errors {
					procUtil.MessageLog.Error(e.Error())
				}
			}
			// Let our volume tracker know we just freed up some disk space.
			// Free the same amount we reserved.
			procUtil.Volume.Release(uint64(result.S3File.Key.Size * 2))
		}

		// Build and send message back to NSQ, indicating whether
		// processing succeeded.
		if result.ErrorMessage != "" && result.Retry == true {
			procUtil.MessageLog.Info("Requeueing %s", result.S3File.Key.Key)
			result.NsqMessage.Requeue(5 * time.Minute)
		} else {
			result.NsqMessage.Finish()
		}

		// We're done processing this, so remove it from the map.
		// If it comes in again, we'll reprocess it again.
		procUtil.UnregisterItem(result.S3File.BagName())
	}
}

// Puts an item into the queue for Fluctus/Fedora metadata processing.
func SendToMetadataQueue(helper *ingesthelper.IngestHelper) {
	err := bagman.Enqueue(helper.ProcUtil.Config.NsqdHttpAddress,
		helper.ProcUtil.Config.MetadataTopic, helper.Result)
	if err != nil {
		errMsg := fmt.Sprintf("Error adding '%s' to metadata queue: %v ",
			helper.Result.S3File.Key.Key, err)
		helper.ProcUtil.MessageLog.Error(errMsg)
		helper.Result.ErrorMessage += errMsg
	} else {
		helper.ProcUtil.MessageLog.Debug("Sent '%s' to metadata queue",
			helper.Result.S3File.Key.Key)
	}
}

// Puts an item into the trouble queue.
func SendToTroubleQueue(helper *ingesthelper.IngestHelper) {
	err := bagman.Enqueue(helper.ProcUtil.Config.NsqdHttpAddress,
		helper.ProcUtil.Config.TroubleTopic, helper.Result)
	if err != nil {
		helper.ProcUtil.MessageLog.Error("Could not send '%s' to trouble queue: %v\n",
			helper.Result.S3File.Key.Key, err)
	} else {
		reason := "Processing failed and we reached the maximum number of retries."
		if helper.IncompleteCopyToS3() {
			reason = "Some files could not be copied to S3."
		}
		helper.Result.ErrorMessage += fmt.Sprintf("%s This item has been queued for administrative review.",
			reason)
		helper.ProcUtil.MessageLog.Warning("Sent '%s' to trouble queue: %s",
			helper.Result.S3File.Key.Key, reason)
	}
}
<commit_msg>Increase timeout. Get more touchy with messages.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/APTrust/bagman"
	"github.com/APTrust/bagman/processutil"
	"github.com/APTrust/bagman/ingesthelper"
	"github.com/bitly/go-nsq"
	"time"
)

type Channels struct {
	FetchChannel   chan *ingesthelper.IngestHelper
	UnpackChannel  chan *ingesthelper.IngestHelper
	StorageChannel chan *ingesthelper.IngestHelper
	CleanUpChannel chan *ingesthelper.IngestHelper
	ResultsChannel chan *ingesthelper.IngestHelper
}

// Global vars.
var procUtil *processutil.ProcessUtil
var channels *Channels

// bag_processor receives messages from nsqd describing
// items in the S3 receiving buckets. Each item/message
// follows this flow:
//
// 1. Fetch channel: fetches the file from S3.
// 2. Unpack channel: untars the bag files, parses and validates
//    the bag, reads tags, generates checksums and generic file
//    UUIDs.
// 3. Storage channel: copies files to S3 permanent storage.
// 4. Results channel: tells the queue whether processing
//    succeeded, and if not, whether the item should be requeued.
//    Also logs results to json and message logs.
// 5. Cleanup channel: cleans up the files after processing
//    completes.
//
// If a failure occurs anywhere in the first three steps,
// processing goes directly to the Results Channel, which
// records the error and the disposition (retry/give up).
//
// As long as the message from nsq contains valid JSON,
// steps 5 and 6 ALWAYS run.
//
// The bag processor has so many responsibilities because
// downloading, untarring and running checksums on
// multi-gigabyte files takes a lot of time. We want to
// avoid having separate processes repeatedly download and
// untar the files, so bag_processor performs all operations
// that require local access to the raw contents of the bags.
func main() {
	requestedConfig := flag.String("config", "", "configuration to run")
	flag.Parse()
	procUtil = processutil.NewProcessUtil(requestedConfig)

	procUtil.MessageLog.Info("Bag Processor started")
	err := procUtil.Config.EnsureFluctusConfig()
	if err != nil {
		procUtil.MessageLog.Fatalf("Required Fluctus config vars are missing: %v", err)
	}

	initChannels()
	initGoRoutines()

	nsqConfig := nsq.NewConfig()
	nsqConfig.Set("max_in_flight", 20)
	nsqConfig.Set("heartbeat_interval", "10s")
	nsqConfig.Set("max_attempts", uint16(procUtil.Config.MaxBagAttempts))
	nsqConfig.Set("read_timeout", "60s")
	nsqConfig.Set("write_timeout", "10s")
	nsqConfig.Set("msg_timeout", "180m")
	consumer, err := nsq.NewConsumer(procUtil.Config.BagProcessorTopic, procUtil.Config.BagProcessorChannel, nsqConfig)
	if err != nil {
		procUtil.MessageLog.Fatalf(err.Error())
	}

	handler := &BagProcessor{}
	consumer.SetHandler(handler)
	consumer.ConnectToNSQLookupd(procUtil.Config.NsqLookupd)

	// This reader blocks until we get an interrupt, so our program does not exit.
	<-consumer.StopChan
}

// Set up the channels. It's essential that that the fetchChannel
// be limited to a relatively low number. If we are downloading
// 1GB tar files, we need space to store the tar file AND the
// untarred version. That's about 2 x 1GB. We do not want to pull
// down 1000 files at once, or we'll run out of disk space!
// If config sets fetchers to 10, we can pull down 10 files at a
// time. The fetch queue could hold 10 * 4 = 40 items, so we'd
// have max 40 tar files + untarred directories on disk at once.
// The number of workers should be close to the number of CPU
// cores.
func initChannels() {
	fetcherBufferSize := procUtil.Config.Fetchers * 4
	workerBufferSize := procUtil.Config.Workers * 10

	channels = &Channels{}
	channels.FetchChannel = make(chan *ingesthelper.IngestHelper, fetcherBufferSize)
	channels.UnpackChannel = make(chan *ingesthelper.IngestHelper, workerBufferSize)
	channels.StorageChannel = make(chan *ingesthelper.IngestHelper, workerBufferSize)
	channels.CleanUpChannel = make(chan *ingesthelper.IngestHelper, workerBufferSize)
	channels.ResultsChannel = make(chan *ingesthelper.IngestHelper, workerBufferSize)
}

// Set up our go routines. We do NOT want one go routine per
// S3 file. If we do that, the system will run out of file handles,
// as we'll have tens of thousands of open connections to S3
// trying to write data into tens of thousands of local files.
func initGoRoutines() {
	for i := 0; i < procUtil.Config.Fetchers; i++ {
		go doFetch()
	}

	for i := 0; i < procUtil.Config.Workers; i++ {
		go doUnpack()
		go saveToStorage()
		go logResult()
		go doCleanUp()
	}
}

type BagProcessor struct {
}

// MessageHandler handles messages from the queue, putting each
// item into the pipleline.
func (*BagProcessor) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	var s3File bagman.S3File
	err := json.Unmarshal(message.Body, &s3File)
	if err != nil {
		procUtil.MessageLog.Error("Could not unmarshal JSON data from nsq:",
			string(message.Body))
		message.Finish()
		return nil
	}

	// If we're not reprocessing on purpose, and this item has already
	// been successfully processed, skip it. There are certain timing
	// conditions that can cause the bucket reader to add items to the
	// queue twice. If we get rid of NSQ, we can get rid of this check.
	if procUtil.Config.SkipAlreadyProcessed == true && ingesthelper.BagNeedsProcessing(&s3File, procUtil) == false {
		procUtil.MessageLog.Info("Marking %s as complete, without processing because "+
			"this bag was successfully processed previously and Config.SkipAlreadyProcessed "+
			"= true", s3File.Key.Key)
		message.Finish()
		return nil
	}

	// Don't start working on a message that we're already working on.
	// Note that the key we include in the syncMap includes multipart
	// bag endings, so we can be working on ncsu.edu/obj.b1of2.tar and
	// ncsu.edu/obj.b2of2.tar at the same time. This is what we want.
	mapErr := procUtil.RegisterItem(s3File.BagName(), message.ID)
	if mapErr != nil {
		procUtil.MessageLog.Info("Marking %s as complete because the file is already "+
			"being processed under another message id.\n", s3File.Key.Key)
		message.Finish()
		return nil
	}

	// Create the result struct and pass it down the pipeline
	helper := ingesthelper.NewIngestHelper(procUtil, message, &s3File)
	channels.FetchChannel <- helper
	procUtil.MessageLog.Debug("Put %s into fetch queue", s3File.Key.Key)
	return nil
}

// -- Step 1 of 5 --
// This runs as a go routine to fetch files from S3.
func doFetch() {
	for helper := range channels.FetchChannel {
		result := helper.Result
		result.NsqMessage.Touch()
		s3Key := result.S3File.Key
		// Disk needs filesize * 2 disk space to accomodate tar file & untarred files
		err := procUtil.Volume.Reserve(uint64(s3Key.Size * 2))
		if err != nil {
			// Not enough room on disk
			procUtil.MessageLog.Warning("Requeueing %s - not enough disk space", s3Key.Key)
			result.ErrorMessage = err.Error()
			result.Retry = true
			channels.ResultsChannel <- helper
		} else {
			procUtil.MessageLog.Info("Fetching %s", s3Key.Key)
			helper.FetchTarFile()
			if result.ErrorMessage != "" {
				// Fetch from S3 failed. Requeue.
				channels.ResultsChannel <- helper
			} else {
				// Got S3 file. Untar it.
				// And touch the message, so nsqd knows we're making progress.
				result.NsqMessage.Touch()
				channels.UnpackChannel <- helper
			}
		}
	}
}

// -- Step 2 of 5 --
// This runs as a go routine to untar files downloaded from S3.
// We calculate checksums and create generic files during the unpack
// stage to avoid having to reprocess large streams of data several times.
func doUnpack() {
	for helper := range channels.UnpackChannel {
		result := helper.Result
		if result.ErrorMessage != "" {
			// Unpack failed. Go to end.
			procUtil.MessageLog.Warning("Nothing to unpack for %s",
				result.S3File.Key.Key)
			channels.ResultsChannel <- helper
		} else {
			// Unpacked! Now process the bag and touch message
			// so nsqd knows we're making progress.
			procUtil.MessageLog.Info("Unpacking %s", result.S3File.Key.Key)
			// Touch when we start
			result.NsqMessage.Touch()
			// Processing can take 90+ minutes for very large files!
			helper.ProcessBagFile()
			// And touch again when we're done
			result.NsqMessage.Touch()
			if result.ErrorMessage == "" {
				// Move to permanent storage if bag processing succeeded
				channels.StorageChannel <- helper
			} else {
				channels.ResultsChannel <- helper
			}
		}
	}
}

// -- Step 3 of 5 --
// This runs as a go routine to save generic files to the permanent
// S3 storage bucket. Unfortunately, there is no concept of transaction
// here. Ideally, either all GenericFiles make to S3 or none do. In
// cases where we're updating an existing bag (i.e. user uploaded a new
// version of it), we may wind up in a state where half of the new files
// make it successfully to S3 and half do not. That would leave us with
// an inconsistent bag, containing half new files and half old files.
// In addition, the failure to copy all files to S3 would result in no
// metadata going to Fedora. So Fedora would not show that any of the
// generic files were overwritten, even though some were. We should alert
// an admin in these cases. The JSON log will have full information about
// the state of all of the files.
func saveToStorage() {
	for helper := range channels.StorageChannel {
		// Touch before and after sending generic files,
		// since that process can take a long time for large bags.
		helper.Result.NsqMessage.Touch()
		err := helper.SaveGenericFiles()
		helper.Result.NsqMessage.Touch()
		if err != nil {
			channels.ResultsChannel <- helper
			continue
		}
		// If there were no errors, put this into the metadata
		// queue, so we can record the events in Fluctus.
		if helper.Result.ErrorMessage == "" {
			SendToMetadataQueue(helper)
		}

		// Pass problem cases off to the trouble queue
		if helper.IncompleteCopyToS3() || helper.FailedAndNoMoreRetries() {
			SendToTroubleQueue(helper)
		}

		// Record the results.
		channels.ResultsChannel <- helper
	}
}

// -- Step 4 of 5 --
// TODO: This code is duplicated in metarecord.go
// This prints to the log the result of the program's attempt to fetch,
// untar, unbag and verify an individual S3 tar file.
// THIS STEP ALWAYS RUNS, EVEN IF PRIOR STEPS FAILED.
func logResult() {
	for helper := range channels.ResultsChannel {
		helper.LogResult()
		channels.CleanUpChannel <- helper
	}
}

// -- Step 5 of 5 --
// This runs as a go routine to remove the files we downloaded
// and untarred.
// THIS STEP ALWAYS RUNS, EVEN IF PRIOR STEPS FAILED.
func doCleanUp() {
	for helper := range channels.CleanUpChannel {
		result := helper.Result
		result.Touch()
		procUtil.MessageLog.Debug("Cleaning up %s", result.S3File.Key.Key)
		if (result.S3File.Key.Key != "" && result.FetchResult != nil &&
			result.FetchResult.LocalTarFile != "") {
			// Clean up any files we downloaded and unpacked
			errors := helper.DeleteLocalFiles()
			if errors != nil && len(errors) > 0 {
				procUtil.MessageLog.Warning("Errors cleaning up %s",
					result.FetchResult.LocalTarFile)
				for _, e := range errors {
					procUtil.MessageLog.Error(e.Error())
				}
			}
			// Let our volume tracker know we just freed up some disk space.
			// Free the same amount we reserved.
			procUtil.Volume.Release(uint64(result.S3File.Key.Size * 2))
		}

		// Build and send message back to NSQ, indicating whether
		// processing succeeded.
		if result.ErrorMessage != "" && result.Retry == true {
			procUtil.MessageLog.Info("Requeueing %s", result.S3File.Key.Key)
			result.NsqMessage.Requeue(5 * time.Minute)
		} else {
			result.NsqMessage.Finish()
		}

		// We're done processing this, so remove it from the map.
		// If it comes in again, we'll reprocess it again.
		procUtil.UnregisterItem(result.S3File.BagName())
	}
}

// Puts an item into the queue for Fluctus/Fedora metadata processing.
func SendToMetadataQueue(helper *ingesthelper.IngestHelper) {
	err := bagman.Enqueue(helper.ProcUtil.Config.NsqdHttpAddress,
		helper.ProcUtil.Config.MetadataTopic, helper.Result)
	if err != nil {
		errMsg := fmt.Sprintf("Error adding '%s' to metadata queue: %v ",
			helper.Result.S3File.Key.Key, err)
		helper.ProcUtil.MessageLog.Error(errMsg)
		helper.Result.ErrorMessage += errMsg
	} else {
		helper.ProcUtil.MessageLog.Debug("Sent '%s' to metadata queue",
			helper.Result.S3File.Key.Key)
	}
}

// Puts an item into the trouble queue.
func SendToTroubleQueue(helper *ingesthelper.IngestHelper) {
	err := bagman.Enqueue(helper.ProcUtil.Config.NsqdHttpAddress,
		helper.ProcUtil.Config.TroubleTopic, helper.Result)
	if err != nil {
		helper.ProcUtil.MessageLog.Error("Could not send '%s' to trouble queue: %v\n",
			helper.Result.S3File.Key.Key, err)
	} else {
		reason := "Processing failed and we reached the maximum number of retries."
		if helper.IncompleteCopyToS3() {
			reason = "Some files could not be copied to S3."
		}
		helper.Result.ErrorMessage += fmt.Sprintf("%s This item has been queued for administrative review.",
			reason)
		helper.ProcUtil.MessageLog.Warning("Sent '%s' to trouble queue: %s",
			helper.Result.S3File.Key.Key, reason)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"net"
	"os"
	"time"
)

type EventLogger struct {
	file *os.File
	enc  *json.Encoder
}

func NewEventLogger(path string) (*EventLogger, error) {
	file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0666)
	return &EventLogger{file, json.NewEncoder(file)}, err
}

func (l *EventLogger) Close() error { return l.file.Close() }

func (l *EventLogger) Log(what string, event map[string]interface{}) {
	event["log_time"] = time.Now()
	event["what"] = what

	if err := l.enc.Encode(event); err != nil {
		panic(err)
	}
}

func (l *EventLogger) LogConn(op string, freq Frequency, conn net.Conn, err error) {
	e := map[string]interface{}{"success": err == nil}

	if err != nil {
		e["error"] = err.Error()
	}

	if freq > 0 {
		e["freq"] = freq
	}

	if conn != nil {
		e["remote_addr"] = conn.RemoteAddr().String()
		e["local_addr"] = conn.LocalAddr().String()
		e["network"] = conn.RemoteAddr().Network()
		e["operation"] = op
	}

	l.Log("connect", e)
}
<commit_msg>Add ok checks for ax25.Conn-methods and don't check for nil interface in event_log.<commit_after>package main

import (
	"encoding/json"
	"net"
	"os"
	"time"
)

type EventLogger struct {
	file *os.File
	enc  *json.Encoder
}

func NewEventLogger(path string) (*EventLogger, error) {
	file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0666)
	return &EventLogger{file, json.NewEncoder(file)}, err
}

func (l *EventLogger) Close() error { return l.file.Close() }

func (l *EventLogger) Log(what string, event map[string]interface{}) {
	event["log_time"] = time.Now()
	event["what"] = what

	if err := l.enc.Encode(event); err != nil {
		panic(err)
	}
}

func (l *EventLogger) LogConn(op string, freq Frequency, conn net.Conn, err error) {
	e := map[string]interface{}{"success": err == nil}

	if err != nil {
		e["error"] = err.Error()
	} else {
		if remote := conn.RemoteAddr(); remote != nil {
			e["remote_addr"] = remote.String()
			e["network"] = conn.RemoteAddr().Network()
		}
		if local := conn.LocalAddr(); local != nil {
			e["local_addr"] = local.String()
		}
	}

	if freq > 0 {
		e["freq"] = freq
	}

	e["operation"] = op

	l.Log("connect", e)
}
<|endoftext|>


<commit_before>// +build darwin,!nucular_shiny nucular_gio

package richtext

import (
	"unsafe"

	"github.com/aarzilli/nucular/font"

	ifont "golang.org/x/image/font"

	"golang.org/x/image/math/fixed"

	"gioui.org/font/opentype"
	"gioui.org/text"
	"gioui.org/unit"
)

type fontFace struct {
	fnt     *opentype.Font
	shaper  *text.Shaper
	size    int
	fsize   fixed.Int26_6
	metrics ifont.Metrics
}

func fontFace2fontFace(f *font.Face) *fontFace {
	return (*fontFace)(unsafe.Pointer(f))
}

func (face *fontFace) layout(str string, width int) *text.Layout {
	if width < 0 {
		width = 1e6
	}
	return face.shaper.Layout(face, text.Font{}, str, text.LayoutOptions{MaxWidth: width})
}

func (face *fontFace) Px(v unit.Value) int {
	return face.size
}

func (rtxt *RichText) calcAdvances(partial int) {
	if rtxt.adv != nil && partial == 0 {
		rtxt.adv = rtxt.adv[:0]
	}
	pos := int32(0)
	var siter styleIterator
	siter.Init(rtxt)
	for _, chunk := range rtxt.chunks[partial:] {
		// Note chunk is a copy of the element in the slice so we can modify it with impunity
		for chunk.len() > 0 {
			len := chunk.len()
			if siter.styleSel.E < pos+len {
				len = siter.styleSel.E - pos
			}

			if chunk.b != nil {
				panic("not implemented")
			}

			txt := fontFace2fontFace(&siter.styleSel.Face).layout(chunk.s[:len], 1e6)
			for _, line := range txt.Lines {
				rtxt.adv = append(rtxt.adv, line.Text.Advances...)
			}

			siter.AdvanceTo(pos + len)
			pos += len
			chunk.s = chunk.s[len:]
		}
	}
}
<commit_msg>richtext: update gio<commit_after>// +build darwin,!nucular_shiny nucular_gio

package richtext

import (
	"unsafe"

	"github.com/aarzilli/nucular/font"

	ifont "golang.org/x/image/font"

	"golang.org/x/image/math/fixed"

	"gioui.org/font/opentype"
	"gioui.org/text"
	"gioui.org/unit"
)

type fontFace struct {
	fnt     *opentype.Font
	shaper  *text.FontRegistry
	size    int
	fsize   fixed.Int26_6
	metrics ifont.Metrics
}

func fontFace2fontFace(f *font.Face) *fontFace {
	return (*fontFace)(unsafe.Pointer(f))
}

func (face *fontFace) layout(str string, width int) []text.Line {
	if width < 0 {
		width = 1e6
	}
	return face.shaper.LayoutString(text.Font{}, fixed.I(face.size), width, str)
}

func (face *fontFace) Px(v unit.Value) int {
	return face.size
}

func (rtxt *RichText) calcAdvances(partial int) {
	if rtxt.adv != nil && partial == 0 {
		rtxt.adv = rtxt.adv[:0]
	}
	pos := int32(0)
	var siter styleIterator
	siter.Init(rtxt)
	for _, chunk := range rtxt.chunks[partial:] {
		// Note chunk is a copy of the element in the slice so we can modify it with impunity
		for chunk.len() > 0 {
			len := chunk.len()
			if siter.styleSel.E < pos+len {
				len = siter.styleSel.E - pos
			}

			if chunk.b != nil {
				panic("not implemented")
			}

			txt := fontFace2fontFace(&siter.styleSel.Face).layout(chunk.s[:len], 1e6)
			for _, line := range txt {
				for i := range line.Layout {
					rtxt.adv = append(rtxt.adv, line.Layout[i].Advance)
				}
			}

			siter.AdvanceTo(pos + len)
			pos += len
			chunk.s = chunk.s[len:]
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package rpc_test

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"sync"

	"code.google.com/p/go.net/websocket"
	gc "launchpad.net/gocheck"

	"github.com/juju/juju/rpc"
	"github.com/juju/juju/rpc/jsoncodec"
)

type dispatchSuite struct {
}

var (
	_          = gc.Suite(&dispatchSuite{})
	serverAddr string
	once       sync.Once
)

type ReflectDummy struct{}

func (*ReflectDummy) DoSomething() {}

func (*dispatchSuite) TestWebsocket(c *gc.C) {
	once.Do(startWebsocketServer)

	respA := request(c, `{"RequestId":1,"Type": "ReflectDummy","Id": "id","Request":"DoSomething", "Params": {}}`)
	c.Assert(respA, gc.Equals, "OK")

	respB := request(c, `{"RequestId":1,"Type": "ReflectDummy","Id": "id","Request":"DoSomething"}`)
	c.Assert(respB, gc.Equals, "OK")
}

func request(c *gc.C, req string) string {
	url := fmt.Sprintf("ws://%s/jsoncodec", serverAddr)
	ws, err := websocket.Dial(url, "", "http://localhost")
	c.Assert(err, gc.IsNil)

	reqdata := []byte(req)
	_, err = ws.Write(reqdata)
	c.Assert(err, gc.IsNil)

	var resp = make([]byte, 512)
	n, err := ws.Read(resp)
	c.Assert(err, gc.IsNil)
	resp = resp[0:n]

	err = ws.Close()
	c.Assert(err, gc.IsNil)

	return string(resp)
}

func startWebsocketServer() {
	codecServer := func(ws *websocket.Conn) {
		codec := jsoncodec.NewWebsocket(ws)
		conn := rpc.NewConn(codec, nil)
		conn.Start()
	}
	http.Handle("/jsoncodec", websocket.Handler(codecServer))
	server := httptest.NewServer(nil)
	serverAddr = server.Listener.Addr().String()
}
<commit_msg>made the test more standard conform<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package rpc_test

import (
	"fmt"
	"net/http"
	"net/http/httptest"

	"code.google.com/p/go.net/websocket"
	gc "launchpad.net/gocheck"

	"github.com/juju/juju/rpc"
	"github.com/juju/juju/rpc/jsoncodec"
)

type dispatchSuite struct {
	server     *httptest.Server
	serverAddr string
	ws         *websocket.Conn
}

var _ = gc.Suite(&dispatchSuite{})

func (s *dispatchSuite) SetUpSuite(c *gc.C) {
	codecServer := func(ws *websocket.Conn) {
		codec := jsoncodec.NewWebsocket(ws)
		conn := rpc.NewConn(codec, nil)

		conn.Start()
		conn.Serve(&DispatchRoot{}, nil)

		<-conn.Dead()
	}
	http.Handle("/jsoncodec", websocket.Handler(codecServer))
	s.server = httptest.NewServer(nil)
	s.serverAddr = s.server.Listener.Addr().String()

	url := fmt.Sprintf("ws://%s/jsoncodec", s.serverAddr)
	ws, err := websocket.Dial(url, "", "http://localhost")
	c.Assert(err, gc.IsNil)
	s.ws = ws
}

func (s *dispatchSuite) TearDownSuite(c *gc.C) {
	err := s.ws.Close()
	c.Assert(err, gc.IsNil)

	s.server.Close()
}

func (s *dispatchSuite) TestWSWithoutParams(c *gc.C) {
	resp := s.request(c, `{"RequestId":1,"Type": "DispatchDummy","Id": "without","Request":"DoSomething"}`)
	c.Assert(resp, gc.Equals, `{"RequestId":1,"Response":{}}`)
}

func (s *dispatchSuite) TestWSWithParams(c *gc.C) {
	resp := s.request(c, `{"RequestId":2,"Type": "DispatchDummy","Id": "with","Request":"DoSomething", "Params": {}}`)
	c.Assert(resp, gc.Equals, `{"RequestId":2,"Response":{}}`)
}

// request performs one request to the test server via websockets.
func (s *dispatchSuite) request(c *gc.C, req string) string {
	reqdata := []byte(req)
	_, err := s.ws.Write(reqdata)
	c.Assert(err, gc.IsNil)

	var resp = make([]byte, 512)
	n, err := s.ws.Read(resp)
	c.Assert(err, gc.IsNil)
	resp = resp[0:n]

	return string(resp)
}

// DispatchRoot simulates the root for the test.
type DispatchRoot struct{}

func (*DispatchRoot) DispatchDummy(id string) (*DispatchDummy, error) {
	return &DispatchDummy{}, nil
}

// DispatchDummy is the type to whish the request is dispatched.
type DispatchDummy struct{}

func (d *DispatchDummy) DoSomething() {}
<|endoftext|>


<commit_before>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
	"bytes"

	"github.com/couchbase/moss"
)

type Iterator struct {
	store  *Store
	ss     moss.Snapshot
	iter   moss.Iterator
	prefix []byte
	start  []byte
	end    []byte
	done   bool
	k      []byte
	v      []byte
}

func (x *Iterator) Seek(seekToKey []byte) {
	x.done = true
	x.k = nil
	x.v = nil

	if bytes.Compare(seekToKey, x.start) < 0 {
		seekToKey = x.start
	}

	iter, err := x.ss.StartIterator(seekToKey, x.end, moss.IteratorOptions{})
	if err != nil {
		x.store.Logf("bleve moss StartIterator err: %v", err)
		return
	}

	err = x.iter.Close()
	if err != nil {
		x.store.Logf("bleve moss iterator.Seek err: %v", err)
		return
	}

	x.iter = iter

	x.checkDone()
}

func (x *Iterator) Next() {
	if x.done {
		return
	}

	x.done = true
	x.k = nil
	x.v = nil

	err := x.iter.Next()
	if err != nil {
		return
	}

	x.checkDone()
}

func (x *Iterator) Current() ([]byte, []byte, bool) {
	return x.k, x.v, !x.done
}

func (x *Iterator) Key() []byte {
	if x.done {
		return nil
	}

	return x.k
}

func (x *Iterator) Value() []byte {
	if x.done {
		return nil
	}

	return x.v
}

func (x *Iterator) Valid() bool {
	return !x.done
}

func (x *Iterator) Close() error {
	var err error

	x.ss = nil

	if x.iter != nil {
		err = x.iter.Close()
		x.iter = nil
	}

	x.prefix = nil
	x.done = true
	x.k = nil
	x.v = nil

	return err
}

func (x *Iterator) checkDone() {
	k, v, err := x.iter.Current()
	if err != nil || (x.prefix != nil && !bytes.HasPrefix(k, x.prefix)) {
		x.done = true
		x.k = nil
		x.v = nil
		return
	}

	x.done = false
	x.k = k
	x.v = v
}
<commit_msg>optimize moss iterator Next() done/k/v maintenance<commit_after>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
	"bytes"

	"github.com/couchbase/moss"
)

type Iterator struct {
	store  *Store
	ss     moss.Snapshot
	iter   moss.Iterator
	prefix []byte
	start  []byte
	end    []byte
	done   bool
	k      []byte
	v      []byte
}

func (x *Iterator) Seek(seekToKey []byte) {
	x.done = true
	x.k = nil
	x.v = nil

	if bytes.Compare(seekToKey, x.start) < 0 {
		seekToKey = x.start
	}

	iter, err := x.ss.StartIterator(seekToKey, x.end, moss.IteratorOptions{})
	if err != nil {
		x.store.Logf("bleve moss StartIterator err: %v", err)
		return
	}

	err = x.iter.Close()
	if err != nil {
		x.store.Logf("bleve moss iterator.Seek err: %v", err)
		return
	}

	x.iter = iter

	x.checkDone()
}

func (x *Iterator) Next() {
	if x.done {
		return
	}

	err := x.iter.Next()
	if err != nil {
		x.done = true
		x.k = nil
		x.v = nil
		return
	}

	x.checkDone()
}

func (x *Iterator) Current() ([]byte, []byte, bool) {
	return x.k, x.v, !x.done
}

func (x *Iterator) Key() []byte {
	if x.done {
		return nil
	}

	return x.k
}

func (x *Iterator) Value() []byte {
	if x.done {
		return nil
	}

	return x.v
}

func (x *Iterator) Valid() bool {
	return !x.done
}

func (x *Iterator) Close() error {
	var err error

	x.ss = nil

	if x.iter != nil {
		err = x.iter.Close()
		x.iter = nil
	}

	x.prefix = nil
	x.done = true
	x.k = nil
	x.v = nil

	return err
}

func (x *Iterator) checkDone() {
	k, v, err := x.iter.Current()
	if err != nil || (x.prefix != nil && !bytes.HasPrefix(k, x.prefix)) {
		x.done = true
		x.k = nil
		x.v = nil
		return
	}

	x.done = false
	x.k = k
	x.v = v
}
<|endoftext|>


<commit_before>/*
   Copyright 2020 Docker, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"testing"
	"time"

	"gotest.tools/v3/assert"
	"gotest.tools/v3/poll"

	"gotest.tools/v3/icmd"

	. "github.com/docker/api/tests/framework"
)

var binDir string

func TestMain(m *testing.M) {
	p, cleanup, err := SetupExistingCLI()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	binDir = p
	exitCode := m.Run()
	cleanup()
	os.Exit(exitCode)
}

func TestCompose(t *testing.T) {
	startTime := strconv.Itoa(int(time.Now().UnixNano()))
	c := NewE2eCLI(t, binDir)
	contextName := "teste2e" + startTime
	stack := contextName

	t.Run("create context", func(t *testing.T) {
		localTestProfile := os.Getenv("TEST_AWS_PROFILE")
		var res *icmd.Result
		if localTestProfile != "" {
			region := os.Getenv("TEST_AWS_REGION")
			assert.Check(t, region != "")
			res = c.RunDockerCmd("context", "create", "ecs", contextName, "--profile", localTestProfile, "--region", region)
			res.Assert(t, icmd.Success)
		} else {
			profile := contextName
			region := os.Getenv("AWS_DEFAULT_REGION")
			secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
			keyID := os.Getenv("AWS_ACCESS_KEY_ID")
			assert.Check(t, keyID != "")
			assert.Check(t, secretKey != "")
			assert.Check(t, region != "")
			res = c.RunDockerCmd("context", "create", "ecs", contextName, "--profile", profile, "--region", region, "--secret-key", secretKey, "--key-id", keyID)
			res.Assert(t, icmd.Success)
		}
		res = c.RunDockerCmd("context", "use", contextName)
		res.Assert(t, icmd.Expected{Out: contextName})
		res = c.RunDockerCmd("context", "ls")
		res.Assert(t, icmd.Expected{Out: contextName + " *"})
	})

	t.Run("compose up", func(t *testing.T) {
		res := c.RunDockerCmd("compose", "up", "--project-name", stack, "-f", "../composefiles/nginx.yaml")
		res.Assert(t, icmd.Success)
	})

	var url string
	t.Run("compose ps", func(t *testing.T) {
		res := c.RunDockerCmd("compose", "ps", "--project-name", stack)
		res.Assert(t, icmd.Success)
		lines := strings.Split(res.Stdout(), "\n")

		assert.Equal(t, 3, len(lines))
		fields := strings.Fields(lines[1])
		assert.Equal(t, 4, len(fields))
		assert.Check(t, strings.Contains(fields[0], stack))
		assert.Equal(t, "nginx", fields[1])
		assert.Equal(t, "1/1", fields[2])
		assert.Check(t, strings.Contains(fields[3], "->80/http"))
		url = "http://" + strings.Replace(fields[3], "->80/http", "", 1)
	})

	t.Run("nginx GET", func(t *testing.T) {
		checkUp := func(t poll.LogT) poll.Result {
			r, err := http.Get(url)
			if err != nil {
				return poll.Continue("Err while getting %s : %v", url, err)
			} else if r.StatusCode != http.StatusOK {
				return poll.Continue("status %s while getting %s", r.Status, url)
			}
			b, err := ioutil.ReadAll(r.Body)
			if err == nil && strings.Contains(string(b), "Welcome to nginx!") {
				return poll.Success()
			}
			return poll.Error(fmt.Errorf("No nginx welcome page received at %s: \n%s", url, string(b)))
		}
		poll.WaitOn(t, checkUp, poll.WithDelay(2*time.Second), poll.WithTimeout(60*time.Second))
	})

	t.Run("compose down", func(t *testing.T) {
		res := c.RunDockerCmd("compose", "down", "--project-name", stack, "-f", "../composefiles/nginx.yaml")
		res.Assert(t, icmd.Success)
	})
}
<commit_msg>More ECS E2E tests (secrets), in // of compose<commit_after>/*
   Copyright 2020 Docker, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"testing"
	"time"

	"gotest.tools/v3/assert"
	"gotest.tools/v3/poll"

	"gotest.tools/v3/icmd"

	. "github.com/docker/api/tests/framework"
)

var binDir string

func TestMain(m *testing.M) {
	p, cleanup, err := SetupExistingCLI()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	binDir = p
	exitCode := m.Run()
	cleanup()
	os.Exit(exitCode)
}

func TestSecrets(t *testing.T) {
	c, testID := setupTest(t)
	secretName := "secret" + testID
	description := "description " + testID

	t.Run("create secret", func(t *testing.T) {
		res := c.RunDockerCmd("secret", "create", secretName, "-u", "user1", "-p", "pass1", "-d", description)
		res.Assert(t, icmd.Success)
		assert.Check(t, strings.Contains(res.Stdout(), "secret:"+secretName))
	})

	t.Run("list secrets", func(t *testing.T) {
		res := c.RunDockerCmd("secret", "list")
		res.Assert(t, icmd.Success)
		assert.Check(t, strings.Contains(res.Stdout(), secretName))
		assert.Check(t, strings.Contains(res.Stdout(), description))
	})

	t.Run("inspect secret", func(t *testing.T) {
		res := c.RunDockerCmd("secret", "inspect", secretName)
		res.Assert(t, icmd.Success)
		assert.Check(t, strings.Contains(res.Stdout(), `"Name": "`+secretName+`"`))
		assert.Check(t, strings.Contains(res.Stdout(), `"Description": "`+description+`"`))
	})

	t.Run("rm secret", func(t *testing.T) {
		res := c.RunDockerCmd("secret", "rm", secretName)
		res.Assert(t, icmd.Success)
		res = c.RunDockerCmd("secret", "list")
		res.Assert(t, icmd.Success)
		assert.Check(t, !strings.Contains(res.Stdout(), secretName))
	})
}

func TestCompose(t *testing.T) {
	c, stack := setupTest(t)

	t.Run("compose up", func(t *testing.T) {
		res := c.RunDockerCmd("compose", "up", "--project-name", stack, "-f", "../composefiles/nginx.yaml")
		res.Assert(t, icmd.Success)
	})

	var url string
	t.Run("compose ps", func(t *testing.T) {
		res := c.RunDockerCmd("compose", "ps", "--project-name", stack)
		res.Assert(t, icmd.Success)
		lines := strings.Split(res.Stdout(), "\n")

		assert.Equal(t, 3, len(lines))
		fields := strings.Fields(lines[1])
		assert.Equal(t, 4, len(fields))
		assert.Check(t, strings.Contains(fields[0], stack))
		assert.Equal(t, "nginx", fields[1])
		assert.Equal(t, "1/1", fields[2])
		assert.Check(t, strings.Contains(fields[3], "->80/http"))
		url = "http://" + strings.Replace(fields[3], "->80/http", "", 1)
	})

	t.Run("nginx GET", func(t *testing.T) {
		checkUp := func(t poll.LogT) poll.Result {
			r, err := http.Get(url)
			if err != nil {
				return poll.Continue("Err while getting %s : %v", url, err)
			} else if r.StatusCode != http.StatusOK {
				return poll.Continue("status %s while getting %s", r.Status, url)
			}
			b, err := ioutil.ReadAll(r.Body)
			if err == nil && strings.Contains(string(b), "Welcome to nginx!") {
				return poll.Success()
			}
			return poll.Error(fmt.Errorf("No nginx welcome page received at %s: \n%s", url, string(b)))
		}
		poll.WaitOn(t, checkUp, poll.WithDelay(2*time.Second), poll.WithTimeout(60*time.Second))
	})

	t.Run("compose down", func(t *testing.T) {
		res := c.RunDockerCmd("compose", "down", "--project-name", stack, "-f", "../composefiles/nginx.yaml")
		res.Assert(t, icmd.Success)
	})
}

func setupTest(t *testing.T) (*E2eCLI, string) {
	startTime := strconv.Itoa(int(time.Now().UnixNano()))
	c := NewParallelE2eCLI(t, binDir)
	contextName := "e2e" + t.Name() + startTime
	stack := contextName
	t.Run("create context", func(t *testing.T) {
		localTestProfile := os.Getenv("TEST_AWS_PROFILE")
		var res *icmd.Result
		if localTestProfile != "" {
			region := os.Getenv("TEST_AWS_REGION")
			assert.Check(t, region != "")
			res = c.RunDockerCmd("context", "create", "ecs", contextName, "--profile", localTestProfile, "--region", region)
			res.Assert(t, icmd.Success)
		} else {
			profile := contextName
			region := os.Getenv("AWS_DEFAULT_REGION")
			secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
			keyID := os.Getenv("AWS_ACCESS_KEY_ID")
			assert.Check(t, keyID != "")
			assert.Check(t, secretKey != "")
			assert.Check(t, region != "")
			res = c.RunDockerCmd("context", "create", "ecs", contextName, "--profile", profile, "--region", region, "--secret-key", secretKey, "--key-id", keyID)
			res.Assert(t, icmd.Success)
		}
		res = c.RunDockerCmd("context", "use", contextName)
		res.Assert(t, icmd.Expected{Out: contextName})
		res = c.RunDockerCmd("context", "ls")
		res.Assert(t, icmd.Expected{Out: contextName + " *"})
	})
	return c, stack
}
<|endoftext|>