<commit_before>package sql

import (
	"fmt"
	"strings"
)

type Query struct {
	SelectedFields []SelectedColumn
	FromTable      table
	Joins          []Join
	WhereRelation  Relation
	orders         []Order
	limit          *int64
	count          *string
}

func Select(fields ...SelectedColumn) *Query {
	query := Query{}

	for _, column := range fields {
		query.SelectedFields = append(query.SelectedFields, column)
	}

	return &query
}

func (q *Query) Count() *Query {
	count := "COUNT(*))"
	q.count = &count
	return q
}

func (q *Query) From(t table) *Query {
	q.FromTable = t

	return q
}

func (q *Query) InnerJoin(leftColumn SelectedColumn, rightColumn SelectedColumn) *Query {
	join := Join{Type: "INNER JOIN", LeftColumn: leftColumn, RightColumn: rightColumn}
	q.Joins = append(q.Joins, join)

	return q
}

func (q *Query) LeftJoin(leftColumn SelectedColumn, rightColumn SelectedColumn) *Query {
	join := Join{Type: "LEFT JOIN", LeftColumn: leftColumn, RightColumn: rightColumn}
	q.Joins = append(q.Joins, join)

	return q
}

func (q *Query) LeftOuterJoin(leftColumn SelectedColumn, rightColumn SelectedColumn) *Query {
	join := Join{Type: "LEFT OUTER JOIN", LeftColumn: leftColumn, RightColumn: rightColumn}
	q.Joins = append(q.Joins, join)

	return q
}

func (q *Query) Limit(value int64) *Query {
	q.limit = &value
	return q
}

func (q *Query) Order(orders ...Order) *Query {
	q.orders = orders
	return q
}

func (q *Query) Where(relation Relation) *Query {
	q.WhereRelation = relation
	return q
}

func (q Query) ToSQL() string {
	selectedColumnStrings := []string{}

	for _, selectedColumn := range q.SelectedFields {
		selectedColumnStrings = append(selectedColumnStrings, fmt.Sprintf("  %v", selectedColumn.SelectionString()))
	}

	joinStrings := []string{}

	for _, join := range q.Joins {
		joinStrings = append(joinStrings, fmt.Sprintf("\n%v %v ON %v = %v", join.Type, join.LeftColumn.Table.Name(), join.LeftColumn.String(), join.RightColumn.String()))
	}

	orderString := ""
	eachOrderStrings := []string{}
	for _, eachOrder := range q.orders {
		eachOrderStrings = append(eachOrderStrings, fmt.Sprintf("%v %v", eachOrder.Column.String(), eachOrder.Direction))
	}
	if len(eachOrderStrings) > 0 {
		orderString = fmt.Sprintf("\nORDER BY %v", strings.Join(eachOrderStrings, ", "))
	}

	limitString := ""
	if q.limit != nil {
		limitString = fmt.Sprintf("\nLIMIT %v", *q.limit)
	}

	whereString := ""
	if q.WhereRelation != nil {
		whereString = fmt.Sprintf("\nWHERE %v", q.WhereRelation.QueryFragment())
	}

	sql := `
SELECT
%v
FROM %v %v%v%v%v;
`

	if q.count != nil {
		return fmt.Sprintf(sql, q.count, q.FromTable.Name(), strings.Join(joinStrings, ""), whereString, orderString, limitString)
	}

	return fmt.Sprintf(sql, strings.Join(selectedColumnStrings, ",\n"), q.FromTable.Name(), strings.Join(joinStrings, ""), whereString, orderString, limitString)
}
<commit_msg>De-referencing :/<commit_after>package sql

import (
	"fmt"
	"strings"
)

type Query struct {
	SelectedFields []SelectedColumn
	FromTable      table
	Joins          []Join
	WhereRelation  Relation
	orders         []Order
	limit          *int64
	count          *string
}

func Select(fields ...SelectedColumn) *Query {
	query := Query{}

	for _, column := range fields {
		query.SelectedFields = append(query.SelectedFields, column)
	}

	return &query
}

func (q *Query) Count() *Query {
	count := "COUNT(*))"
	q.count = &count
	return q
}

func (q *Query) From(t table) *Query {
	q.FromTable = t

	return q
}

func (q *Query) InnerJoin(leftColumn SelectedColumn, rightColumn SelectedColumn) *Query {
	join := Join{Type: "INNER JOIN", LeftColumn: leftColumn, RightColumn: rightColumn}
	q.Joins = append(q.Joins, join)

	return q
}

func (q *Query) LeftJoin(leftColumn SelectedColumn, rightColumn SelectedColumn) *Query {
	join := Join{Type: "LEFT JOIN", LeftColumn: leftColumn, RightColumn: rightColumn}
	q.Joins = append(q.Joins, join)

	return q
}

func (q *Query) LeftOuterJoin(leftColumn SelectedColumn, rightColumn SelectedColumn) *Query {
	join := Join{Type: "LEFT OUTER JOIN", LeftColumn: leftColumn, RightColumn: rightColumn}
	q.Joins = append(q.Joins, join)

	return q
}

func (q *Query) Limit(value int64) *Query {
	q.limit = &value
	return q
}

func (q *Query) Order(orders ...Order) *Query {
	q.orders = orders
	return q
}

func (q *Query) Where(relation Relation) *Query {
	q.WhereRelation = relation
	return q
}

func (q Query) ToSQL() string {
	selectedColumnStrings := []string{}

	for _, selectedColumn := range q.SelectedFields {
		selectedColumnStrings = append(selectedColumnStrings, fmt.Sprintf("  %v", selectedColumn.SelectionString()))
	}

	joinStrings := []string{}

	for _, join := range q.Joins {
		joinStrings = append(joinStrings, fmt.Sprintf("\n%v %v ON %v = %v", join.Type, join.LeftColumn.Table.Name(), join.LeftColumn.String(), join.RightColumn.String()))
	}

	orderString := ""
	eachOrderStrings := []string{}
	for _, eachOrder := range q.orders {
		eachOrderStrings = append(eachOrderStrings, fmt.Sprintf("%v %v", eachOrder.Column.String(), eachOrder.Direction))
	}
	if len(eachOrderStrings) > 0 {
		orderString = fmt.Sprintf("\nORDER BY %v", strings.Join(eachOrderStrings, ", "))
	}

	limitString := ""
	if q.limit != nil {
		limitString = fmt.Sprintf("\nLIMIT %v", *q.limit)
	}

	whereString := ""
	if q.WhereRelation != nil {
		whereString = fmt.Sprintf("\nWHERE %v", q.WhereRelation.QueryFragment())
	}

	sql := `
SELECT
%v
FROM %v %v%v%v%v;
`

	if q.count != nil {
		return fmt.Sprintf(sql, *q.count, q.FromTable.Name(), strings.Join(joinStrings, ""), whereString, orderString, limitString)
	}

	return fmt.Sprintf(sql, strings.Join(selectedColumnStrings, ",\n"), q.FromTable.Name(), strings.Join(joinStrings, ""), whereString, orderString, limitString)
}
<|endoftext|>


<commit_before>package sqlgen

import (
	"context"
	"database/sql"
	"errors"

	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/log"
)

// DB uses a *sql.DB connection that is established by its owner. DB assumes the
// database connection exists and is alive at all times during the lifecycle of
// the object.
type DB struct {
	Conn   *sql.DB
	Schema *Schema
}

func NewDB(conn *sql.DB, schema *Schema) *DB {
	return &DB{
		Conn:   conn,
		Schema: schema,
	}
}

// Query fetches a collection of rows from the database
//
// result should be a pointer to a slice of pointers to structs, for example:
//
//   var users []*User
//   if err := db.Query(ctx, &users, nil, nil); err != nil {
//
func (db *DB) Query(ctx context.Context, result interface{}, filter Filter, options *SelectOptions) error {
	query, err := db.Schema.MakeSelect(result, filter, options)
	if err != nil {
		return err
	}
	clause, fields := query.ToSQL()

	if span := opentracing.SpanFromContext(ctx); span != nil {
		span, ctx = opentracing.StartSpanFromContext(ctx, "thunder.sqlgen.UpsertRow")
		span.LogFields(log.String("query", clause))
		defer span.Finish()
	}

	res, err := db.QueryExecer(ctx).Query(clause, fields...)
	if err != nil {
		return err
	}
	defer res.Close()

	rows, err := db.Schema.ParseRows(query, res)
	if err != nil {
		return err
	}

	return CopySlice(result, rows)
}

// QueryRow fetches a single row from the database
//
// result should be a pointer to a pointer to a struct, for example:
//
//   var user *User
//   if err := db.Query(ctx, &user, Filter{"id": 10}, nil); err != nil {
//
func (db *DB) QueryRow(ctx context.Context, result interface{}, filter Filter, options *SelectOptions) error {
	query, err := db.Schema.MakeSelectRow(result, filter, options)
	if err != nil {
		return err
	}
	querySql, fields := query.ToSQL()

	if span := opentracing.SpanFromContext(ctx); span != nil {
		span, ctx = opentracing.StartSpanFromContext(ctx, "thunder.sqlgen.QueryRow")
		span.LogFields(log.String("query", querySql))
		defer span.Finish()
	}

	res, err := db.QueryExecer(ctx).Query(querySql, fields...)
	if err != nil {
		return err
	}
	defer res.Close()

	rows, err := db.Schema.ParseRows(query, res)
	if err != nil {
		return err
	}

	return CopySingletonSlice(result, rows)
}

// InsertRow inserts a single row into the database
//
// row should be a pointer to a struct, for example:
//
//   user := &User{Name: "foo"}
//   if err := db.InsertRow(ctx, user); err != nil {
//
func (db *DB) InsertRow(ctx context.Context, row interface{}) (sql.Result, error) {
	query, err := db.Schema.MakeInsertRow(row)
	if err != nil {
		return nil, err
	}

	clause, args := query.ToSQL()

	if span := opentracing.SpanFromContext(ctx); span != nil {
		span, ctx = opentracing.StartSpanFromContext(ctx, "thunder.sqlgen.InsertRow")
		span.LogFields(log.String("query", clause))
		defer span.Finish()
	}

	return db.QueryExecer(ctx).Exec(clause, args...)
}

// UpsertRow inserts a single row into the database
//
// row should be a pointer to a struct, for example:
//
//   user := &User{Name: "foo"}
//   if err := db.UpsertRow(ctx, user); err != nil {
//
func (db *DB) UpsertRow(ctx context.Context, row interface{}) (sql.Result, error) {
	query, err := db.Schema.MakeUpsertRow(row)
	if err != nil {
		return nil, err
	}

	clause, args := query.ToSQL()

	if span := opentracing.SpanFromContext(ctx); span != nil {
		span, ctx = opentracing.StartSpanFromContext(ctx, "thunder.sqlgen.UpsertRow")
		span.LogFields(log.String("query", clause))
		defer span.Finish()
	}

	return db.QueryExecer(ctx).Exec(clause, args...)
}

// UpdateRow updates a single row in the database, identified by the row's primary key
//
// row should be a pointer to a struct, for example:
//
//   user := &User{Id; 10, Name: "bar"}
//   if err := db.UpdateRow(ctx, user); err != nil {
//
func (db *DB) UpdateRow(ctx context.Context, row interface{}) error {
	query, err := db.Schema.MakeUpdateRow(row)
	if err != nil {
		return err
	}

	clause, args := query.ToSQL()

	if span := opentracing.SpanFromContext(ctx); span != nil {
		span, ctx = opentracing.StartSpanFromContext(ctx, "thunder.sqlgen.UpdateRow")
		span.LogFields(log.String("query", clause))
		defer span.Finish()
	}

	_, err = db.QueryExecer(ctx).Exec(clause, args...)
	return err
}

// DeleteRow deletes a single row from the database, identified by the row's primary key
//
// row should be a pointer to a struct, for example:
//
//   user := &User{Id; 10}
//   if err := db.DeleteRow(ctx, user); err != nil {
//
func (db *DB) DeleteRow(ctx context.Context, row interface{}) error {
	query, err := db.Schema.MakeDeleteRow(row)
	if err != nil {
		return err
	}

	clause, args := query.ToSQL()

	if span := opentracing.SpanFromContext(ctx); span != nil {
		span, ctx = opentracing.StartSpanFromContext(ctx, "thunder.sqlgen.DeleteRow")
		span.LogFields(log.String("query", clause))
		defer span.Finish()
	}

	_, err = db.QueryExecer(ctx).Exec(clause, args...)
	return err
}

// txKey is used as a key for a context.Context to hold a transaction.
//
// With multiple open databases, each database can store its own transactions in a context.
type txKey struct {
	db *DB
}

func (db *DB) WithTx(ctx context.Context) (context.Context, *sql.Tx, error) {
	maybeTx := ctx.Value(txKey{db: db})
	if maybeTx != nil {
		return nil, nil, errors.New("already in a tx")
	}

	tx, err := db.Conn.Begin()
	if err != nil {
		return nil, nil, err
	}
	return context.WithValue(ctx, txKey{db: db}, tx), tx, nil
}

func (db *DB) WithExistingTx(ctx context.Context, tx *sql.Tx) (context.Context, error) {
	maybeTx := ctx.Value(txKey{db: db})
	if maybeTx != nil {
		return nil, errors.New("already in a tx")
	}

	return context.WithValue(ctx, txKey{db: db}, tx), nil
}

func (db *DB) HasTx(ctx context.Context) bool {
	return ctx.Value(txKey{db: db}) != nil
}

// A QueryExecer is either a *sql.Tx or a *sql.DB.
type QueryExecer interface {
	Query(query string, args ...interface{}) (*sql.Rows, error)
	QueryRow(query string, args ...interface{}) *sql.Row
	Exec(query string, args ...interface{}) (sql.Result, error)
}

func (db *DB) QueryExecer(ctx context.Context) QueryExecer {
	maybeTx := ctx.Value(txKey{db: db})
	if maybeTx != nil {
		return maybeTx.(*sql.Tx)
	}
	return db.Conn
}
<commit_msg>sqlgen: deduplicate common code shared between Query and QueryRow<commit_after>package sqlgen

import (
	"context"
	"database/sql"
	"errors"

	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/log"
)

// DB uses a *sql.DB connection that is established by its owner. DB assumes the
// database connection exists and is alive at all times during the lifecycle of
// the object.
type DB struct {
	Conn   *sql.DB
	Schema *Schema
}

func NewDB(conn *sql.DB, schema *Schema) *DB {
	return &DB{
		Conn:   conn,
		Schema: schema,
	}
}

func (db *DB) query(ctx context.Context, query *sqlgen.SelectQuery) ([]interface{}, error) {
	clause, fields := query.ToSQL()

	if span := opentracing.SpanFromContext(ctx); span != nil {
		span, ctx = opentracing.StartSpanFromContext(ctx, "thunder.sqlgen.query")
		span.LogFields(log.String("query", querySql))
		defer span.Finish()
	}

	res, err := db.QueryExecer(ctx).Query(querySql, fields...)
	if err != nil {
		return err
	}
	defer res.Close()

	return db.Schema.ParseRows(query, res)
}

// Query fetches a collection of rows from the database
//
// result should be a pointer to a slice of pointers to structs, for example:
//
//   var users []*User
//   if err := db.Query(ctx, &users, nil, nil); err != nil {
//
func (db *DB) Query(ctx context.Context, result interface{}, filter Filter, options *SelectOptions) error {
	query, err := db.Schema.MakeSelect(result, filter, options)
	if err != nil {
		return err
	}

	rows, err := db.query(ctx, query)
	if err != nil {
		return err
	}

	return CopySlice(result, rows)
}

// QueryRow fetches a single row from the database
//
// result should be a pointer to a pointer to a struct, for example:
//
//   var user *User
//   if err := db.Query(ctx, &user, Filter{"id": 10}, nil); err != nil {
//
func (db *DB) QueryRow(ctx context.Context, result interface{}, filter Filter, options *SelectOptions) error {
	query, err := db.Schema.MakeSelectRow(result, filter, options)
	if err != nil {
		return err
	}

	rows, err := db.query(ctx, query)
	if err != nil {
		return err
	}

	return CopySingletonSlice(result, rows)
}

// InsertRow inserts a single row into the database
//
// row should be a pointer to a struct, for example:
//
//   user := &User{Name: "foo"}
//   if err := db.InsertRow(ctx, user); err != nil {
//
func (db *DB) InsertRow(ctx context.Context, row interface{}) (sql.Result, error) {
	query, err := db.Schema.MakeInsertRow(row)
	if err != nil {
		return nil, err
	}

	clause, args := query.ToSQL()

	if span := opentracing.SpanFromContext(ctx); span != nil {
		span, ctx = opentracing.StartSpanFromContext(ctx, "thunder.sqlgen.InsertRow")
		span.LogFields(log.String("query", clause))
		defer span.Finish()
	}

	return db.QueryExecer(ctx).Exec(clause, args...)
}

// UpsertRow inserts a single row into the database
//
// row should be a pointer to a struct, for example:
//
//   user := &User{Name: "foo"}
//   if err := db.UpsertRow(ctx, user); err != nil {
//
func (db *DB) UpsertRow(ctx context.Context, row interface{}) (sql.Result, error) {
	query, err := db.Schema.MakeUpsertRow(row)
	if err != nil {
		return nil, err
	}

	clause, args := query.ToSQL()

	if span := opentracing.SpanFromContext(ctx); span != nil {
		span, ctx = opentracing.StartSpanFromContext(ctx, "thunder.sqlgen.UpsertRow")
		span.LogFields(log.String("query", clause))
		defer span.Finish()
	}

	return db.QueryExecer(ctx).Exec(clause, args...)
}

// UpdateRow updates a single row in the database, identified by the row's primary key
//
// row should be a pointer to a struct, for example:
//
//   user := &User{Id; 10, Name: "bar"}
//   if err := db.UpdateRow(ctx, user); err != nil {
//
func (db *DB) UpdateRow(ctx context.Context, row interface{}) error {
	query, err := db.Schema.MakeUpdateRow(row)
	if err != nil {
		return err
	}

	clause, args := query.ToSQL()

	if span := opentracing.SpanFromContext(ctx); span != nil {
		span, ctx = opentracing.StartSpanFromContext(ctx, "thunder.sqlgen.UpdateRow")
		span.LogFields(log.String("query", clause))
		defer span.Finish()
	}

	_, err = db.QueryExecer(ctx).Exec(clause, args...)
	return err
}

// DeleteRow deletes a single row from the database, identified by the row's primary key
//
// row should be a pointer to a struct, for example:
//
//   user := &User{Id; 10}
//   if err := db.DeleteRow(ctx, user); err != nil {
//
func (db *DB) DeleteRow(ctx context.Context, row interface{}) error {
	query, err := db.Schema.MakeDeleteRow(row)
	if err != nil {
		return err
	}

	clause, args := query.ToSQL()

	if span := opentracing.SpanFromContext(ctx); span != nil {
		span, ctx = opentracing.StartSpanFromContext(ctx, "thunder.sqlgen.DeleteRow")
		span.LogFields(log.String("query", clause))
		defer span.Finish()
	}

	_, err = db.QueryExecer(ctx).Exec(clause, args...)
	return err
}

// txKey is used as a key for a context.Context to hold a transaction.
//
// With multiple open databases, each database can store its own transactions in a context.
type txKey struct {
	db *DB
}

func (db *DB) WithTx(ctx context.Context) (context.Context, *sql.Tx, error) {
	maybeTx := ctx.Value(txKey{db: db})
	if maybeTx != nil {
		return nil, nil, errors.New("already in a tx")
	}

	tx, err := db.Conn.Begin()
	if err != nil {
		return nil, nil, err
	}
	return context.WithValue(ctx, txKey{db: db}, tx), tx, nil
}

func (db *DB) WithExistingTx(ctx context.Context, tx *sql.Tx) (context.Context, error) {
	maybeTx := ctx.Value(txKey{db: db})
	if maybeTx != nil {
		return nil, errors.New("already in a tx")
	}

	return context.WithValue(ctx, txKey{db: db}, tx), nil
}

func (db *DB) HasTx(ctx context.Context) bool {
	return ctx.Value(txKey{db: db}) != nil
}

// A QueryExecer is either a *sql.Tx or a *sql.DB.
type QueryExecer interface {
	Query(query string, args ...interface{}) (*sql.Rows, error)
	QueryRow(query string, args ...interface{}) *sql.Row
	Exec(query string, args ...interface{}) (sql.Result, error)
}

func (db *DB) QueryExecer(ctx context.Context) QueryExecer {
	maybeTx := ctx.Value(txKey{db: db})
	if maybeTx != nil {
		return maybeTx.(*sql.Tx)
	}
	return db.Conn
}
<|endoftext|>


<commit_before>// A FUSE filesystem that shunts all request to an underlying file
// system.  Its main purpose is to provide test coverage without
// having to build an actual synthetic filesystem.

package fuse

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"syscall"
)

var _ = fmt.Println
var _ = log.Println

type LoopbackFileSystem struct {
	root string

	DefaultPathFilesystem
}

func NewLoopbackFileSystem(root string) (out *LoopbackFileSystem) {
	out = new(LoopbackFileSystem)
	out.root = root

	return out
}

func (me *LoopbackFileSystem) GetPath(relPath string) string {
	return filepath.Join(me.root, relPath)
}

func (me *LoopbackFileSystem) GetAttr(name string) (*Attr, Status) {
	fullPath := me.GetPath(name)
	fi, err := os.Lstat(fullPath)
	if err != nil {
		return nil, ENOENT
	}
	out := new(Attr)
	CopyFileInfo(fi, out)

	return out, OK
}

func (me *LoopbackFileSystem) OpenDir(name string) (stream chan DirEntry, status Status) {
	// What other ways beyond O_RDONLY are there to open
	// directories?
	f, err := os.Open(me.GetPath(name), os.O_RDONLY, 0)
	if err != nil {
		return nil, OsErrorToFuseError(err)
	}
	output := make(chan DirEntry, 500)
	go func() {
		for {
			want := 500
			infos, err := f.Readdir(want)
			for i, _ := range infos {
				output <- DirEntry{
					Name: infos[i].Name,
					Mode: infos[i].Mode,
				}
			}
			if len(infos) < want {
				break
			}
			if err != nil {
				// TODO - how to signal error
				break
			}
		}
		output <- DirEntry{}
		f.Close()
	}()

	return output, OK
}

func (me *LoopbackFileSystem) Open(name string, flags uint32) (fuseFile RawFuseFile, status Status) {
	f, err := os.Open(me.GetPath(name), int(flags), 0)
	if err != nil {
		return nil, OsErrorToFuseError(err)
	}
	return &LoopbackFile{file: f}, OK
}

func (me *LoopbackFileSystem) Chmod(path string, mode uint32) (code Status) {
	err := os.Chmod(me.GetPath(path), mode)
	return OsErrorToFuseError(err)
}

func (me *LoopbackFileSystem) Chown(path string, uid uint32, gid uint32) (code Status) {
	return OsErrorToFuseError(os.Chown(me.GetPath(path), int(uid), int(gid)))
}

func (me *LoopbackFileSystem) Truncate(path string, offset uint64) (code Status) {
	return OsErrorToFuseError(os.Truncate(me.GetPath(path), int64(offset)))
}

func (me *LoopbackFileSystem) Utimens(path string, AtimeNs uint64, MtimeNs uint64) (code Status) {
	return OsErrorToFuseError(os.Chtimes(me.GetPath(path), int64(AtimeNs), int64(MtimeNs)))
}

func (me *LoopbackFileSystem) Readlink(name string) (out string, code Status) {
	f, err := os.Readlink(me.GetPath(name))
	return f, OsErrorToFuseError(err)
}

func (me *LoopbackFileSystem) Mknod(name string, mode uint32, dev uint32) (code Status) {
	return Status(syscall.Mknod(me.GetPath(name), mode, int(dev)))
}

func (me *LoopbackFileSystem) Mkdir(path string, mode uint32) (code Status) {
	return OsErrorToFuseError(os.Mkdir(me.GetPath(path), mode))
}

// Don't use os.Remove, it removes twice (unlink followed by rmdir).
func (me *LoopbackFileSystem) Unlink(name string) (code Status) {
	return Status(syscall.Unlink(me.GetPath(name)))
}

func (me *LoopbackFileSystem) Rmdir(name string) (code Status) {
	return Status(syscall.Rmdir(me.GetPath(name)))
}

func (me *LoopbackFileSystem) Symlink(pointedTo string, linkName string) (code Status) {
	return OsErrorToFuseError(os.Symlink(pointedTo, me.GetPath(linkName)))
}

func (me *LoopbackFileSystem) Rename(oldPath string, newPath string) (code Status) {
	err := os.Rename(me.GetPath(oldPath), me.GetPath(newPath))
	return OsErrorToFuseError(err)
}

func (me *LoopbackFileSystem) Link(orig string, newName string) (code Status) {
	return OsErrorToFuseError(os.Link(me.GetPath(orig), me.GetPath(newName)))
}

func (me *LoopbackFileSystem) Access(name string, mode uint32) (code Status) {
	return Status(syscall.Access(me.GetPath(name), mode))
}

func (me *LoopbackFileSystem) Create(path string, flags uint32, mode uint32) (fuseFile RawFuseFile, code Status) {
	f, err := os.Open(me.GetPath(path), int(flags)|os.O_CREAT, mode)
	return &LoopbackFile{file: f}, OsErrorToFuseError(err)
}

func (me *LoopbackFileSystem) GetXAttr(name string, attr string) ([]byte, Status) {
	data, errNo := GetXAttr(me.GetPath(name), attr)

	return data, Status(errNo)
}

func (me *LoopbackFileSystem) SetOptions(options *PathFileSystemConnectorOptions) {
	options.NegativeTimeout = 100.0
	options.AttrTimeout = 100.0
	options.EntryTimeout = 100.0
}

////////////////////////////////////////////////////////////////

type LoopbackFile struct {
	file *os.File

	DefaultRawFuseFile
}

func (me *LoopbackFile) Read(input *ReadIn, buffers *BufferPool) ([]byte, Status) {
	slice := buffers.AllocBuffer(input.Size)

	n, err := me.file.ReadAt(slice, int64(input.Offset))
	if err == os.EOF {
		// TODO - how to signal EOF?
		return slice[:n], OK
	}
	return slice[:n], OsErrorToFuseError(err)
}

func (me *LoopbackFile) Write(input *WriteIn, data []byte) (uint32, Status) {
	n, err := me.file.WriteAt(data, int64(input.Offset))
	return uint32(n), OsErrorToFuseError(err)
}

func (me *LoopbackFile) Release() {
	me.file.Close()
}

func (me *LoopbackFile) Fsync(*FsyncIn) (code Status) {
	return Status(syscall.Fsync(me.file.Fd()))
}
<commit_msg>Set more conservative values of cache timeouts.<commit_after>// A FUSE filesystem that shunts all request to an underlying file
// system.  Its main purpose is to provide test coverage without
// having to build an actual synthetic filesystem.

package fuse

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"syscall"
)

var _ = fmt.Println
var _ = log.Println

type LoopbackFileSystem struct {
	root string

	DefaultPathFilesystem
}

func NewLoopbackFileSystem(root string) (out *LoopbackFileSystem) {
	out = new(LoopbackFileSystem)
	out.root = root

	return out
}

func (me *LoopbackFileSystem) GetPath(relPath string) string {
	return filepath.Join(me.root, relPath)
}

func (me *LoopbackFileSystem) GetAttr(name string) (*Attr, Status) {
	fullPath := me.GetPath(name)
	fi, err := os.Lstat(fullPath)
	if err != nil {
		return nil, ENOENT
	}
	out := new(Attr)
	CopyFileInfo(fi, out)

	return out, OK
}

func (me *LoopbackFileSystem) OpenDir(name string) (stream chan DirEntry, status Status) {
	// What other ways beyond O_RDONLY are there to open
	// directories?
	f, err := os.Open(me.GetPath(name), os.O_RDONLY, 0)
	if err != nil {
		return nil, OsErrorToFuseError(err)
	}
	output := make(chan DirEntry, 500)
	go func() {
		for {
			want := 500
			infos, err := f.Readdir(want)
			for i, _ := range infos {
				output <- DirEntry{
					Name: infos[i].Name,
					Mode: infos[i].Mode,
				}
			}
			if len(infos) < want {
				break
			}
			if err != nil {
				// TODO - how to signal error
				break
			}
		}
		output <- DirEntry{}
		f.Close()
	}()

	return output, OK
}

func (me *LoopbackFileSystem) Open(name string, flags uint32) (fuseFile RawFuseFile, status Status) {
	f, err := os.Open(me.GetPath(name), int(flags), 0)
	if err != nil {
		return nil, OsErrorToFuseError(err)
	}
	return &LoopbackFile{file: f}, OK
}

func (me *LoopbackFileSystem) Chmod(path string, mode uint32) (code Status) {
	err := os.Chmod(me.GetPath(path), mode)
	return OsErrorToFuseError(err)
}

func (me *LoopbackFileSystem) Chown(path string, uid uint32, gid uint32) (code Status) {
	return OsErrorToFuseError(os.Chown(me.GetPath(path), int(uid), int(gid)))
}

func (me *LoopbackFileSystem) Truncate(path string, offset uint64) (code Status) {
	return OsErrorToFuseError(os.Truncate(me.GetPath(path), int64(offset)))
}

func (me *LoopbackFileSystem) Utimens(path string, AtimeNs uint64, MtimeNs uint64) (code Status) {
	return OsErrorToFuseError(os.Chtimes(me.GetPath(path), int64(AtimeNs), int64(MtimeNs)))
}

func (me *LoopbackFileSystem) Readlink(name string) (out string, code Status) {
	f, err := os.Readlink(me.GetPath(name))
	return f, OsErrorToFuseError(err)
}

func (me *LoopbackFileSystem) Mknod(name string, mode uint32, dev uint32) (code Status) {
	return Status(syscall.Mknod(me.GetPath(name), mode, int(dev)))
}

func (me *LoopbackFileSystem) Mkdir(path string, mode uint32) (code Status) {
	return OsErrorToFuseError(os.Mkdir(me.GetPath(path), mode))
}

// Don't use os.Remove, it removes twice (unlink followed by rmdir).
func (me *LoopbackFileSystem) Unlink(name string) (code Status) {
	return Status(syscall.Unlink(me.GetPath(name)))
}

func (me *LoopbackFileSystem) Rmdir(name string) (code Status) {
	return Status(syscall.Rmdir(me.GetPath(name)))
}

func (me *LoopbackFileSystem) Symlink(pointedTo string, linkName string) (code Status) {
	return OsErrorToFuseError(os.Symlink(pointedTo, me.GetPath(linkName)))
}

func (me *LoopbackFileSystem) Rename(oldPath string, newPath string) (code Status) {
	err := os.Rename(me.GetPath(oldPath), me.GetPath(newPath))
	return OsErrorToFuseError(err)
}

func (me *LoopbackFileSystem) Link(orig string, newName string) (code Status) {
	return OsErrorToFuseError(os.Link(me.GetPath(orig), me.GetPath(newName)))
}

func (me *LoopbackFileSystem) Access(name string, mode uint32) (code Status) {
	return Status(syscall.Access(me.GetPath(name), mode))
}

func (me *LoopbackFileSystem) Create(path string, flags uint32, mode uint32) (fuseFile RawFuseFile, code Status) {
	f, err := os.Open(me.GetPath(path), int(flags)|os.O_CREAT, mode)
	return &LoopbackFile{file: f}, OsErrorToFuseError(err)
}

func (me *LoopbackFileSystem) GetXAttr(name string, attr string) ([]byte, Status) {
	data, errNo := GetXAttr(me.GetPath(name), attr)

	return data, Status(errNo)
}

func (me *LoopbackFileSystem) FillOptions(options *PathFileSystemConnectorOptions) {
	options.NegativeTimeout = 3.0
	options.AttrTimeout = 3.0
	options.EntryTimeout = 3.0
}

////////////////////////////////////////////////////////////////

type LoopbackFile struct {
	file *os.File

	DefaultRawFuseFile
}

func (me *LoopbackFile) Read(input *ReadIn, buffers *BufferPool) ([]byte, Status) {
	slice := buffers.AllocBuffer(input.Size)

	n, err := me.file.ReadAt(slice, int64(input.Offset))
	if err == os.EOF {
		// TODO - how to signal EOF?
		return slice[:n], OK
	}
	return slice[:n], OsErrorToFuseError(err)
}

func (me *LoopbackFile) Write(input *WriteIn, data []byte) (uint32, Status) {
	n, err := me.file.WriteAt(data, int64(input.Offset))
	return uint32(n), OsErrorToFuseError(err)
}

func (me *LoopbackFile) Release() {
	me.file.Close()
}

func (me *LoopbackFile) Fsync(*FsyncIn) (code Status) {
	return Status(syscall.Fsync(me.file.Fd()))
}
<|endoftext|>


<commit_before>package qbit

import "sync"

var pools map[int]*sync.Pool = make(map[int]*sync.Pool)
var rw = sync.RWMutex{}

const poolOn = false

func readPool(batchSize int) (*sync.Pool, bool) {
	pool, ok := pools[batchSize]
	return pool, ok
}

func getPool(batchSize int) *sync.Pool {
	rw.Lock()
	defer rw.Unlock()

	pool, ok := readPool(batchSize)
	if ok {
		return pool
	} else {
		return createPool(batchSize)
	}
}

func createPool(batchSize int) *sync.Pool {
	pool := &sync.Pool{}
	pool.New = func() interface{} {
		return make([]interface{}, batchSize)
	}
	pools[batchSize] = pool
	return pool
}

func makeBuffer(batchSize int) []interface{} {
	if poolOn {
		return getPool(batchSize).Get().([]interface{})
	} else {
		return make([]interface{}, batchSize)
	}
}

func recycleBuffer(batchSize int, buffer []interface{}) {
	if poolOn {
		getPool(batchSize).Put(buffer)
	}
}
<commit_msg>turn pool back on<commit_after>package qbit

import "sync"

var pools map[int]*sync.Pool = make(map[int]*sync.Pool)
var rw = sync.RWMutex{}

const poolOn = true

func readPool(batchSize int) (*sync.Pool, bool) {
	pool, ok := pools[batchSize]
	return pool, ok
}

func getPool(batchSize int) *sync.Pool {
	rw.Lock()
	defer rw.Unlock()

	pool, ok := readPool(batchSize)
	if ok {
		return pool
	} else {
		return createPool(batchSize)
	}
}

func createPool(batchSize int) *sync.Pool {
	pool := &sync.Pool{}
	pool.New = func() interface{} {
		return make([]interface{}, batchSize)
	}
	pools[batchSize] = pool
	return pool
}

func makeBuffer(batchSize int) []interface{} {
	if poolOn {
		return getPool(batchSize).Get().([]interface{})
	} else {
		return make([]interface{}, batchSize)
	}
}

func recycleBuffer(batchSize int, buffer []interface{}) {
	if poolOn {
		getPool(batchSize).Put(buffer)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v2_0

import (
	"bytes"
	"errors"
	"reflect"

	"github.com/coreos/ignition/config/v2_0/types"
	"github.com/coreos/ignition/config/validate"
	astjson "github.com/coreos/ignition/config/validate/astjson"
	"github.com/coreos/ignition/config/validate/report"

	json "github.com/ajeddeloh/go-json"
	"go4.org/errorutil"
)

var (
	ErrCloudConfig = errors.New("not a config (found coreos-cloudconfig)")
	ErrEmpty       = errors.New("not a config (empty)")
	ErrScript      = errors.New("not a config (found coreos-cloudinit script)")
	ErrDeprecated  = errors.New("config format deprecated")
	ErrInvalid     = errors.New("config is not valid")
)

// Parse parses the raw config into a types.Config struct and generates a report of any
// errors, warnings, info, and deprecations it encountered
func Parse(rawConfig []byte) (types.Config, report.Report, error) {
	if isEmpty(rawConfig) {
		return types.Config{}, report.Report{}, ErrEmpty
	} else if isCloudConfig(rawConfig) {
		return types.Config{}, report.Report{}, ErrCloudConfig
	} else if isScript(rawConfig) {
		return types.Config{}, report.Report{}, ErrScript
	}

	var err error
	var config types.Config

	// These errors are fatal and the config should not be further validated
	if err = json.Unmarshal(rawConfig, &config); err == nil {
		versionReport := config.Ignition.Version.Validate()
		if versionReport.IsFatal() {
			return types.Config{}, versionReport, ErrInvalid
		}
	}

	// Handle json syntax and type errors first, since they are fatal but have offset info
	if serr, ok := err.(*json.SyntaxError); ok {
		line, col, highlight := errorutil.HighlightBytePosition(bytes.NewReader(rawConfig), serr.Offset)
		return types.Config{},
			report.Report{
				Entries: []report.Entry{{
					Kind:      report.EntryError,
					Message:   serr.Error(),
					Line:      line,
					Column:    col,
					Highlight: highlight,
				}},
			},
			ErrInvalid
	}

	if terr, ok := err.(*json.UnmarshalTypeError); ok {
		line, col, highlight := errorutil.HighlightBytePosition(bytes.NewReader(rawConfig), terr.Offset)
		return types.Config{},
			report.Report{
				Entries: []report.Entry{{
					Kind:      report.EntryError,
					Message:   terr.Error(),
					Line:      line,
					Column:    col,
					Highlight: highlight,
				}},
			},
			ErrInvalid
	}

	// Handle other fatal errors (i.e. invalid version)
	if err != nil {
		return types.Config{}, report.ReportFromError(err, report.EntryError), err
	}

	// Unmarshal again to a json.Node to get offset information for building a report
	var ast json.Node
	var r report.Report
	configValue := reflect.ValueOf(config)
	if err := json.Unmarshal(rawConfig, &ast); err != nil {
		r.Add(report.Entry{
			Kind:    report.EntryWarning,
			Message: "Ignition could not unmarshal your config for reporting line numbers. This should never happen. Please file a bug.",
		})
		r.Merge(validate.ValidateWithoutSource(configValue))
	} else {
		r.Merge(validate.Validate(configValue, astjson.FromJsonRoot(ast), bytes.NewReader(rawConfig), true))
	}

	if r.IsFatal() {
		return types.Config{}, r, ErrInvalid
	}

	return config, r, nil
}

func isEmpty(userdata []byte) bool {
	return len(userdata) == 0
}
<commit_msg>config/v2_0: changes to fallback to v1 parsing and translate on error<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v2_0

import (
	"errors"

	"github.com/coreos/ignition/config/util"
	"github.com/coreos/ignition/config/v1"
	"github.com/coreos/ignition/config/v2_0/types"
	"github.com/coreos/ignition/config/validate/report"

	json "github.com/ajeddeloh/go-json"
	"github.com/coreos/go-semver/semver"
)

var (
	ErrCloudConfig    = errors.New("not a config (found coreos-cloudconfig)")
	ErrEmpty          = errors.New("not a config (empty)")
	ErrScript         = errors.New("not a config (found coreos-cloudinit script)")
	ErrDeprecated     = errors.New("config format deprecated")
	ErrInvalid        = errors.New("config is not valid")
	ErrVersionUnknown = errors.New("unknown config version")
)

// Parse parses the raw config into a types.Config struct and generates a report of any
// errors, warnings, info, and deprecations it encountered
func Parse(rawConfig []byte) (types.Config, report.Report, error) {
	if isEmpty(rawConfig) {
		return types.Config{}, report.Report{}, ErrEmpty
	} else if isCloudConfig(rawConfig) {
		return types.Config{}, report.Report{}, ErrCloudConfig
	} else if isScript(rawConfig) {
		return types.Config{}, report.Report{}, ErrScript
	}

	var err error
	var config types.Config

	err = json.Unmarshal(rawConfig, &config)

	if err != nil || semver.Version(config.Ignition.Version).LessThan(types.MaxVersion) {
		// We can fail unmarshaling if it's an older config. Attempt to parse
		// it as such.
		config, rpt, err := v1.Parse(rawConfig)
		if err != nil {
			return types.Config{}, rpt, err
		}

		rpt.Merge(report.ReportFromError(ErrDeprecated, report.EntryDeprecated))
		return TranslateFromV1(config), rpt, err
	}

	if semver.Version(config.Ignition.Version) != types.MaxVersion {
		return types.Config{}, report.Report{}, ErrInvalid
	}

	rpt := util.ValidateConfig(rawConfig, config)
	if rpt.IsFatal() {
		return types.Config{}, rpt, ErrInvalid
	}

	return config, rpt, nil
}

func isEmpty(userdata []byte) bool {
	return len(userdata) == 0
}
<|endoftext|>


<commit_before>package vulcand

import (
	"context"
	"encoding/json"
	"fmt"
	"sync"
	"time"

	etcd "github.com/coreos/etcd/client"
	"github.com/mailgun/log"
)

const (
	localEtcdProxy = "http://127.0.0.1:2379"
	frontendFmt    = "%s/frontends/%s.%s/frontend"
	middlewareFmt  = "%s/frontends/%s.%s/middlewares/%s"
	backendFmt     = "%s/backends/%s/backend"
	serverFmt      = "%s/backends/%s/servers/%s"

	defaultRegistrationTTL = 30 * time.Second
)

type Config struct {
	Chroot string
	TTL    time.Duration
}

type Registry struct {
	cfg           Config
	backendSpec   *backendSpec
	frontendSpecs []*frontendSpec
	etcdKeysAPI   etcd.KeysAPI
	ctx           context.Context
	cancelFunc    context.CancelFunc
	wg            sync.WaitGroup
}

func NewRegistry(cfg Config, appname, ip string, port int) (*Registry, error) {
	backendSpec, err := newBackendSpec(appname, ip, port)
	if err != nil {
		return nil, fmt.Errorf("Failed to create backend: err=(%s)", err)
	}

	if cfg.TTL <= 0 {
		cfg.TTL = defaultRegistrationTTL
	}

	etcdCfg := etcd.Config{Endpoints: []string{localEtcdProxy}}
	etcdClt, err := etcd.New(etcdCfg)
	if err != nil {
		return nil, err
	}
	ctx, cancelFunc := context.WithCancel(context.Background())
	go func() {
		for {
			err := etcdClt.AutoSync(ctx, 10*time.Second)
			if err == context.DeadlineExceeded || err == context.Canceled {
				break
			}
			fmt.Print(err)
		}
	}()
	etcdKeysAPI := etcd.NewKeysAPI(etcdClt)
	c := Registry{
		cfg:         cfg,
		backendSpec: backendSpec,
		etcdKeysAPI: etcdKeysAPI,
		ctx:         ctx,
		cancelFunc:  cancelFunc,
	}
	return &c, nil
}

func (r *Registry) AddFrontend(host, path string, methods []string, middlewares []Middleware) {
	r.frontendSpecs = append(r.frontendSpecs, newFrontendSpec(r.backendSpec.AppName, host, path, methods, middlewares))
}

func (r *Registry) Start() error {
	if err := r.registerBackendType(r.backendSpec); err != nil {
		return fmt.Errorf("failed to register backend type: err=(%v)", err)
	}
	if err := r.registerBackendServer(r.backendSpec, r.cfg.TTL); err != nil {
		return fmt.Errorf("failed to register backend server: err=(%v)", err)
	}
	r.wg.Add(1)
	go r.heartbeat()

	for _, fes := range r.frontendSpecs {
		if err := r.registerFrontend(fes); err != nil {
			r.cancelFunc()
			return fmt.Errorf("failed to register frontend: err=(%v)", err)
		}
	}
	return nil
}

func (r *Registry) Stop() {
	r.cancelFunc()
	r.wg.Wait()
}

func (r *Registry) heartbeat() {
	defer r.wg.Done()
	tick := time.NewTicker(r.cfg.TTL * 3 / 4)
	for {
		select {
		case <-tick.C:
			if err := r.registerBackendServer(r.backendSpec, r.cfg.TTL); err != nil {
				log.Errorf("Heartbeat failed: err=(%v)", err)
			}
		case <-r.ctx.Done():
			err := r.removeBackendServer(r.backendSpec)
			log.Infof("Heartbeat stopped: err=(%v)", err)
			return
		}
	}
}

func (r *Registry) registerBackendType(bes *backendSpec) error {
	betKey := fmt.Sprintf(backendFmt, r.cfg.Chroot, bes.AppName)
	betVal := bes.typeSpec()
	_, err := r.etcdKeysAPI.Set(r.ctx, betKey, betVal, nil)
	return err
}

func (r *Registry) registerBackendServer(bes *backendSpec, ttl time.Duration) error {
	besKey := fmt.Sprintf(serverFmt, r.cfg.Chroot, bes.AppName, bes.ID)
	besVar := bes.serverSpec()
	_, err := r.etcdKeysAPI.Set(r.ctx, besKey, besVar, &etcd.SetOptions{TTL: ttl})
	return err
}

func (r *Registry) removeBackendServer(bes *backendSpec) error {
	besKey := fmt.Sprintf(serverFmt, r.cfg.Chroot, bes.AppName, bes.ID)
	_, err := r.etcdKeysAPI.Delete(context.Background(), besKey, nil)
	return err
}

func (r *Registry) registerFrontend(fes *frontendSpec) error {
	feKey := fmt.Sprintf(frontendFmt, r.cfg.Chroot, fes.Host, fes.ID)
	feVal := fes.spec()
	_, err := r.etcdKeysAPI.Set(r.ctx, feKey, feVal, nil)
	if err != nil {
		return err
	}
	for i, m := range fes.Middlewares {
		m.Priority = i
		mwKey := fmt.Sprintf(middlewareFmt, r.cfg.Chroot, fes.Host, fes.ID, m.ID)
		mwVal, err := json.Marshal(m)
		if err != nil {
			return err
		}
		_, err = r.etcdKeysAPI.Set(r.ctx, mwKey, string(mwVal), nil)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>NewRegistry() config can now accept optional etcd.Config{}<commit_after>package vulcand

import (
	"context"
	"encoding/json"
	"fmt"
	"sync"
	"time"

	etcd "github.com/coreos/etcd/client"
	"github.com/mailgun/log"
)

const (
	localEtcdProxy = "http://127.0.0.1:2379"
	frontendFmt    = "%s/frontends/%s.%s/frontend"
	middlewareFmt  = "%s/frontends/%s.%s/middlewares/%s"
	backendFmt     = "%s/backends/%s/backend"
	serverFmt      = "%s/backends/%s/servers/%s"

	defaultRegistrationTTL = 30 * time.Second
)

type Config struct {
	Etcd   *etcd.Config
	Chroot string
	TTL    time.Duration
}

type Registry struct {
	cfg           Config
	backendSpec   *backendSpec
	frontendSpecs []*frontendSpec
	etcdKeysAPI   etcd.KeysAPI
	ctx           context.Context
	cancelFunc    context.CancelFunc
	wg            sync.WaitGroup
}

func NewRegistry(cfg Config, appname, ip string, port int) (*Registry, error) {
	backendSpec, err := newBackendSpec(appname, ip, port)
	if err != nil {
		return nil, fmt.Errorf("Failed to create backend: err=(%s)", err)
	}

	if cfg.TTL <= 0 {
		cfg.TTL = defaultRegistrationTTL
	}

	etcdConfig := cfg.Etcd
	if etcdConfig == nil {
		etcdConfig = &etcd.Config{Endpoints: []string{localEtcdProxy}}
	}

	etcdClt, err := etcd.New(*etcdConfig)
	if err != nil {
		return nil, err
	}
	ctx, cancelFunc := context.WithCancel(context.Background())
	go func() {
		for {
			err := etcdClt.AutoSync(ctx, 10*time.Second)
			if err == context.DeadlineExceeded || err == context.Canceled {
				break
			}
			fmt.Print(err)
		}
	}()
	etcdKeysAPI := etcd.NewKeysAPI(etcdClt)
	c := Registry{
		cfg:         cfg,
		backendSpec: backendSpec,
		etcdKeysAPI: etcdKeysAPI,
		ctx:         ctx,
		cancelFunc:  cancelFunc,
	}
	return &c, nil
}

func (r *Registry) AddFrontend(host, path string, methods []string, middlewares []Middleware) {
	r.frontendSpecs = append(r.frontendSpecs, newFrontendSpec(r.backendSpec.AppName, host, path, methods, middlewares))
}

func (r *Registry) Start() error {
	if err := r.registerBackendType(r.backendSpec); err != nil {
		return fmt.Errorf("failed to register backend type: err=(%v)", err)
	}
	if err := r.registerBackendServer(r.backendSpec, r.cfg.TTL); err != nil {
		return fmt.Errorf("failed to register backend server: err=(%v)", err)
	}
	r.wg.Add(1)
	go r.heartbeat()

	for _, fes := range r.frontendSpecs {
		if err := r.registerFrontend(fes); err != nil {
			r.cancelFunc()
			return fmt.Errorf("failed to register frontend: err=(%v)", err)
		}
	}
	return nil
}

func (r *Registry) Stop() {
	r.cancelFunc()
	r.wg.Wait()
}

func (r *Registry) heartbeat() {
	defer r.wg.Done()
	tick := time.NewTicker(r.cfg.TTL * 3 / 4)
	for {
		select {
		case <-tick.C:
			if err := r.registerBackendServer(r.backendSpec, r.cfg.TTL); err != nil {
				log.Errorf("Heartbeat failed: err=(%v)", err)
			}
		case <-r.ctx.Done():
			err := r.removeBackendServer(r.backendSpec)
			log.Infof("Heartbeat stopped: err=(%v)", err)
			return
		}
	}
}

func (r *Registry) registerBackendType(bes *backendSpec) error {
	betKey := fmt.Sprintf(backendFmt, r.cfg.Chroot, bes.AppName)
	betVal := bes.typeSpec()
	_, err := r.etcdKeysAPI.Set(r.ctx, betKey, betVal, nil)
	return err
}

func (r *Registry) registerBackendServer(bes *backendSpec, ttl time.Duration) error {
	besKey := fmt.Sprintf(serverFmt, r.cfg.Chroot, bes.AppName, bes.ID)
	besVar := bes.serverSpec()
	_, err := r.etcdKeysAPI.Set(r.ctx, besKey, besVar, &etcd.SetOptions{TTL: ttl})
	return err
}

func (r *Registry) removeBackendServer(bes *backendSpec) error {
	besKey := fmt.Sprintf(serverFmt, r.cfg.Chroot, bes.AppName, bes.ID)
	_, err := r.etcdKeysAPI.Delete(context.Background(), besKey, nil)
	return err
}

func (r *Registry) registerFrontend(fes *frontendSpec) error {
	feKey := fmt.Sprintf(frontendFmt, r.cfg.Chroot, fes.Host, fes.ID)
	feVal := fes.spec()
	_, err := r.etcdKeysAPI.Set(r.ctx, feKey, feVal, nil)
	if err != nil {
		return err
	}
	for i, m := range fes.Middlewares {
		m.Priority = i
		mwKey := fmt.Sprintf(middlewareFmt, r.cfg.Chroot, fes.Host, fes.ID, m.ID)
		mwVal, err := json.Marshal(m)
		if err != nil {
			return err
		}
		_, err = r.etcdKeysAPI.Set(r.ctx, mwKey, string(mwVal), nil)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package cache

import (
	"time"
	"net/http"
	"net/http/httptest"
	"github.com/mholt/caddy/caddyhttp/httpserver"
	"github.com/pquerna/cachecontrol/cacheobject"
	"github.com/nicolasazrak/caddy-cache/storage"
)

type CacheHandler struct {
	Client storage.Storage
	Next   httpserver.Handler
}

func respond(response * storage.CachedResponse, w http.ResponseWriter) {
	for k, values := range response.HeaderMap {
		for _, v := range values {
			w.Header().Set(k, v)
		}
	}
	w.WriteHeader(response.Code)
	w.Write(response.Body)
}

func shouldUseCache(r *http.Request) bool {
	if r.Method != "GET" && r.Method != "HEAD" {
		// Only cache Get and head request
		return false
	}

	// Add more logic like get params, ?nocache=true
	return true
}

func getCacheableStatus(req *http.Request, res *httptest.ResponseRecorder) *cacheobject.ObjectResults {
	reqDir, _ := cacheobject.ParseRequestCacheControl(req.Header.Get("Cache-Control"))
	resDir, _ := cacheobject.ParseResponseCacheControl(res.Header().Get("Cache-Control"))
	expiresHeader, _ := http.ParseTime(res.Header().Get("Expires"))
	dateHeader, _ := http.ParseTime(res.Header().Get("Date"))
	lastModifiedHeader, _ := http.ParseTime(res.Header().Get("Last-Modified"))

	obj := cacheobject.Object{
		RespDirectives:         resDir,
		RespHeaders:            res.Header(),
		RespStatusCode:         res.Code,
		RespExpiresHeader:      expiresHeader,
		RespDateHeader:         dateHeader,
		RespLastModifiedHeader: lastModifiedHeader,

		ReqDirectives: reqDir,
		ReqHeaders:    req.Header,
		ReqMethod:     req.Method,

		NowUTC: time.Now().UTC(),
	}

	rv := cacheobject.ObjectResults{}
	cacheobject.CachableObject(&obj, &rv)
	cacheobject.ExpirationObject(&obj, &rv)
	return &rv
}

func isCacheable(rv *cacheobject.ObjectResults) bool {
	return len(rv.OutReasons) == 0 && rv.OutExpirationTime.Sub(time.Now().UTC()) > 0
}

func getKey(r *http.Request) string {
	return "contentcache:" + r.Host + ":" + r.Method + ":" + r.URL.Path
}

func (h CacheHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {

	if !shouldUseCache(r) {
		return h.Next.ServeHTTP(w, r)
	}

	cached, err := h.Client.Get(getKey(r))
	if err != nil {
		return http.StatusInternalServerError, err
	}

	if cached == nil {
		rec := httptest.NewRecorder()
		_, err := h.Next.ServeHTTP(rec, r)

		response := storage.CachedResponse {
			Body: rec.Body.Bytes(),
			HeaderMap: rec.HeaderMap,
			Code: rec.Code,
		}

		cacheableStatus := getCacheableStatus(r, rec)
		if isCacheable(cacheableStatus) {
			err = h.Client.Set(getKey(r), &response, cacheableStatus.OutExpirationTime)
			if err != nil {
				return http.StatusInternalServerError, err
			}
		}

		respond(&response, w)
		return response.Code, err
	} else {
		respond(cached, w)
		return cached.Code, nil
	}
}
<commit_msg>Improve isCacheable function<commit_after>package cache

import (
	"time"
	"net/http"
	"net/http/httptest"
	"github.com/mholt/caddy/caddyhttp/httpserver"
	"github.com/pquerna/cachecontrol/cacheobject"
	"github.com/nicolasazrak/caddy-cache/storage"
)

type CacheHandler struct {
	Client storage.Storage
	Next   httpserver.Handler
}

func respond(response * storage.CachedResponse, w http.ResponseWriter) {
	for k, values := range response.HeaderMap {
		for _, v := range values {
			w.Header().Set(k, v)
		}
	}
	w.WriteHeader(response.Code)
	w.Write(response.Body)
}

func shouldUseCache(r *http.Request) bool {
	if r.Method != "GET" && r.Method != "HEAD" {
		// Only cache Get and head request
		return false
	}

	// Add more logic like get params, ?nocache=true
	return true
}

func getCacheableStatus(req *http.Request, res *httptest.ResponseRecorder) (bool, time.Time) {
	reqDir, _ := cacheobject.ParseRequestCacheControl(req.Header.Get("Cache-Control"))
	resDir, _ := cacheobject.ParseResponseCacheControl(res.Header().Get("Cache-Control"))
	expiresHeader, _ := http.ParseTime(res.Header().Get("Expires"))
	dateHeader, _ := http.ParseTime(res.Header().Get("Date"))
	lastModifiedHeader, _ := http.ParseTime(res.Header().Get("Last-Modified"))

	obj := cacheobject.Object{
		RespDirectives:         resDir,
		RespHeaders:            res.Header(),
		RespStatusCode:         res.Code,
		RespExpiresHeader:      expiresHeader,
		RespDateHeader:         dateHeader,
		RespLastModifiedHeader: lastModifiedHeader,

		ReqDirectives: reqDir,
		ReqHeaders:    req.Header,
		ReqMethod:     req.Method,

		NowUTC: time.Now().UTC(),
	}

	rv := cacheobject.ObjectResults{}
	cacheobject.CachableObject(&obj, &rv)
	cacheobject.ExpirationObject(&obj, &rv)
	isCacheable := len(rv.OutReasons) == 0 && rv.OutExpirationTime.Sub(time.Now().UTC()) > 0
	return isCacheable, rv.OutExpirationTime
}

func getKey(r *http.Request) string {
	return "contentcache:" + r.Host + ":" + r.Method + ":" + r.URL.Path
}

func (h CacheHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {

	if !shouldUseCache(r) {
		return h.Next.ServeHTTP(w, r)
	}

	cached, err := h.Client.Get(getKey(r))
	if err != nil {
		return http.StatusInternalServerError, err
	}

	if cached == nil {
		rec := httptest.NewRecorder()
		_, err := h.Next.ServeHTTP(rec, r)

		response := storage.CachedResponse {
			Body: rec.Body.Bytes(),
			HeaderMap: rec.HeaderMap,
			Code: rec.Code,
		}

		isCacheable, expirationTime := getCacheableStatus(r, rec)
		if isCacheable {
			err = h.Client.Set(getKey(r), &response, expirationTime)
			if err != nil {
				return http.StatusInternalServerError, err
			}
		}

		respond(&response, w)
		return response.Code, err
	} else {
		respond(cached, w)
		return cached.Code, nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package vorbis provides Ogg/Vorbis decoder.
package vorbis

import (
	"io"
	"runtime"

	"github.com/hajimehoshi/ebiten/audio"
	"github.com/jfreymuth/oggvorbis"
)

type decoded struct {
	data       []float32
	totalBytes int
	readBytes  int
	posInBytes int
	source     io.Closer
	decoder    *oggvorbis.Reader
}

func (d *decoded) readUntil(posInBytes int) error {
	c := 0
	buffer := make([]float32, 8192)
	for d.readBytes < posInBytes {
		n, err := d.decoder.Read(buffer)
		if n > 0 {
			// Actual read bytes might exceed the total bytes.
			if d.readBytes+n*2 > d.totalBytes {
				n = (d.totalBytes - d.readBytes) / 2
			}
			p := d.readBytes / 2
			for i := 0; i < n; i++ {
				d.data[p+i] = buffer[i]
			}
			d.readBytes += n * 2
		}
		if err == io.EOF {
			if err := d.source.Close(); err != nil {
				return err
			}
			break
		}
		if err != nil {
			return err
		}
		c++
		if c%4 == 0 {
			runtime.Gosched()
		}
	}
	return nil
}

func (d *decoded) Read(b []uint8) (int, error) {
	l := d.totalBytes - d.posInBytes
	if l > len(b) {
		l = len(b)
	}
	if l < 0 {
		return 0, io.EOF
	}
	// l must be even so that d.posInBytes is always even.
	l = l / 2 * 2
	if err := d.readUntil(d.posInBytes + l); err != nil {
		return 0, err
	}
	for i := 0; i < l/2; i++ {
		f := d.data[d.posInBytes/2+i]
		s := int16(f * (1<<15 - 1))
		b[2*i] = uint8(s)
		b[2*i+1] = uint8(s >> 8)
	}
	d.posInBytes += l
	if d.posInBytes == d.totalBytes {
		return l, io.EOF
	}
	return l, nil
}

func (d *decoded) Seek(offset int64, whence int) (int64, error) {
	next := int64(0)
	switch whence {
	case io.SeekStart:
		next = offset
	case io.SeekCurrent:
		next = int64(d.posInBytes) + offset
	case io.SeekEnd:
		next = int64(d.totalBytes) + offset
	}
	// pos should be always even
	next = next / 2 * 2
	d.posInBytes = int(next)
	if err := d.readUntil(d.posInBytes); err != nil {
		return 0, err
	}
	return next, nil
}

func (d *decoded) Close() error {
	runtime.SetFinalizer(d, nil)
	return nil
}

func (d *decoded) Size() int64 {
	return int64(d.totalBytes)
}

// decode accepts an ogg stream and returns a decorded stream.
func decode(in audio.ReadSeekCloser) (*decoded, int, int, error) {
	// TODO: Lazy evaluation
	r, err := oggvorbis.NewReader(in)
	if err != nil {
		return nil, 0, 0, err
	}
	d := &decoded{
		data:       make([]float32, r.Length()*2),
		totalBytes: int(r.Length()) * 4, // TODO: What if length is 0?
		posInBytes: 0,
		source:     in,
		decoder:    r,
	}
	runtime.SetFinalizer(d, (*decoded).Close)
	if _, err := d.Read(make([]uint8, 65536)); err != nil {
		return nil, 0, 0, err
	}
	if _, err := d.Seek(0, io.SeekStart); err != nil {
		return nil, 0, 0, err
	}
	return d, r.Channels(), r.SampleRate(), nil
}
<commit_msg>audio/vorbis: Adjust Gosched timing (#297)<commit_after>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package vorbis provides Ogg/Vorbis decoder.
package vorbis

import (
	"io"
	"runtime"

	"github.com/hajimehoshi/ebiten/audio"
	"github.com/jfreymuth/oggvorbis"
)

type decoded struct {
	data       []float32
	totalBytes int
	readBytes  int
	posInBytes int
	source     io.Closer
	decoder    *oggvorbis.Reader
}

func (d *decoded) readUntil(posInBytes int) error {
	c := 0
	buffer := make([]float32, 8192)
	for d.readBytes < posInBytes {
		n, err := d.decoder.Read(buffer)
		if n > 0 {
			// Actual read bytes might exceed the total bytes.
			if d.readBytes+n*2 > d.totalBytes {
				n = (d.totalBytes - d.readBytes) / 2
			}
			p := d.readBytes / 2
			for i := 0; i < n; i++ {
				d.data[p+i] = buffer[i]
			}
			d.readBytes += n * 2
		}
		if err == io.EOF {
			if err := d.source.Close(); err != nil {
				return err
			}
			break
		}
		if err != nil {
			return err
		}
		c++
		if c%2 == 0 {
			runtime.Gosched()
		}
	}
	return nil
}

func (d *decoded) Read(b []uint8) (int, error) {
	l := d.totalBytes - d.posInBytes
	if l > len(b) {
		l = len(b)
	}
	if l < 0 {
		return 0, io.EOF
	}
	// l must be even so that d.posInBytes is always even.
	l = l / 2 * 2
	if err := d.readUntil(d.posInBytes + l); err != nil {
		return 0, err
	}
	for i := 0; i < l/2; i++ {
		f := d.data[d.posInBytes/2+i]
		s := int16(f * (1<<15 - 1))
		b[2*i] = uint8(s)
		b[2*i+1] = uint8(s >> 8)
	}
	d.posInBytes += l
	if d.posInBytes == d.totalBytes {
		return l, io.EOF
	}
	return l, nil
}

func (d *decoded) Seek(offset int64, whence int) (int64, error) {
	next := int64(0)
	switch whence {
	case io.SeekStart:
		next = offset
	case io.SeekCurrent:
		next = int64(d.posInBytes) + offset
	case io.SeekEnd:
		next = int64(d.totalBytes) + offset
	}
	// pos should be always even
	next = next / 2 * 2
	d.posInBytes = int(next)
	if err := d.readUntil(d.posInBytes); err != nil {
		return 0, err
	}
	return next, nil
}

func (d *decoded) Close() error {
	runtime.SetFinalizer(d, nil)
	return nil
}

func (d *decoded) Size() int64 {
	return int64(d.totalBytes)
}

// decode accepts an ogg stream and returns a decorded stream.
func decode(in audio.ReadSeekCloser) (*decoded, int, int, error) {
	// TODO: Lazy evaluation
	r, err := oggvorbis.NewReader(in)
	if err != nil {
		return nil, 0, 0, err
	}
	d := &decoded{
		data:       make([]float32, r.Length()*2),
		totalBytes: int(r.Length()) * 4, // TODO: What if length is 0?
		posInBytes: 0,
		source:     in,
		decoder:    r,
	}
	runtime.SetFinalizer(d, (*decoded).Close)
	if _, err := d.Read(make([]uint8, 65536)); err != nil {
		return nil, 0, 0, err
	}
	if _, err := d.Seek(0, io.SeekStart); err != nil {
		return nil, 0, 0, err
	}
	return d, r.Channels(), r.SampleRate(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/gogo/protobuf/proto"
	log "github.com/golang/glog"
	mesos "github.com/mesos/mesos-go/mesosproto"
	util "github.com/mesos/mesos-go/mesosutil"
	sched "github.com/mesos/mesos-go/scheduler"
)

type NoneScheduler struct {
	C             chan *Command
	nextCommand   *Command
	commands      map[string]*Command
	container     *mesos.ContainerInfo
	uris          []*mesos.CommandInfo_URI
	frameworkId   *mesos.FrameworkID
	cpuPerTask    float64
	memPerTask    float64
	tasksLaunched int
	tasksFinished int
	tasksFailed   int
	totalTasks    int
	running       bool
}

func NewNoneScheduler(container *mesos.ContainerInfo, uris []*mesos.CommandInfo_URI, cpus, mem float64) *NoneScheduler {
	return &NoneScheduler{
		C:             make(chan *Command, 10),
		nextCommand:   nil,
		commands:      make(map[string]*Command, 10),
		container:     container,
		uris:          uris,
		frameworkId:   nil,
		cpuPerTask:    cpus,
		memPerTask:    mem,
		tasksLaunched: 0,
		tasksFinished: 0,
		tasksFailed:   0,
		totalTasks:    0,
		running:       true,
	}
}

func (sched *NoneScheduler) HasFailures() bool {
	return sched.tasksFailed > 0
}

func (sched *NoneScheduler) Registered(driver sched.SchedulerDriver, frameworkId *mesos.FrameworkID, masterInfo *mesos.MasterInfo) {
	log.Infoln("Framework Registered with Master", masterInfo)
	sched.frameworkId = frameworkId
}

func (sched *NoneScheduler) Reregistered(driver sched.SchedulerDriver, masterInfo *mesos.MasterInfo) {}

func (sched *NoneScheduler) Disconnected(sched.SchedulerDriver) {
	log.Infoln("Framework Disconnected")
}

// process incoming offers and try to schedule new tasks as they come in on the channel
func (sched *NoneScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	for _, offer := range offers {
		cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "cpus"
		})
		cpus := 0.0
		for _, res := range cpuResources {
			cpus += res.GetScalar().GetValue()
		}

		memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "mem"
		})
		mems := 0.0
		for _, res := range memResources {
			mems += res.GetScalar().GetValue()
		}

		log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems)

		remainingCpus := cpus
		remainingMems := mems

		if sched.nextCommand == nil {
			sched.fetchNextCommand()
		}

		// try to schedule as may tasks as possible for this single offer
		var tasks []*mesos.TaskInfo
		for sched.nextCommand != nil &&
			sched.cpuPerTask <= remainingCpus &&
			sched.memPerTask <= remainingMems {

			sched.tasksLaunched++

			tId := strconv.Itoa(sched.tasksLaunched)
			sched.nextCommand.Id = tId
			taskId := &mesos.TaskID{
				Value: proto.String(tId),
			}

			task := &mesos.TaskInfo{
				Name:    proto.String("none-task-" + taskId.GetValue()),
				TaskId:  taskId,
				SlaveId: offer.SlaveId,
				Command: sched.prepareCommandInfo(sched.nextCommand),
				Resources: []*mesos.Resource{
					util.NewScalarResource("cpus", sched.cpuPerTask),
					util.NewScalarResource("mem", sched.memPerTask),
				},
				Container: sched.container,
			}
			log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

			tasks = append(tasks, task)
			sched.commands[tId] = sched.nextCommand

			remainingCpus -= sched.cpuPerTask
			remainingMems -= sched.memPerTask
			sched.fetchNextCommand()
		}
		log.Infoln("Launching", len(tasks), "tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
	}
}

func (sched *NoneScheduler) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) {
	log.Infoln("Status update: task", status.TaskId.GetValue(), "is in state", status.State.Enum().String())

	c := sched.commands[status.GetTaskId().GetValue()]
	if c == nil {
		log.Errorf("Unable to find command for task %s", status.GetTaskId().GetValue())
	} else {
		c.Status = status

		if status.GetState() == mesos.TaskState_TASK_RUNNING {
			c.StdoutPailer = sched.createAndStartPailer(driver, status, "cmd.stdout", os.Stdout)
			c.StderrPailer = sched.createAndStartPailer(driver, status, "cmd.stderr", os.Stderr)
		}
	}

	if status.GetState() == mesos.TaskState_TASK_FINISHED ||
		status.GetState() == mesos.TaskState_TASK_FAILED {
		sched.tasksFinished++

		if status.GetState() == mesos.TaskState_TASK_FAILED {
			sched.tasksFailed++
		}

		if c != nil {
			c.StopPailers()
		}
	}

	// stop if Commands channel was closed and all tasks have finished
	if !sched.running && sched.tasksFinished >= sched.totalTasks {
		log.Infoln("Total tasks completed, stopping framework.")
		for _, c := range sched.commands {
			c.WaitForPailers()
		}
		driver.Stop(false)
	}

	if status.GetState() == mesos.TaskState_TASK_LOST ||
		status.GetState() == mesos.TaskState_TASK_KILLED {
		sched.tasksFailed++
		log.Infoln(
			"Aborting because task", status.TaskId.GetValue(),
			"is in unexpected state", status.State.String(),
			"with message", status.GetMessage(),
		)
		driver.Abort()
	}
}

func (sched *NoneScheduler) OfferRescinded(driver sched.SchedulerDriver, offer *mesos.OfferID) {
	log.Infoln("Rescined offer", *offer)
}

func (sched *NoneScheduler) FrameworkMessage(driver sched.SchedulerDriver, exec *mesos.ExecutorID, slave *mesos.SlaveID, message string) {
	log.Infof("Framework message: %s", message)
}

func (sched *NoneScheduler) SlaveLost(driver sched.SchedulerDriver, offer *mesos.SlaveID) {
	log.Infoln("Lost slave", *offer)
}
func (sched *NoneScheduler) ExecutorLost(sched.SchedulerDriver, *mesos.ExecutorID, *mesos.SlaveID, int) {
}

func (sched *NoneScheduler) Error(driver sched.SchedulerDriver, err string) {
	log.Infoln("Scheduler received error:", err)
}

// private

func (sched *NoneScheduler) prepareCommandInfo(cmd *Command) *mesos.CommandInfo {
	value := "sh"
	shell := false
	var args []string

	if sched.container == nil {
		args = []string{"", "-c", fmt.Sprintf("( %s ) > cmd.stdout 2> cmd.stderr", sched.nextCommand.Cmd)}
	} else {
		args = []string{"-c", fmt.Sprintf("( %s ) > /${MESOS_SANDBOX}/cmd.stdout 2> /${MESOS_SANDBOX}/cmd.stderr", sched.nextCommand.Cmd)}
	}

	return &mesos.CommandInfo{
		Shell:     &shell,
		Value:     &value,
		Arguments: args,
		Uris:      sched.uris,
	}
}

func (sched *NoneScheduler) fetchNextCommand() {
	select {
	case sched.nextCommand = <-sched.C:
		if sched.nextCommand != nil {
			sched.totalTasks++
			log.Infoln("Schedule next command from queue:", strings.TrimSpace(sched.nextCommand.Cmd))
		} else {
			// channel was closed, stop listening for new commands
			sched.running = false
		}
	default:
		sched.nextCommand = nil
	}
}

func printer(f *os.File, out chan string) {
	for {
		f.WriteString(<-out)
	}
}

func (sched *NoneScheduler) createAndStartPailer(driver sched.SchedulerDriver, status *mesos.TaskStatus, file string, w *os.File) *Pailer {
	p, err := NewPailer(master, status.GetSlaveId(), sched.frameworkId, status.GetTaskId(), file)
	if err != nil {
		log.Errorf("Unable to start pailer for task %s: %s\n", status.GetTaskId().GetValue(), err)
		return nil
	} else {
		p.Start()
		go printer(w, p.C)
		return p
	}
}
<commit_msg>minor refactoring<commit_after>package main

import (
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/gogo/protobuf/proto"
	log "github.com/golang/glog"
	mesos "github.com/mesos/mesos-go/mesosproto"
	util "github.com/mesos/mesos-go/mesosutil"
	sched "github.com/mesos/mesos-go/scheduler"
)

type NoneScheduler struct {
	C             chan *Command
	nextCommand   *Command
	commands      map[string]*Command
	container     *mesos.ContainerInfo
	uris          []*mesos.CommandInfo_URI
	frameworkId   *mesos.FrameworkID
	cpuPerTask    float64
	memPerTask    float64
	tasksLaunched int
	tasksFinished int
	tasksFailed   int
	totalTasks    int
	running       bool
}

func NewNoneScheduler(container *mesos.ContainerInfo, uris []*mesos.CommandInfo_URI, cpus, mem float64) *NoneScheduler {
	return &NoneScheduler{
		C:             make(chan *Command, 10),
		nextCommand:   nil,
		commands:      make(map[string]*Command, 10),
		container:     container,
		uris:          uris,
		frameworkId:   nil,
		cpuPerTask:    cpus,
		memPerTask:    mem,
		tasksLaunched: 0,
		tasksFinished: 0,
		tasksFailed:   0,
		totalTasks:    0,
		running:       true,
	}
}

func (sched *NoneScheduler) HasFailures() bool {
	return sched.tasksFailed > 0
}

func (sched *NoneScheduler) Registered(driver sched.SchedulerDriver, frameworkId *mesos.FrameworkID, masterInfo *mesos.MasterInfo) {
	log.Infoln("Framework Registered with Master", masterInfo)
	sched.frameworkId = frameworkId
}

func (sched *NoneScheduler) Reregistered(driver sched.SchedulerDriver, masterInfo *mesos.MasterInfo) {}

func (sched *NoneScheduler) Disconnected(sched.SchedulerDriver) {
	log.Infoln("Framework Disconnected")
}

// process incoming offers and try to schedule new tasks as they come in on the channel
func (sched *NoneScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	for _, offer := range offers {
		cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "cpus"
		})
		cpus := 0.0
		for _, res := range cpuResources {
			cpus += res.GetScalar().GetValue()
		}

		memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "mem"
		})
		mems := 0.0
		for _, res := range memResources {
			mems += res.GetScalar().GetValue()
		}

		log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems)

		remainingCpus := cpus
		remainingMems := mems

		if sched.nextCommand == nil {
			sched.fetchNextCommand()
		}

		// try to schedule as may tasks as possible for this single offer
		var tasks []*mesos.TaskInfo
		for sched.nextCommand != nil &&
			sched.cpuPerTask <= remainingCpus &&
			sched.memPerTask <= remainingMems {

			task := sched.prepareTaskInfo(offer)
			tasks = append(tasks, task)

			remainingCpus -= sched.cpuPerTask
			remainingMems -= sched.memPerTask
			sched.fetchNextCommand()
		}
		log.Infoln("Launching", len(tasks), "tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(1)})
	}
}

func (sched *NoneScheduler) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) {
	log.Infoln("Status update: task", status.TaskId.GetValue(), "is in state", status.State.Enum().String())

	c := sched.commands[status.GetTaskId().GetValue()]
	if c == nil {
		log.Errorf("Unable to find command for task %s", status.GetTaskId().GetValue())
	} else {
		c.Status = status

		if status.GetState() == mesos.TaskState_TASK_RUNNING {
			c.StdoutPailer = sched.createAndStartPailer(driver, status, "cmd.stdout", os.Stdout)
			c.StderrPailer = sched.createAndStartPailer(driver, status, "cmd.stderr", os.Stderr)
		}
	}

	if status.GetState() == mesos.TaskState_TASK_FINISHED ||
		status.GetState() == mesos.TaskState_TASK_FAILED {
		sched.tasksFinished++

		if status.GetState() == mesos.TaskState_TASK_FAILED {
			sched.tasksFailed++
		}

		if c != nil {
			c.StopPailers()
		}
	}

	// stop if Commands channel was closed and all tasks have finished
	if !sched.running && sched.tasksFinished >= sched.totalTasks {
		log.Infoln("Total tasks completed, stopping framework.")
		for _, c := range sched.commands {
			c.WaitForPailers()
		}
		driver.Stop(false)
	}

	if status.GetState() == mesos.TaskState_TASK_LOST ||
		status.GetState() == mesos.TaskState_TASK_KILLED {
		sched.tasksFailed++
		log.Infoln(
			"Aborting because task", status.TaskId.GetValue(),
			"is in unexpected state", status.State.String(),
			"with message", status.GetMessage(),
		)
		driver.Abort()
	}
}

func (sched *NoneScheduler) OfferRescinded(driver sched.SchedulerDriver, offer *mesos.OfferID) {
	log.Infoln("Rescined offer", *offer)
}

func (sched *NoneScheduler) FrameworkMessage(driver sched.SchedulerDriver, exec *mesos.ExecutorID, slave *mesos.SlaveID, message string) {
	log.Infof("Framework message: %s", message)
}

func (sched *NoneScheduler) SlaveLost(driver sched.SchedulerDriver, offer *mesos.SlaveID) {
	log.Infoln("Lost slave", *offer)
}
func (sched *NoneScheduler) ExecutorLost(sched.SchedulerDriver, *mesos.ExecutorID, *mesos.SlaveID, int) {
}

func (sched *NoneScheduler) Error(driver sched.SchedulerDriver, err string) {
	log.Infoln("Scheduler received error:", err)
}

// private

func (sched *NoneScheduler) prepareTaskInfo(offer *mesos.Offer) *mesos.TaskInfo {
	tId := strconv.Itoa(sched.tasksLaunched)
	sched.tasksLaunched++
	sched.nextCommand.Id = tId
	sched.commands[tId] = sched.nextCommand

	taskId := &mesos.TaskID{
		Value: proto.String(tId),
	}

	task := &mesos.TaskInfo{
		Name:    proto.String("none-task-" + taskId.GetValue()),
		TaskId:  taskId,
		SlaveId: offer.SlaveId,
		Command: sched.prepareCommandInfo(sched.nextCommand),
		Resources: []*mesos.Resource{
			util.NewScalarResource("cpus", sched.cpuPerTask),
			util.NewScalarResource("mem", sched.memPerTask),
		},
		Container: sched.container,
	}
	log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

	return task
}

func (sched *NoneScheduler) prepareCommandInfo(cmd *Command) *mesos.CommandInfo {
	value := "sh"
	shell := false
	var args []string

	if sched.container == nil {
		args = []string{"", "-c", fmt.Sprintf("( %s ) > cmd.stdout 2> cmd.stderr", sched.nextCommand.Cmd)}
	} else {
		args = []string{"-c", fmt.Sprintf("( %s ) > /${MESOS_SANDBOX}/cmd.stdout 2> /${MESOS_SANDBOX}/cmd.stderr", sched.nextCommand.Cmd)}
	}

	return &mesos.CommandInfo{
		Shell:     &shell,
		Value:     &value,
		Arguments: args,
		Uris:      sched.uris,
	}
}

func (sched *NoneScheduler) fetchNextCommand() {
	select {
	case sched.nextCommand = <-sched.C:
		if sched.nextCommand != nil {
			sched.totalTasks++
			log.Infoln("Schedule next command from queue:", strings.TrimSpace(sched.nextCommand.Cmd))
		} else {
			// channel was closed, stop listening for new commands
			sched.running = false
		}
	default:
		sched.nextCommand = nil
	}
}

func printer(f *os.File, out chan string) {
	for {
		f.WriteString(<-out)
	}
}

func (sched *NoneScheduler) createAndStartPailer(driver sched.SchedulerDriver, status *mesos.TaskStatus, file string, w *os.File) *Pailer {
	p, err := NewPailer(master, status.GetSlaveId(), sched.frameworkId, status.GetTaskId(), file)
	if err != nil {
		log.Errorf("Unable to start pailer for task %s: %s\n", status.GetTaskId().GetValue(), err)
		return nil
	} else {
		p.Start()
		go printer(w, p.C)
		return p
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

type Cache struct {
	Root string
}

type CacheEntry struct {
	URL      *url.URL
	FilePath string
	sync.RWMutex
}

func (cache *Cache) UrlToFilePath(url *url.URL) string {
	s := []string{cache.Root, url.Host}
	s = append(s, strings.Split(url.Path, "/")...)
	return path.Join(s...)
}

func (cache *Cache) Keys() []*url.URL {
	keys := make([]*url.URL, 0)
	filepath.Walk(cache.Root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			warningf(cache, "Listing keys: %s", err)
			return err
		}

		if info.IsDir() {
			return nil
		}

		relPath, err := filepath.Rel(cache.Root, path)
		if err == nil {
			pathParts := filepath.SplitList(relPath)
			url := &url.URL{Scheme: "http", Host: pathParts[0], Path: strings.Join(pathParts[1:], "/")}
			keys = append(keys, url)
		}

		return nil
	})
	return keys
}

func (cache *Cache) GetEntry(url *url.URL) *CacheEntry {
	filePath := cache.UrlToFilePath(url)
	return &CacheEntry{URL: url, FilePath: filePath}
}

func (cacheEntry *CacheEntry) GetContent() ([]byte, time.Time, error) {
	cacheEntry.RLock()
	defer cacheEntry.RUnlock()

	fileInfo, err := os.Stat(cacheEntry.FilePath)
	if err != nil {
		return nil, time.Time{}, err
	}

	content, err := ioutil.ReadFile(cacheEntry.FilePath)
	if err != nil {
		return nil, time.Time{}, err
	}

	return content, fileInfo.ModTime(), nil
}

func (cacheEntry *CacheEntry) FreshenContent(content []byte, mtime time.Time) (bool, error) {
	cacheEntry.Lock()
	defer cacheEntry.Unlock()

	tracef(cacheEntry, "FreshenContent")

	fileInfo, _ := os.Stat(cacheEntry.FilePath)

	if fileInfo != nil && mtime.Before(fileInfo.ModTime()) {
		infof(cacheEntry, "FreshenContent: mtime is not fresher than cache entry: %s < %s", mtime, cacheEntry)
		return false, nil
	}

	dir, _ := path.Split(cacheEntry.FilePath)

	if err := os.MkdirAll(dir, 0777); err != nil {
		return false, err
	}

	debugf(cacheEntry, "Writing content")

	if err := ioutil.WriteFile(cacheEntry.FilePath, content, 0666); err != nil {
		return false, err
	}

	debugf(cacheEntry, "Setting mtime to %s", mtime)

	if err := os.Chtimes(cacheEntry.FilePath, mtime, mtime); err != nil {
		return false, err
	} else {
		return true, nil
	}
}

func (cacheEntry *CacheEntry) Delete() error {
	cacheEntry.Lock()
	defer cacheEntry.Unlock()
	return os.Remove(cacheEntry.FilePath)
}
<commit_msg>tracef<commit_after>package main

import (
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

type Cache struct {
	Root string
}

type CacheEntry struct {
	URL      *url.URL
	FilePath string
	sync.RWMutex
}

func (cache *Cache) UrlToFilePath(url *url.URL) string {
	s := []string{cache.Root, url.Host}
	s = append(s, strings.Split(url.Path, "/")...)
	return path.Join(s...)
}

func (cache *Cache) Keys() []*url.URL {
	keys := make([]*url.URL, 0)
	filepath.Walk(cache.Root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			warningf(cache, "Listing keys: %s", err)
			return err
		}

		if info.IsDir() {
			return nil
		}

		relPath, err := filepath.Rel(cache.Root, path)
		if err == nil {
			pathParts := filepath.SplitList(relPath)
			url := &url.URL{Scheme: "http", Host: pathParts[0], Path: strings.Join(pathParts[1:], "/")}
			keys = append(keys, url)
		}

		return nil
	})
	return keys
}

func (cache *Cache) GetEntry(url *url.URL) *CacheEntry {
	filePath := cache.UrlToFilePath(url)
	return &CacheEntry{URL: url, FilePath: filePath}
}

func (cacheEntry *CacheEntry) GetContent() ([]byte, time.Time, error) {
	cacheEntry.RLock()
	defer cacheEntry.RUnlock()

	fileInfo, err := os.Stat(cacheEntry.FilePath)
	if err != nil {
		return nil, time.Time{}, err
	}

	content, err := ioutil.ReadFile(cacheEntry.FilePath)
	if err != nil {
		return nil, time.Time{}, err
	}

	return content, fileInfo.ModTime(), nil
}

func (cacheEntry *CacheEntry) FreshenContent(content []byte, mtime time.Time) (bool, error) {
	cacheEntry.Lock()
	defer cacheEntry.Unlock()

	tracef(cacheEntry, "FreshenContent()")

	fileInfo, _ := os.Stat(cacheEntry.FilePath)

	if fileInfo != nil && mtime.Before(fileInfo.ModTime()) {
		infof(cacheEntry, "FreshenContent: mtime is not fresher than cache entry: %s < %s", mtime, cacheEntry)
		return false, nil
	}

	dir, _ := path.Split(cacheEntry.FilePath)

	if err := os.MkdirAll(dir, 0777); err != nil {
		return false, err
	}

	debugf(cacheEntry, "Writing content")

	if err := ioutil.WriteFile(cacheEntry.FilePath, content, 0666); err != nil {
		return false, err
	}

	debugf(cacheEntry, "Setting mtime to %s", mtime)

	if err := os.Chtimes(cacheEntry.FilePath, mtime, mtime); err != nil {
		return false, err
	} else {
		return true, nil
	}
}

func (cacheEntry *CacheEntry) Delete() error {
	cacheEntry.Lock()
	defer cacheEntry.Unlock()
	return os.Remove(cacheEntry.FilePath)
}
<|endoftext|>


<commit_before>package scheduler

import (
	"github.com/concourse/atc/builder"
	"github.com/concourse/atc/builds"
	"github.com/concourse/atc/config"
	"github.com/pivotal-golang/lager"
)

type SchedulerDB interface {
	CreateBuildWithInputs(job string, inputs builds.VersionedResources) (builds.Build, error)
	GetLatestInputVersions([]config.Input) (builds.VersionedResources, error)
	GetBuildForInputs(job string, inputs builds.VersionedResources) (builds.Build, error)

	GetNextPendingBuild(job string) (builds.Build, builds.VersionedResources, error)
}

type Scheduler struct {
	DB      SchedulerDB
	Builder builder.Builder
	Logger  lager.Logger
}

func (s *Scheduler) BuildLatestInputs(job config.Job) error {
	buildLog := s.Logger.Session("build-latest")

	inputs, err := s.DB.GetLatestInputVersions(job.Inputs)
	if err != nil {
		buildLog.Error("failed-to-get-latest-input-versions", err)
		return err
	}

	_, err = s.DB.GetBuildForInputs(job.Name, inputs)
	if err == nil {
		buildLog.Info("already-built")
		return nil
	}

	build, err := s.DB.CreateBuildWithInputs(job.Name, inputs)
	if err != nil {
		buildLog.Error("failed-to-create-build", err, lager.Data{
			"inputs": inputs,
		})
		return err
	}

	buildLog.Info("building", lager.Data{
		"build":  build,
		"inputs": inputs,
	})

	err = s.Builder.Build(build, job, inputs)
	if err != nil {
		buildLog.Error("failed-to-build", err)
		return err
	}

	return nil
}

func (s *Scheduler) TryNextPendingBuild(job config.Job) error {
	buildLog := s.Logger.Session("trigger-pending")

	build, inputs, err := s.DB.GetNextPendingBuild(job.Name)
	if err != nil {
		buildLog.Error("failed-to-get-next-pending-build", err)
		return err
	}

	err = s.Builder.Build(build, job, inputs)
	if err != nil {
		buildLog.Error("failed-to-build", err)
		return err
	}

	return nil
}
<commit_msg>remove typically noisy log lines<commit_after>package scheduler

import (
	"github.com/concourse/atc/builder"
	"github.com/concourse/atc/builds"
	"github.com/concourse/atc/config"
	"github.com/pivotal-golang/lager"
)

type SchedulerDB interface {
	CreateBuildWithInputs(job string, inputs builds.VersionedResources) (builds.Build, error)
	GetLatestInputVersions([]config.Input) (builds.VersionedResources, error)
	GetBuildForInputs(job string, inputs builds.VersionedResources) (builds.Build, error)

	GetNextPendingBuild(job string) (builds.Build, builds.VersionedResources, error)
}

type Scheduler struct {
	DB      SchedulerDB
	Builder builder.Builder
	Logger  lager.Logger
}

func (s *Scheduler) BuildLatestInputs(job config.Job) error {
	buildLog := s.Logger.Session("build-latest")

	inputs, err := s.DB.GetLatestInputVersions(job.Inputs)
	if err != nil {
		buildLog.Error("failed-to-get-latest-input-versions", err)
		return err
	}

	_, err = s.DB.GetBuildForInputs(job.Name, inputs)
	if err == nil {
		return nil
	}

	build, err := s.DB.CreateBuildWithInputs(job.Name, inputs)
	if err != nil {
		buildLog.Error("failed-to-create-build", err, lager.Data{
			"inputs": inputs,
		})
		return err
	}

	buildLog.Info("building", lager.Data{
		"build":  build,
		"inputs": inputs,
	})

	err = s.Builder.Build(build, job, inputs)
	if err != nil {
		buildLog.Error("failed-to-build", err)
		return err
	}

	return nil
}

func (s *Scheduler) TryNextPendingBuild(job config.Job) error {
	buildLog := s.Logger.Session("trigger-pending")

	build, inputs, err := s.DB.GetNextPendingBuild(job.Name)
	if err != nil {
		return err
	}

	err = s.Builder.Build(build, job, inputs)
	if err != nil {
		buildLog.Error("failed-to-build", err)
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"time"

	"github.com/square/keywhiz-fs/log"
)

// SecretBackend represents an interface for storing secrets.
type SecretBackend interface {
	Secret(string) (secret *Secret, err error)
	SecretList() (secretList []Secret, ok bool)
}

// Timeouts contains configuration for timeouts:
// timeout_backend_deadline: optimistic timeout to wait for cache
// timeout_max_wait: timeout for client to get data from server
type Timeouts struct {
	// FUSE may make many lookups in quick succession. If cached data is recent within the threshold,
	// a backend request is not attempted.
	Fresh time.Duration
	// BackendDeadline is distinct from the backend timeout. It is an optimistic timeout to wait
	// until resorting to cached data.
	BackendDeadline time.Duration
	MaxWait         time.Duration
	// Controls how long to keep a deleted entry before purging it.
	DeletionDelay time.Duration
}

// Cache contains necessary state to return secrets, using previously cached content or retrieving
// from a server if necessary.
type Cache struct {
	*log.Logger
	secretMap *SecretMap
	backend   SecretBackend
	timeouts  Timeouts
	now       func() time.Time
}

type secretResult struct {
	secret *Secret
	err    error
}

// NewCache initializes a Cache.
func NewCache(backend SecretBackend, timeouts Timeouts, logConfig log.Config, now func() time.Time) *Cache {
	logger := log.New("kwfs_cache", logConfig)
	return &Cache{logger, NewSecretMap(timeouts, now), backend, timeouts, now}
}

// Warmup reads the secret list from the backend to prime the cache.
// Should only be called after creating a new cache on startup.
func (c *Cache) Warmup() {
	// Attempt to warmup cache
	newMap := NewSecretMap(c.timeouts, c.now)
	secrets, ok := c.backend.SecretList()
	if ok {
		for _, backendSecret := range secrets {
			newMap.Put(backendSecret.Name, backendSecret)
		}
		c.secretMap.Overwrite(newMap)
	} else {
		c.Warnf("Failed to warmup cache on startup")
	}
}

// Clear empties the internal cache. This function does not honor the
// delayed deletion contract. The function is called when the user deletes
// .clear_cache.
func (c *Cache) Clear() {
	c.Infof("Cache cleared")
	c.secretMap = NewSecretMap(c.timeouts, c.now)
}

// Secret retrieves a Secret by name from cache or a server.
//
// Cache logic:
//  1. Check cache for secret.
//			* If entry is fresh, return cache entry.
//			* If entry is not fresh, call backend.
//  2. Ask backend for secret (with timeout).
//			* If backend returns success: update cache, return.
//			* If backend returns deleted: set delayed deletion, return data from cache.
//  3. If timeout backend deadline hit return whatever we have.
func (c *Cache) Secret(name string) (*Secret, bool) {
	// Perform cache lookup first
	cacheResult := c.cacheSecret(name)

	var secret *Secret
	var success bool

	if cacheResult != nil {
		secret = &cacheResult.Secret
		success = true
	}

	// If cache succeeded, and entry is very recent, return cache result
	if success && (time.Since(cacheResult.Time) < c.timeouts.Fresh) {
		return &cacheResult.Secret, success
	}

	backendDeadline := time.After(c.timeouts.BackendDeadline)
	backendDone := c.backendSecret(name)

	select {
	case s := <-backendDone:
		if s.err == nil {
			secret = s.secret
			success = true
		}
		if _, ok := s.err.(SecretDeleted); ok {
			c.secretMap.Delete(name)
		}
	case <-backendDeadline:
		c.Errorf("Backend timeout on secret fetch for '%s'", name)
	}

	return secret, success
}

// SecretList returns a listing of Secrets from cache or a server.
//
// Cache logic:
//  * If backend returns fast: update cache, return.
//  * If timeout backend deadline: return cache entries, background update cache.
//  * If timeout max wait: return cache version.
func (c *Cache) SecretList() []Secret {
	// Perform cache lookup first
	var secretList []Secret

	cacheResult := c.cacheSecretList()
	if cacheResult != nil {
		secretList = cacheResult
	}

	backendDeadline := time.After(c.timeouts.BackendDeadline)
	backendDone := c.backendSecretList()

	for {
		select {
		case backendResult := <-backendDone:
			return backendResult
		case <-backendDeadline:
			c.Errorf("Backend timeout for secret list")
			return secretList
		}
	}
}

// Add inserts a secret into the cache. If a secret is already in the cache with a matching
// identifier, it will be overridden  This method is most useful for testing since lookups
// may add data to the cache.
func (c *Cache) Add(s Secret) {
	c.secretMap.Put(s.Name, s)
}

// Len returns the number of values stored in the cache. This method is most useful for testing.
func (c *Cache) Len() int {
	return c.secretMap.Len()
}

// cacheSecret retrieves a secret from the cache.
func (c *Cache) cacheSecret(name string) *SecretTime {
	secret, ok := c.secretMap.Get(name)
	if ok && len(secret.Secret.Content) > 0 {
		c.Debugf("Cache hit: %v", name)
		return &secret
	}
	c.Debugf("Cache miss: %v", name)
	return nil
}

// cacheSecretList retrieves a secret listing from the cache.
func (c *Cache) cacheSecretList() []Secret {
	values := c.secretMap.Values()
	secrets := make([]Secret, len(values))
	for i, v := range values {
		secrets[i] = v.Secret
	}
	return secrets
}

// backendSecret retrieves a secret from the backend and updates the cache.
//
// Retrieval is concurrent, so a channel is returned to communicate a successful value.
// The channel will not be fulfilled on error.
func (c *Cache) backendSecret(name string) chan secretResult {
	secretc := make(chan secretResult)
	go func() {
		defer close(secretc)
		secret, err := c.backend.Secret(name)
		secretc <- secretResult{secret, err}
		if err == nil {
			c.secretMap.Put(name, *secret)
		}
	}()
	return secretc
}

// backendSecretList retrieves a secret listing from the backend and updates the cache.
//
// Retrieval is concurrent, so a channel is returned to communicate successful values. The channel
// will not be fulfilled on error.
func (c *Cache) backendSecretList() chan []Secret {
	secretsc := make(chan []Secret, 1)
	go func() {
		secrets, ok := c.backend.SecretList()
		if !ok {
			// Don't close the channel so that we use the result from the cache.
			return
		}

		newMap := NewSecretMap(c.timeouts, c.now)
		for _, backendSecret := range secrets {
			if len(backendSecret.Content) == 0 {
				// The backend didn't return any content. The cache might contain a secret with content, in
				// which case we want to keep the cache's value (and not schedule it for delayed deletion).
				if s, ok := c.secretMap.Get(backendSecret.Name); ok && len(s.Secret.Content) > 0 {
					newMap.Put(backendSecret.Name, s.Secret)
				} else {
					// We don't have content for this secret. TODO: explain under what circumstances this
					// can happen.
					newMap.Put(backendSecret.Name, backendSecret)
				}
			} else {
				// Cache the latest info.
				newMap.Put(backendSecret.Name, backendSecret)
			}
		}
		c.secretMap.Replace(newMap)

		// TODO: copy-pasta from cacheSecretList(), should refactor.
		values := c.secretMap.Values()
		secrets = make([]Secret, len(values))
		for i, v := range values {
			secrets[i] = v.Secret
		}
		secretsc <- secrets
		close(secretsc)
	}()
	return secretsc
}
<commit_msg>Clarify with some comments<commit_after>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"time"

	"github.com/square/keywhiz-fs/log"
)

// SecretBackend represents an interface for storing secrets.
type SecretBackend interface {
	Secret(string) (secret *Secret, err error)
	SecretList() (secretList []Secret, ok bool)
}

// Timeouts contains configuration for timeouts:
// timeout_backend_deadline: optimistic timeout to wait for cache
// timeout_max_wait: timeout for client to get data from server
type Timeouts struct {
	// FUSE may make many lookups in quick succession. If cached data is recent within the threshold,
	// a backend request is not attempted.
	Fresh time.Duration
	// BackendDeadline is distinct from the backend timeout. It is an optimistic timeout to wait
	// until resorting to cached data.
	BackendDeadline time.Duration
	MaxWait         time.Duration
	// Controls how long to keep a deleted entry before purging it.
	DeletionDelay time.Duration
}

// Cache contains necessary state to return secrets, using previously cached content or retrieving
// from a server if necessary.
type Cache struct {
	*log.Logger
	secretMap *SecretMap
	backend   SecretBackend
	timeouts  Timeouts
	now       func() time.Time
}

type secretResult struct {
	secret *Secret
	err    error
}

// NewCache initializes a Cache.
func NewCache(backend SecretBackend, timeouts Timeouts, logConfig log.Config, now func() time.Time) *Cache {
	logger := log.New("kwfs_cache", logConfig)
	return &Cache{logger, NewSecretMap(timeouts, now), backend, timeouts, now}
}

// Warmup reads the secret list from the backend to prime the cache.
// Should only be called after creating a new cache on startup.
func (c *Cache) Warmup() {
	// Attempt to warmup cache
	newMap := NewSecretMap(c.timeouts, c.now)
	secrets, ok := c.backend.SecretList()
	if ok {
		for _, backendSecret := range secrets {
			newMap.Put(backendSecret.Name, backendSecret)
		}
		c.secretMap.Overwrite(newMap)
	} else {
		c.Warnf("Failed to warmup cache on startup")
	}
}

// Clear empties the internal cache. This function does not honor the
// delayed deletion contract. The function is called when the user deletes
// .clear_cache.
func (c *Cache) Clear() {
	c.Infof("Cache cleared")
	c.secretMap = NewSecretMap(c.timeouts, c.now)
}

// Secret retrieves a Secret by name from cache or a server.
//
// Cache logic:
//  1. Check cache for secret.
//			* If entry is fresh, return cache entry.
//			* If entry is not fresh, call backend.
//  2. Ask backend for secret (with timeout).
//			* If backend returns success: update cache, return.
//			* If backend returns deleted: set delayed deletion, return data from cache.
//  3. If timeout backend deadline hit return whatever we have.
func (c *Cache) Secret(name string) (*Secret, bool) {
	// Perform cache lookup first
	cacheResult := c.cacheSecret(name)

	var secret *Secret
	var success bool

	if cacheResult != nil {
		secret = &cacheResult.Secret
		success = true
	}

	// If cache succeeded, and entry is very recent, return cache result
	if success && (time.Since(cacheResult.Time) < c.timeouts.Fresh) {
		return &cacheResult.Secret, success
	}

	backendDeadline := time.After(c.timeouts.BackendDeadline)
	backendDone := c.backendSecret(name)

	select {
	case s := <-backendDone:
		if s.err == nil {
			secret = s.secret
			success = true
		}
		if _, ok := s.err.(SecretDeleted); ok {
			c.secretMap.Delete(name)
		}
	case <-backendDeadline:
		c.Errorf("Backend timeout on secret fetch for '%s'", name)
	}

	return secret, success
}

// SecretList returns a listing of Secrets from cache or a server.
//
// Cache logic:
//  * If backend returns fast: update cache, return.
//  * If timeout backend deadline: return cache entries, background update cache.
//  * If timeout max wait: return cache version.
func (c *Cache) SecretList() []Secret {
	// Perform cache lookup first
	var secretList []Secret

	cacheResult := c.cacheSecretList()
	if cacheResult != nil {
		secretList = cacheResult
	}

	backendDeadline := time.After(c.timeouts.BackendDeadline)
	backendDone := c.backendSecretList()

	for {
		select {
		case backendResult := <-backendDone:
			return backendResult
		case <-backendDeadline:
			c.Errorf("Backend timeout for secret list")
			return secretList
		}
	}
}

// Add inserts a secret into the cache. If a secret is already in the cache with a matching
// identifier, it will be overridden  This method is most useful for testing since lookups
// may add data to the cache.
func (c *Cache) Add(s Secret) {
	c.secretMap.Put(s.Name, s)
}

// Len returns the number of values stored in the cache. This method is most useful for testing.
func (c *Cache) Len() int {
	return c.secretMap.Len()
}

// cacheSecret retrieves a secret from the cache.
func (c *Cache) cacheSecret(name string) *SecretTime {
	secret, ok := c.secretMap.Get(name)
	if ok && len(secret.Secret.Content) > 0 {
		c.Debugf("Cache hit: %v", name)
		return &secret
	}
	c.Debugf("Cache miss: %v", name)
	return nil
}

// cacheSecretList retrieves a secret listing from the cache.
func (c *Cache) cacheSecretList() []Secret {
	values := c.secretMap.Values()
	secrets := make([]Secret, len(values))
	for i, v := range values {
		secrets[i] = v.Secret
	}
	return secrets
}

// backendSecret retrieves a secret from the backend and updates the cache.
//
// Retrieval is concurrent, so a channel is returned to communicate a successful value.
// The channel will not be fulfilled on error.
func (c *Cache) backendSecret(name string) chan secretResult {
	secretc := make(chan secretResult)
	go func() {
		defer close(secretc)
		secret, err := c.backend.Secret(name)
		secretc <- secretResult{secret, err}
		if err == nil {
			c.secretMap.Put(name, *secret)
		}
	}()
	return secretc
}

// backendSecretList retrieves a secret listing from the backend and updates the cache.
//
// Retrieval is concurrent, so a channel is returned to communicate successful values. The channel
// will not be fulfilled on error.
func (c *Cache) backendSecretList() chan []Secret {
	secretsc := make(chan []Secret, 1)
	go func() {
		secrets, ok := c.backend.SecretList()
		if !ok {
			// Don't close the channel so that we use the result from the cache.
			return
		}

		newMap := NewSecretMap(c.timeouts, c.now)
		for _, backendSecret := range secrets {
			if len(backendSecret.Content) == 0 {
				// The backend didn't return any content. The cache might contain a secret with content, in
				// which case we want to keep the cache's value (and not schedule it for delayed deletion).
				if s, ok := c.secretMap.Get(backendSecret.Name); ok && len(s.Secret.Content) > 0 {
					newMap.Put(backendSecret.Name, s.Secret)
				} else {
					// We don't have content for this secret. This happens when the cache has never seen a given secret
					// (at startup or when a new secret is added).
					// can happen.
					newMap.Put(backendSecret.Name, backendSecret)
				}
			} else {
				// TODO: explain why this case can happen. It doesn't seem like it can,
				// listing secrets always returns just the names.
				// Cache the latest info.
				newMap.Put(backendSecret.Name, backendSecret)
			}
		}
		c.secretMap.Replace(newMap)

		// TODO: copy-pasta from cacheSecretList(), should refactor.
		values := c.secretMap.Values()
		secrets = make([]Secret, len(values))
		for i, v := range values {
			secrets[i] = v.Secret
		}
		secretsc <- secrets
		close(secretsc)
	}()
	return secretsc
}
<|endoftext|>