<commit_before>package server

import (
	"fmt"
	"sort"
	"time"

	"github.com/dancannon/gorethink"
	"github.com/golang/protobuf/jsonpb"
	"github.com/golang/protobuf/proto"
	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/persist"
	"go.pedge.io/google-protobuf"
	"go.pedge.io/pkg/time"
	"go.pedge.io/proto/rpclog"
	"go.pedge.io/proto/time"
	"golang.org/x/net/context"
)

const (
	jobInfosTable      Table = "job_infos"
	pipelineInfosTable Table = "pipeline_infos"

	pipelineNameIndex         Index = "pipeline_name"
	pipelineNameAndInputIndex Index = "pipeline_name_and_input"
	inputIndex                Index = "input"
)

type Table string
type PrimaryKey string
type Index string

var (
	marshaller = &jsonpb.Marshaler{}

	tables = []Table{
		jobInfosTable,
		pipelineInfosTable,
	}

	tableToTableCreateOpts = map[Table][]gorethink.TableCreateOpts{
		jobInfosTable: []gorethink.TableCreateOpts{
			gorethink.TableCreateOpts{
				PrimaryKey: "JobId",
			},
		},
		pipelineInfosTable: []gorethink.TableCreateOpts{
			gorethink.TableCreateOpts{
				PrimaryKey: "PipelineName",
			},
		},
	}
)

// InitDBs prepares a RethinkDB instance to be used by the rethink server.
// Rethink servers will error if they are pointed at databases that haven't had InitDBs run on them.
// InitDBs is idempotent (unless rethink dies in the middle of the function)
func InitDBs(address string, databaseName string) error {
	session, err := gorethink.Connect(gorethink.ConnectOpts{Address: address})
	if err != nil {
		return err
	}
	if _, err := gorethink.DBCreate(databaseName).RunWrite(session); err != nil {
		if _, ok := err.(gorethink.RQLRuntimeError); ok {
			return nil
		}
		return err
	}
	for _, table := range tables {
		tableCreateOpts, ok := tableToTableCreateOpts[table]
		if ok {
			if _, err := gorethink.DB(databaseName).TableCreate(table, tableCreateOpts...).RunWrite(session); err != nil {
				return err
			}
		} else {
			if _, err := gorethink.DB(databaseName).TableCreate(table).RunWrite(session); err != nil {
				return err
			}
		}
	}

	// Create some indexes for the jobInfosTable
	if _, err := gorethink.DB(databaseName).Table(jobInfosTable).IndexCreate(pipelineNameIndex).RunWrite(session); err != nil {
		return err
	}
	if _, err := gorethink.DB(databaseName).Table(jobInfosTable).IndexCreateFunc(
		pipelineNameAndInputIndex,
		func(row gorethink.Term) interface{} {
			return []interface{}{
				row.Field("PipelineName"),
				row.Field("InputIndex"),
			}
		}).RunWrite(session); err != nil {
		return err
	}
	if _, err := gorethink.DB(databaseName).Table(jobInfosTable).IndexCreateFunc(
		inputIndex,
		func(row gorethink.Term) interface{} {
			return row.Field("InputIndex")
		}).RunWrite(session); err != nil {
		return err
	}
	return nil
}

type rethinkAPIServer struct {
	protorpclog.Logger
	session      *gorethink.Session
	databaseName string
	timer        pkgtime.Timer
}

func newRethinkAPIServer(address string, databaseName string) (*rethinkAPIServer, error) {
	session, err := gorethink.Connect(gorethink.ConnectOpts{Address: address})
	if err != nil {
		return nil, err
	}
	return &rethinkAPIServer{
		protorpclog.NewLogger("pachyderm.pps.persist.API"),
		session,
		databaseName,
		pkgtime.NewSystemTimer(),
	}, nil
}

func (a *rethinkAPIServer) Close() error {
	return a.session.Close()
}

// JobId cannot be set
// Timestamp cannot be set
func (a *rethinkAPIServer) CreateJobInfo(ctx context.Context, request *persist.JobInfo) (response *persist.JobInfo, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if request.JobId != "" {
		return nil, fmt.Errorf("request.JobId should be unset")
	}
	if request.CreatedAt != nil {
		return nil, fmt.Errorf("request.CreatedAt should be unset")
	}
	if request.CommitIndex != "" {
		return nil, fmt.Errorf("request.CommitIndex should be unset")
	}
	request.JobId = uuid.NewWithoutDashes()
	request.CreatedAt = prototime.TimeToTimestamp(time.Now())
	request.CommitIndex = commitIndex(request.InputCommit)
	if err := a.insertMessage(jobInfosTable, request); err != nil {
		return nil, err
	}
	return request, nil
}

func (a *rethinkAPIServer) InspectJob(ctx context.Context, request *pps.InspectJobRequest) (response *persist.JobInfo, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	jobInfo := &persist.JobInfo{}
	var mustHaveFields []interface{}
	if request.BlockOutput {
		mustHaveFields = append(mustHaveFields, "OutputCommit")
	}
	if request.BlockState {
		mustHaveFields = append(mustHaveFields, "State")
	}
	if err := a.waitMessageByPrimaryKey(
		jobInfosTable,
		request.Job.Id,
		jobInfo,
		func(jobInfo gorethink.Term) gorethink.Term {
			return jobInfo.HasFields(mustHaveFields...)
		},
	); err != nil {
		return nil, err
	}
	return jobInfo, nil
}

func (a *rethinkAPIServer) ListJobInfos(ctx context.Context, request *pps.ListJobRequest) (response *persist.JobInfos, retErr error) {
	defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
	query := a.getTerm(jobInfosTable)
	if request.Pipeline != nil && len(request.InputCommit) > 0 {
		query = query.GetAllByIndex(
			pipelineNameAndInputIndex,
			gorethink.Expr([]interface{}{request.Pipeline.Name, commitIndex(request.InputCommit)}),
		)
	} else if request.Pipeline != nil {
		query = query.GetAllByIndex(
			pipelineNameIndex,
			request.Pipeline.Name,
		)
	} else if len(request.InputCommit) > 0 {
		query = query.GetAllByIndex(
			inputIndex,
			gorethink.Expr(commitIndex(request.InputCommit)),
		)
	}
	cursor, err := query.Run(a.session)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := cursor.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	result := &persist.JobInfos{}
	for {
		jobInfo := &persist.JobInfo{}
		if !cursor.Next(jobInfo) {
			break
		}
		result.JobInfo = append(result.JobInfo, jobInfo)
	}
	if err := cursor.Err(); err != nil {
		return nil, err
	}
	return result, nil
}

func (a *rethinkAPIServer) DeleteJobInfo(ctx context.Context, request *pps.Job) (response *google_protobuf.Empty, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if err := a.deleteMessageByPrimaryKey(jobInfosTable, request.Id); err != nil {
		return nil, err
	}
	return google_protobuf.EmptyInstance, nil
}

func (a *rethinkAPIServer) CreateJobOutput(ctx context.Context, request *persist.JobOutput) (response *google_protobuf.Empty, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if err := a.updateMessage(jobInfosTable, request); err != nil {
		return nil, err
	}
	return google_protobuf.EmptyInstance, nil
}

func (a *rethinkAPIServer) CreateJobState(ctx context.Context, request *persist.JobState) (response *google_protobuf.Empty, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if err := a.updateMessage(jobInfosTable, request); err != nil {
		return nil, err
	}
	return google_protobuf.EmptyInstance, nil
}

// timestamp cannot be set
func (a *rethinkAPIServer) CreatePipelineInfo(ctx context.Context, request *persist.PipelineInfo) (response *persist.PipelineInfo, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if request.CreatedAt != nil {
		return nil, ErrTimestampSet
	}
	request.CreatedAt = a.now()
	if err := a.insertMessage(pipelineInfosTable, request); err != nil {
		return nil, err
	}
	return request, nil
}

func (a *rethinkAPIServer) GetPipelineInfo(ctx context.Context, request *pps.Pipeline) (response *persist.PipelineInfo, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	pipelineInfo := &persist.PipelineInfo{}
	if err := a.getMessageByPrimaryKey(pipelineInfosTable, request.Name, pipelineInfo); err != nil {
		return nil, err
	}
	return pipelineInfo, nil
}

func (a *rethinkAPIServer) ListPipelineInfos(ctx context.Context, request *google_protobuf.Empty) (response *persist.PipelineInfos, retErr error) {
	defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
	query := a.getTerm(pipelineInfosTable)
	cursor, err := query.Run(a.session)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := cursor.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	result := &persist.PipelineInfos{}
	for {
		pipelineInfo := &persist.PipelineInfo{}
		if !cursor.Next(pipelineInfo) {
			break
		}
		result.PipelineInfo = append(result.PipelineInfo, pipelineInfo)
	}
	if err := cursor.Err(); err != nil {
		return nil, err
	}
	return result, nil
}

func (a *rethinkAPIServer) DeletePipelineInfo(ctx context.Context, request *pps.Pipeline) (response *google_protobuf.Empty, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if err := a.deleteMessageByPrimaryKey(pipelineInfosTable, request.Name); err != nil {
		return nil, err
	}
	return google_protobuf.EmptyInstance, nil
}

func (a *rethinkAPIServer) insertMessage(table Table, message proto.Message) error {
	_, err := a.getTerm(table).Insert(message).RunWrite(a.session)
	return err
}

func (a *rethinkAPIServer) updateMessage(table Table, message proto.Message) error {
	_, err := a.getTerm(table).Update(message).RunWrite(a.session)
	return err
}

func (a *rethinkAPIServer) getMessageByPrimaryKey(table Table, key interface{}, message proto.Message) error {
	cursor, err := a.getTerm(table).Get(key).Default(gorethink.Error("value not found")).Run(a.session)
	if err != nil {
		return err
	}
	if cursor.Next(message) {
		return cursor.Err()
	}
	return nil
}

func (a *rethinkAPIServer) deleteMessageByPrimaryKey(table Table, value interface{}) error {
	_, err := a.getTerm(table).Get(value).Delete().RunWrite(a.session)
	return err
}

func (a *rethinkAPIServer) waitMessageByPrimaryKey(
	table Table,
	key interface{},
	message proto.Message,
	predicate func(term gorethink.Term) gorethink.Term,
) error {
	cursor, err :=
		a.getTerm(table).
			Get(key).
			Default(gorethink.Error("value not found")).
			Changes().
			Field("new_val").
			Filter(predicate).
			Run(a.session)
	if err != nil {
		return err
	}
	return cursor.One(message)
}

func (a *rethinkAPIServer) getTerm(table Table) gorethink.Term {
	return gorethink.DB(a.databaseName).Table(table)
}

func (a *rethinkAPIServer) now() *google_protobuf.Timestamp {
	return prototime.TimeToTimestamp(a.timer.Now())
}

func commitIndex(commits []*pfs.Commit) string {
	var commitIDs []string
	for _, commit := range commits {
		commitIDs = append(commitIDs, commit.Id[0:10])
	}
	sort.Strings(commitIDs)
	var result []byte
	for _, commitID := range commitIDs {
		result = append(result, commitID...)
	}
	return string(result)
}
<commit_msg>Fixes tests. One doesn't block like Next???<commit_after>package server

import (
	"fmt"
	"sort"
	"time"

	"github.com/dancannon/gorethink"
	"github.com/golang/protobuf/jsonpb"
	"github.com/golang/protobuf/proto"
	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/persist"
	"go.pedge.io/google-protobuf"
	"go.pedge.io/pkg/time"
	"go.pedge.io/proto/rpclog"
	"go.pedge.io/proto/time"
	"golang.org/x/net/context"
)

const (
	jobInfosTable      Table = "job_infos"
	pipelineInfosTable Table = "pipeline_infos"

	pipelineNameIndex         Index = "pipeline_name"
	pipelineNameAndInputIndex Index = "pipeline_name_and_input"
	inputIndex                Index = "input"
)

type Table string
type PrimaryKey string
type Index string

var (
	marshaller = &jsonpb.Marshaler{}

	tables = []Table{
		jobInfosTable,
		pipelineInfosTable,
	}

	tableToTableCreateOpts = map[Table][]gorethink.TableCreateOpts{
		jobInfosTable: []gorethink.TableCreateOpts{
			gorethink.TableCreateOpts{
				PrimaryKey: "JobId",
			},
		},
		pipelineInfosTable: []gorethink.TableCreateOpts{
			gorethink.TableCreateOpts{
				PrimaryKey: "PipelineName",
			},
		},
	}
)

// InitDBs prepares a RethinkDB instance to be used by the rethink server.
// Rethink servers will error if they are pointed at databases that haven't had InitDBs run on them.
// InitDBs is idempotent (unless rethink dies in the middle of the function)
func InitDBs(address string, databaseName string) error {
	session, err := gorethink.Connect(gorethink.ConnectOpts{Address: address})
	if err != nil {
		return err
	}
	if _, err := gorethink.DBCreate(databaseName).RunWrite(session); err != nil {
		if _, ok := err.(gorethink.RQLRuntimeError); ok {
			return nil
		}
		return err
	}
	for _, table := range tables {
		tableCreateOpts, ok := tableToTableCreateOpts[table]
		if ok {
			if _, err := gorethink.DB(databaseName).TableCreate(table, tableCreateOpts...).RunWrite(session); err != nil {
				return err
			}
		} else {
			if _, err := gorethink.DB(databaseName).TableCreate(table).RunWrite(session); err != nil {
				return err
			}
		}
	}

	// Create some indexes for the jobInfosTable
	if _, err := gorethink.DB(databaseName).Table(jobInfosTable).IndexCreate(pipelineNameIndex).RunWrite(session); err != nil {
		return err
	}
	if _, err := gorethink.DB(databaseName).Table(jobInfosTable).IndexCreateFunc(
		pipelineNameAndInputIndex,
		func(row gorethink.Term) interface{} {
			return []interface{}{
				row.Field("PipelineName"),
				row.Field("InputIndex"),
			}
		}).RunWrite(session); err != nil {
		return err
	}
	if _, err := gorethink.DB(databaseName).Table(jobInfosTable).IndexCreateFunc(
		inputIndex,
		func(row gorethink.Term) interface{} {
			return row.Field("InputIndex")
		}).RunWrite(session); err != nil {
		return err
	}
	return nil
}

type rethinkAPIServer struct {
	protorpclog.Logger
	session      *gorethink.Session
	databaseName string
	timer        pkgtime.Timer
}

func newRethinkAPIServer(address string, databaseName string) (*rethinkAPIServer, error) {
	session, err := gorethink.Connect(gorethink.ConnectOpts{Address: address})
	if err != nil {
		return nil, err
	}
	return &rethinkAPIServer{
		protorpclog.NewLogger("pachyderm.pps.persist.API"),
		session,
		databaseName,
		pkgtime.NewSystemTimer(),
	}, nil
}

func (a *rethinkAPIServer) Close() error {
	return a.session.Close()
}

// JobId cannot be set
// Timestamp cannot be set
func (a *rethinkAPIServer) CreateJobInfo(ctx context.Context, request *persist.JobInfo) (response *persist.JobInfo, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if request.JobId != "" {
		return nil, fmt.Errorf("request.JobId should be unset")
	}
	if request.CreatedAt != nil {
		return nil, fmt.Errorf("request.CreatedAt should be unset")
	}
	if request.CommitIndex != "" {
		return nil, fmt.Errorf("request.CommitIndex should be unset")
	}
	request.JobId = uuid.NewWithoutDashes()
	request.CreatedAt = prototime.TimeToTimestamp(time.Now())
	request.CommitIndex = commitIndex(request.InputCommit)
	if err := a.insertMessage(jobInfosTable, request); err != nil {
		return nil, err
	}
	return request, nil
}

func (a *rethinkAPIServer) InspectJob(ctx context.Context, request *pps.InspectJobRequest) (response *persist.JobInfo, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	jobInfo := &persist.JobInfo{}
	var mustHaveFields []interface{}
	if request.BlockOutput {
		mustHaveFields = append(mustHaveFields, "OutputCommit")
	}
	if request.BlockState {
		mustHaveFields = append(mustHaveFields, "State")
	}
	if err := a.waitMessageByPrimaryKey(
		jobInfosTable,
		request.Job.Id,
		jobInfo,
		func(jobInfo gorethink.Term) gorethink.Term {
			return jobInfo.HasFields(mustHaveFields...)
		},
	); err != nil {
		return nil, err
	}
	return jobInfo, nil
}

func (a *rethinkAPIServer) ListJobInfos(ctx context.Context, request *pps.ListJobRequest) (response *persist.JobInfos, retErr error) {
	defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
	query := a.getTerm(jobInfosTable)
	if request.Pipeline != nil && len(request.InputCommit) > 0 {
		query = query.GetAllByIndex(
			pipelineNameAndInputIndex,
			gorethink.Expr([]interface{}{request.Pipeline.Name, commitIndex(request.InputCommit)}),
		)
	} else if request.Pipeline != nil {
		query = query.GetAllByIndex(
			pipelineNameIndex,
			request.Pipeline.Name,
		)
	} else if len(request.InputCommit) > 0 {
		query = query.GetAllByIndex(
			inputIndex,
			gorethink.Expr(commitIndex(request.InputCommit)),
		)
	}
	cursor, err := query.Run(a.session)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := cursor.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	result := &persist.JobInfos{}
	for {
		jobInfo := &persist.JobInfo{}
		if !cursor.Next(jobInfo) {
			break
		}
		result.JobInfo = append(result.JobInfo, jobInfo)
	}
	if err := cursor.Err(); err != nil {
		return nil, err
	}
	return result, nil
}

func (a *rethinkAPIServer) DeleteJobInfo(ctx context.Context, request *pps.Job) (response *google_protobuf.Empty, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if err := a.deleteMessageByPrimaryKey(jobInfosTable, request.Id); err != nil {
		return nil, err
	}
	return google_protobuf.EmptyInstance, nil
}

func (a *rethinkAPIServer) CreateJobOutput(ctx context.Context, request *persist.JobOutput) (response *google_protobuf.Empty, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if err := a.updateMessage(jobInfosTable, request); err != nil {
		return nil, err
	}
	return google_protobuf.EmptyInstance, nil
}

func (a *rethinkAPIServer) CreateJobState(ctx context.Context, request *persist.JobState) (response *google_protobuf.Empty, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if err := a.updateMessage(jobInfosTable, request); err != nil {
		return nil, err
	}
	return google_protobuf.EmptyInstance, nil
}

// timestamp cannot be set
func (a *rethinkAPIServer) CreatePipelineInfo(ctx context.Context, request *persist.PipelineInfo) (response *persist.PipelineInfo, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if request.CreatedAt != nil {
		return nil, ErrTimestampSet
	}
	request.CreatedAt = a.now()
	if err := a.insertMessage(pipelineInfosTable, request); err != nil {
		return nil, err
	}
	return request, nil
}

func (a *rethinkAPIServer) GetPipelineInfo(ctx context.Context, request *pps.Pipeline) (response *persist.PipelineInfo, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	pipelineInfo := &persist.PipelineInfo{}
	if err := a.getMessageByPrimaryKey(pipelineInfosTable, request.Name, pipelineInfo); err != nil {
		return nil, err
	}
	return pipelineInfo, nil
}

func (a *rethinkAPIServer) ListPipelineInfos(ctx context.Context, request *google_protobuf.Empty) (response *persist.PipelineInfos, retErr error) {
	defer func(start time.Time) { a.Log(request, response, retErr, time.Since(start)) }(time.Now())
	query := a.getTerm(pipelineInfosTable)
	cursor, err := query.Run(a.session)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := cursor.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	result := &persist.PipelineInfos{}
	for {
		pipelineInfo := &persist.PipelineInfo{}
		if !cursor.Next(pipelineInfo) {
			break
		}
		result.PipelineInfo = append(result.PipelineInfo, pipelineInfo)
	}
	if err := cursor.Err(); err != nil {
		return nil, err
	}
	return result, nil
}

func (a *rethinkAPIServer) DeletePipelineInfo(ctx context.Context, request *pps.Pipeline) (response *google_protobuf.Empty, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if err := a.deleteMessageByPrimaryKey(pipelineInfosTable, request.Name); err != nil {
		return nil, err
	}
	return google_protobuf.EmptyInstance, nil
}

func (a *rethinkAPIServer) insertMessage(table Table, message proto.Message) error {
	_, err := a.getTerm(table).Insert(message).RunWrite(a.session)
	return err
}

func (a *rethinkAPIServer) updateMessage(table Table, message proto.Message) error {
	_, err := a.getTerm(table).Update(message).RunWrite(a.session)
	return err
}

func (a *rethinkAPIServer) getMessageByPrimaryKey(table Table, key interface{}, message proto.Message) error {
	cursor, err := a.getTerm(table).Get(key).Default(gorethink.Error("value not found")).Run(a.session)
	if err != nil {
		return err
	}
	if cursor.Next(message) {
		return cursor.Err()
	}
	return nil
}

func (a *rethinkAPIServer) deleteMessageByPrimaryKey(table Table, value interface{}) (retErr error) {
	_, err := a.getTerm(table).Get(value).Delete().RunWrite(a.session)
	return err
}

func (a *rethinkAPIServer) waitMessageByPrimaryKey(
	table Table,
	key interface{},
	message proto.Message,
	predicate func(term gorethink.Term) gorethink.Term,
) (retErr error) {
	cursor, err :=
		a.getTerm(table).
			Get(key).
			Default(gorethink.Error("value not found")).
			Changes().
			Field("new_val").
			Filter(predicate).
			Run(a.session)
	if err != nil {
		return err
	}
	defer func() {
		if err := cursor.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	cursor.Next(message)
	return cursor.Err()
}

func (a *rethinkAPIServer) getTerm(table Table) gorethink.Term {
	return gorethink.DB(a.databaseName).Table(table)
}

func (a *rethinkAPIServer) now() *google_protobuf.Timestamp {
	return prototime.TimeToTimestamp(a.timer.Now())
}

func commitIndex(commits []*pfs.Commit) string {
	var commitIDs []string
	for _, commit := range commits {
		commitIDs = append(commitIDs, commit.Id[0:10])
	}
	sort.Strings(commitIDs)
	var result []byte
	for _, commitID := range commitIDs {
		result = append(result, commitID...)
	}
	return string(result)
}
<|endoftext|>


<commit_before>package avail

import (
	"encoding/xml"
	"io/ioutil"
	"net/http"
	"strconv"
)

type Feed struct {
	BaseURL string
}

// Create a new feed
func NewFeed(url string) *Feed {
	a := &Feed{url + "/InfoPoint/rest/"}

	return a
}

// Make a new GET request to the requested API endpoint
func (a *Feed) NewAvailRequest(action string) (*http.Request, error) {
	url := a.BaseURL + action

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")

	return req, nil
}

func (a *Feed) VisibleRoutes() ([]Route, error) {
	var container ArrayOfRoute

	req, err := a.NewAvailRequest("routes/getvisibleroutes")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return container.Route, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return container.Route, err
	}

	err = xml.Unmarshal(body, &container)
	if err != nil {
		return container.Route, err
	}

	return container.Route, nil
}

func (a *Feed) Stops() ([]Stop, error) {
	var container ArrayOfStop

	req, err := a.NewAvailRequest("stops/getallstops")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return container.Stop, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return container.Stop, err
	}

	err = xml.Unmarshal(body, &container)
	if err != nil {
		return container.Stop, err
	}

	return container.Stop, nil
}

func (a *Feed) CurrentMessages() ([]PublicMessage, error) {
	var container ArrayOfPublicMessage

	req, err := a.NewAvailRequest("PublicMessages/GetCurrentMessages")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return container.PublicMessage, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return container.PublicMessage, err
	}

	err = xml.Unmarshal(body, &container)
	if err != nil {
		return container.PublicMessage, err
	}

	return container.PublicMessage, nil
}

func (a *Feed) Route(id int) (Route, error) {
	var container Route

	req, err := a.NewAvailRequest("routedetails/get/" + strconv.Itoa(id))

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return container, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return container, err
	}

	err = xml.Unmarshal(body, &container)
	if err != nil {
		return container, err
	}

	return container, nil
}

func (a *Feed) StopDeparture(id int) (StopDeparture, error) {
	var container ArrayOfStopDeparture

	req, err := a.NewAvailRequest("stopdepartures/get/" + strconv.Itoa(id))

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return container.StopDeparture, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return container.StopDeparture, err
	}

	err = xml.Unmarshal(body, &container)
	if err != nil {
		return container.StopDeparture, err
	}

	return container.StopDeparture, nil
}
<commit_msg>AppEngine stuff<commit_after>package avail

import (
	"encoding/xml"
	"io/ioutil"
	"net/http"
	"strconv"
)

type Feed struct {
	BaseURL string
	client  *http.Client
}

// Create a new feed
func NewFeed(url string) *Feed {
	a := &Feed{
		BaseURL: url + "/InfoPoint/rest/",
		client:  &http.Client{},
	}

	return a
}

// For AppEngine
func (a *Feed) SetClient(client *http.Client) {
	a.client = client
}

// Make a new GET request to the requested API endpoint
func (a *Feed) NewAvailRequest(action string) (*http.Request, error) {
	url := a.BaseURL + action

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")

	return req, nil
}

func (a *Feed) VisibleRoutes() ([]Route, error) {
	var container ArrayOfRoute

	req, err := a.NewAvailRequest("routes/getvisibleroutes")

	resp, err := a.client.Do(req)
	if err != nil {
		return container.Route, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return container.Route, err
	}

	err = xml.Unmarshal(body, &container)
	if err != nil {
		return container.Route, err
	}

	return container.Route, nil
}

func (a *Feed) Stops() ([]Stop, error) {
	var container ArrayOfStop

	req, err := a.NewAvailRequest("stops/getallstops")

	resp, err := a.client.Do(req)
	if err != nil {
		return container.Stop, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return container.Stop, err
	}

	err = xml.Unmarshal(body, &container)
	if err != nil {
		return container.Stop, err
	}

	return container.Stop, nil
}

func (a *Feed) CurrentMessages() ([]PublicMessage, error) {
	var container ArrayOfPublicMessage

	req, err := a.NewAvailRequest("PublicMessages/GetCurrentMessages")

	resp, err := a.client.Do(req)
	if err != nil {
		return container.PublicMessage, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return container.PublicMessage, err
	}

	err = xml.Unmarshal(body, &container)
	if err != nil {
		return container.PublicMessage, err
	}

	return container.PublicMessage, nil
}

func (a *Feed) Route(id int) (Route, error) {
	var container Route

	req, err := a.NewAvailRequest("routedetails/get/" + strconv.Itoa(id))

	resp, err := a.client.Do(req)
	if err != nil {
		return container, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return container, err
	}

	err = xml.Unmarshal(body, &container)
	if err != nil {
		return container, err
	}

	return container, nil
}

func (a *Feed) StopDeparture(id int) (StopDeparture, error) {
	var container ArrayOfStopDeparture

	req, err := a.NewAvailRequest("stopdepartures/get/" + strconv.Itoa(id))

	resp, err := a.client.Do(req)
	if err != nil {
		return container.StopDeparture, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return container.StopDeparture, err
	}

	err = xml.Unmarshal(body, &container)
	if err != nil {
		return container.StopDeparture, err
	}

	return container.StopDeparture, nil
}
<|endoftext|>


<commit_before>package zipfs

import (
	"archive/zip"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"sort"
	"strings"
	"sync"
	"time"
)

var (
	errFileClosed       = errors.New("file closed")
	errFileSystemClosed = errors.New("filesystem closed")
	errNotDirectory     = errors.New("not a directory")
	errDirectory        = errors.New("is a directory")
)

// FileSystem is a file system based on a ZIP file.
// It implements the http.FileSystem interface.
type FileSystem struct {
	readerAt  io.ReaderAt
	closer    io.Closer
	reader    *zip.Reader
	fileInfos fileInfoMap
}

// New will open the Zip file specified by name and
// return a new FileSystem based on that Zip file.
func New(name string) (*FileSystem, error) {
	file, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	fi, err := file.Stat()
	if err != nil {
		return nil, err
	}
	return NewFromReaderAt(file, fi.Size(), file)
}

// New will return a new FileSystem based on the given zip file (io.ReaderAt plus its size, and what to close).
func NewFromReaderAt(file io.ReaderAt, size int64, closer io.Closer) (*FileSystem, error) {
	zipReader, err := zip.NewReader(file, size)
	if err != nil {
		return nil, err
	}

	// Separate the file into an io.ReaderAt and an io.Closer.
	// Earlier versions of the code allowed for opening a filesystem
	// just with an io.ReaderAt. Not also that thw zip.Reader is
	// not actually used outside of this function so it probably
	// does not need to be in the FileSystem structure. Keeping it
	// there for now but may remove it in future.
	fs := &FileSystem{
		closer:    closer,
		readerAt:  file,
		reader:    zipReader,
		fileInfos: fileInfoMap{},
	}

	// Build a map of file paths to speed lookup.
	// Note that this assumes that there are not a very
	// large number of files in the ZIP file.
	//
	// Because we iterate through the map it seems reasonable
	// to attach each fileInfo to it's parent directory. Once again,
	// reasonable if the ZIP file does not contain a very large number
	// of entries.
	for _, zf := range fs.reader.File {
		fi := fs.fileInfos.FindOrCreate(zf.Name)
		fi.zipFile = zf
		fiParent := fs.fileInfos.FindOrCreateParent(zf.Name)
		fiParent.fileInfos = append(fiParent.fileInfos, fi)
	}

	// Sort all of the list of fileInfos in each directory.
	for _, fi := range fs.fileInfos {
		if len(fi.fileInfos) > 1 {
			sort.Sort(fi.fileInfos)
		}
	}

	return fs, nil
}

// Open implements the http.FileSystem interface.
// A http.File is returned, which can be served by
// the http.FileServer implementation.
func (fs *FileSystem) Open(name string) (http.File, error) {
	fi, err := fs.openFileInfo(name)
	if err != nil {
		return nil, err
	}

	return fi.openReader(name), nil
}

// Close closes the file system's underlying ZIP file and
// releases all memory allocated to internal data structures.
func (fs *FileSystem) Close() error {
	fs.reader = nil
	fs.readerAt = nil
	var err error
	if fs.closer != nil {
		err = fs.closer.Close()
		fs.closer = nil
	}
	fs.fileInfos = nil
	return err
}

type fileInfoList []*fileInfo

func (fl fileInfoList) Len() int {
	return len(fl)
}

func (fl fileInfoList) Less(i, j int) bool {
	name1 := fl[i].Name()
	name2 := fl[j].Name()
	return name1 < name2
}

func (fl fileInfoList) Swap(i, j int) {
	fi := fl[i]
	fl[i] = fl[j]
	fl[j] = fi
}

func (fs *FileSystem) openFileInfo(name string) (*fileInfo, error) {
	if fs.readerAt == nil {
		return nil, errFileSystemClosed
	}
	name = path.Clean(name)
	trimmedName := strings.TrimLeft(name, "/")
	fi := fs.fileInfos[trimmedName]
	if fi == nil {
		return nil, &os.PathError{Op: "Open", Path: name, Err: os.ErrNotExist}
	}

	return fi, nil
}

// fileMap keeps track of fileInfos
type fileInfoMap map[string]*fileInfo

func (fm fileInfoMap) FindOrCreate(name string) *fileInfo {
	strippedName := strings.TrimRight(name, "/")
	fi := fm[name]
	if fi == nil {
		fi = &fileInfo{
			name: name,
		}
		fm[name] = fi
		if strippedName != name {
			// directories get two entries: with and without trailing slash
			fm[strippedName] = fi
		}
	}
	return fi
}

func (fm fileInfoMap) FindOrCreateParent(name string) *fileInfo {
	strippedName := strings.TrimRight(name, "/")
	dirName := path.Dir(strippedName)
	if dirName == "." {
		dirName = "/"
	} else if !strings.HasSuffix(dirName, "/") {
		dirName = dirName + "/"
	}
	return fm.FindOrCreate(dirName)
}

// fileInfo implements the os.FileInfo interface.
type fileInfo struct {
	name      string
	fs        *FileSystem
	zipFile   *zip.File
	fileInfos fileInfoList
	tempPath  string
	mutex     sync.Mutex
}

func (fi *fileInfo) Name() string {
	return path.Base(fi.name)
}

func (fi *fileInfo) Size() int64 {
	if fi.zipFile == nil {
		return 0
	}
	if fi.zipFile.UncompressedSize64 == 0 {
		return int64(fi.zipFile.UncompressedSize)
	}
	return int64(fi.zipFile.UncompressedSize64)
}

func (fi *fileInfo) Mode() os.FileMode {
	if fi.zipFile == nil || fi.IsDir() {
		return 0555 | os.ModeDir
	}
	return 0444
}

var dirTime = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC)

func (fi *fileInfo) ModTime() time.Time {
	if fi.zipFile == nil {
		return dirTime
	}
	return fi.zipFile.ModTime()
}

func (fi *fileInfo) IsDir() bool {
	if fi.zipFile == nil {
		return true
	}
	return fi.zipFile.Mode().IsDir()
}

func (fi *fileInfo) Sys() interface{} {
	return fi.zipFile
}

func (fi *fileInfo) openReader(name string) *fileReader {
	return &fileReader{
		fileInfo: fi,
		name:     name,
	}
}

func (fi *fileInfo) readdir() ([]os.FileInfo, error) {
	if !fi.Mode().IsDir() {
		return nil, errNotDirectory
	}

	v := make([]os.FileInfo, len(fi.fileInfos))
	for i, fi := range fi.fileInfos {
		v[i] = fi
	}
	return v, nil
}

type fileReader struct {
	name     string // the name used to open
	fileInfo *fileInfo
	reader   io.ReadCloser
	file     *os.File
	closed   bool
	readdir  []os.FileInfo
}

func (f *fileReader) Close() error {
	var errs []error
	if f.reader != nil {
		err := f.reader.Close()
		errs = append(errs, err)
	}
	var tempFile string
	if f.file != nil {
		tempFile = f.file.Name()
		err := f.file.Close()
		errs = append(errs, err)
	}
	if tempFile != "" {
		err := os.Remove(tempFile)
		errs = append(errs, err)
	}

	f.closed = true

	for _, err := range errs {
		if err != nil {
			return f.pathError("Close", err)
		}
	}
	return nil
}

func (f *fileReader) Read(p []byte) (n int, err error) {
	if f.closed {
		return 0, f.pathError("Read", errFileClosed)
	}
	if f.file != nil {
		return f.file.Read(p)
	}
	if f.reader == nil {
		f.reader, err = f.fileInfo.zipFile.Open()
		if err != nil {
			return 0, err
		}
	}
	return f.reader.Read(p)
}

func (f *fileReader) Seek(offset int64, whence int) (int64, error) {
	if f.closed {
		return 0, f.pathError("Seek", errFileClosed)
	}

	// The reader cannot seek, so close it.
	if f.reader != nil {
		if err := f.reader.Close(); err != nil {
			return 0, err
		}
	}

	// A special case for when there is no file created and the seek is
	// to the beginning of the file. Just open (or re-open) the reader
	// at the beginning of the file.
	if f.file == nil && offset == 0 && whence == 0 {
		var err error
		f.reader, err = f.fileInfo.zipFile.Open()
		return 0, err
	}

	if err := f.createTempFile(); err != nil {
		return 0, err
	}

	return f.file.Seek(offset, whence)
}

func (f *fileReader) Readdir(count int) ([]os.FileInfo, error) {
	var err error
	var osFileInfos []os.FileInfo

	if count > 0 {
		if f.readdir == nil {
			f.readdir, err = f.fileInfo.readdir()
			if err != nil {
				return nil, f.pathError("Readdir", err)
			}
		}
		if len(f.readdir) >= count {
			osFileInfos = f.readdir[0:count]
			f.readdir = f.readdir[count:]
		} else {
			osFileInfos = f.readdir
			f.readdir = nil
			err = io.EOF
		}
	} else {
		osFileInfos, err = f.fileInfo.readdir()
		if err != nil {
			return nil, f.pathError("Readdir", err)
		}
	}

	return osFileInfos, err
}

func (f *fileReader) Stat() (os.FileInfo, error) {
	return f.fileInfo, nil
}

func (f *fileReader) createTempFile() error {
	if f.reader != nil {
		if err := f.reader.Close(); err != nil {
			return err
		}
		f.reader = nil
	}
	if f.file == nil {
		// Open a file that contains the contents of the zip file.
		osFile, err := createTempFile(f.fileInfo.zipFile)
		if err != nil {
			return err
		}

		f.file = osFile
	}
	return nil
}

func (f *fileReader) pathError(op string, err error) error {
	return &os.PathError{
		Op:   op,
		Path: f.name,
		Err:  err,
	}
}

// createTempFile creates a temporary file with the contents of the
// zip file. Used to implement io.Seeker interface.
func createTempFile(f *zip.File) (*os.File, error) {
	reader, err := f.Open()
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	tempFile, err := ioutil.TempFile("", "zipfs")
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(tempFile, reader)
	if err != nil {
		tempFile.Close()
		os.Remove(tempFile.Name())
		return nil, err
	}
	_, err = tempFile.Seek(0, os.SEEK_SET)
	if err != nil {
		tempFile.Close()
		os.Remove(tempFile.Name())
		return nil, err
	}

	return tempFile, nil
}
<commit_msg>elaborate comment of NewReaderAt<commit_after>package zipfs

import (
	"archive/zip"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"sort"
	"strings"
	"sync"
	"time"
)

var (
	errFileClosed       = errors.New("file closed")
	errFileSystemClosed = errors.New("filesystem closed")
	errNotDirectory     = errors.New("not a directory")
	errDirectory        = errors.New("is a directory")
)

// FileSystem is a file system based on a ZIP file.
// It implements the http.FileSystem interface.
type FileSystem struct {
	readerAt  io.ReaderAt
	closer    io.Closer
	reader    *zip.Reader
	fileInfos fileInfoMap
}

// New will open the Zip file specified by name and
// return a new FileSystem based on that Zip file.
func New(name string) (*FileSystem, error) {
	file, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	fi, err := file.Stat()
	if err != nil {
		return nil, err
	}
	return NewFromReaderAt(file, fi.Size(), file)
}

// NewFromReaderAt will open the Zip file accessible by readerAt with the given size.
// The closer will be called when the file system is closed
// (you can use io/ioutil.NopCloser(nil) for a closer doing nothing, or your file to have it being closed).
func NewFromReaderAt(readerAt io.ReaderAt, size int64, closer io.Closer) (*FileSystem, error) {
	zipReader, err := zip.NewReader(readerAt, size)
	if err != nil {
		return nil, err
	}

	// Separate the file into an io.ReaderAt and an io.Closer.
	// Earlier versions of the code allowed for opening a filesystem
	// just with an io.ReaderAt. Not also that thw zip.Reader is
	// not actually used outside of this function so it probably
	// does not need to be in the FileSystem structure. Keeping it
	// there for now but may remove it in future.
	fs := &FileSystem{
		closer:    closer,
		readerAt:  readerAt,
		reader:    zipReader,
		fileInfos: fileInfoMap{},
	}

	// Build a map of file paths to speed lookup.
	// Note that this assumes that there are not a very
	// large number of files in the ZIP file.
	//
	// Because we iterate through the map it seems reasonable
	// to attach each fileInfo to it's parent directory. Once again,
	// reasonable if the ZIP file does not contain a very large number
	// of entries.
	for _, zf := range fs.reader.File {
		fi := fs.fileInfos.FindOrCreate(zf.Name)
		fi.zipFile = zf
		fiParent := fs.fileInfos.FindOrCreateParent(zf.Name)
		fiParent.fileInfos = append(fiParent.fileInfos, fi)
	}

	// Sort all of the list of fileInfos in each directory.
	for _, fi := range fs.fileInfos {
		if len(fi.fileInfos) > 1 {
			sort.Sort(fi.fileInfos)
		}
	}

	return fs, nil
}

// Open implements the http.FileSystem interface.
// A http.File is returned, which can be served by
// the http.FileServer implementation.
func (fs *FileSystem) Open(name string) (http.File, error) {
	fi, err := fs.openFileInfo(name)
	if err != nil {
		return nil, err
	}

	return fi.openReader(name), nil
}

// Close closes the file system's underlying ZIP file and
// releases all memory allocated to internal data structures.
func (fs *FileSystem) Close() error {
	fs.reader = nil
	fs.readerAt = nil
	var err error
	if fs.closer != nil {
		err = fs.closer.Close()
		fs.closer = nil
	}
	fs.fileInfos = nil
	return err
}

type fileInfoList []*fileInfo

func (fl fileInfoList) Len() int {
	return len(fl)
}

func (fl fileInfoList) Less(i, j int) bool {
	name1 := fl[i].Name()
	name2 := fl[j].Name()
	return name1 < name2
}

func (fl fileInfoList) Swap(i, j int) {
	fi := fl[i]
	fl[i] = fl[j]
	fl[j] = fi
}

func (fs *FileSystem) openFileInfo(name string) (*fileInfo, error) {
	if fs.readerAt == nil {
		return nil, errFileSystemClosed
	}
	name = path.Clean(name)
	trimmedName := strings.TrimLeft(name, "/")
	fi := fs.fileInfos[trimmedName]
	if fi == nil {
		return nil, &os.PathError{Op: "Open", Path: name, Err: os.ErrNotExist}
	}

	return fi, nil
}

// fileMap keeps track of fileInfos
type fileInfoMap map[string]*fileInfo

func (fm fileInfoMap) FindOrCreate(name string) *fileInfo {
	strippedName := strings.TrimRight(name, "/")
	fi := fm[name]
	if fi == nil {
		fi = &fileInfo{
			name: name,
		}
		fm[name] = fi
		if strippedName != name {
			// directories get two entries: with and without trailing slash
			fm[strippedName] = fi
		}
	}
	return fi
}

func (fm fileInfoMap) FindOrCreateParent(name string) *fileInfo {
	strippedName := strings.TrimRight(name, "/")
	dirName := path.Dir(strippedName)
	if dirName == "." {
		dirName = "/"
	} else if !strings.HasSuffix(dirName, "/") {
		dirName = dirName + "/"
	}
	return fm.FindOrCreate(dirName)
}

// fileInfo implements the os.FileInfo interface.
type fileInfo struct {
	name      string
	fs        *FileSystem
	zipFile   *zip.File
	fileInfos fileInfoList
	tempPath  string
	mutex     sync.Mutex
}

func (fi *fileInfo) Name() string {
	return path.Base(fi.name)
}

func (fi *fileInfo) Size() int64 {
	if fi.zipFile == nil {
		return 0
	}
	if fi.zipFile.UncompressedSize64 == 0 {
		return int64(fi.zipFile.UncompressedSize)
	}
	return int64(fi.zipFile.UncompressedSize64)
}

func (fi *fileInfo) Mode() os.FileMode {
	if fi.zipFile == nil || fi.IsDir() {
		return 0555 | os.ModeDir
	}
	return 0444
}

var dirTime = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC)

func (fi *fileInfo) ModTime() time.Time {
	if fi.zipFile == nil {
		return dirTime
	}
	return fi.zipFile.ModTime()
}

func (fi *fileInfo) IsDir() bool {
	if fi.zipFile == nil {
		return true
	}
	return fi.zipFile.Mode().IsDir()
}

func (fi *fileInfo) Sys() interface{} {
	return fi.zipFile
}

func (fi *fileInfo) openReader(name string) *fileReader {
	return &fileReader{
		fileInfo: fi,
		name:     name,
	}
}

func (fi *fileInfo) readdir() ([]os.FileInfo, error) {
	if !fi.Mode().IsDir() {
		return nil, errNotDirectory
	}

	v := make([]os.FileInfo, len(fi.fileInfos))
	for i, fi := range fi.fileInfos {
		v[i] = fi
	}
	return v, nil
}

type fileReader struct {
	name     string // the name used to open
	fileInfo *fileInfo
	reader   io.ReadCloser
	file     *os.File
	closed   bool
	readdir  []os.FileInfo
}

func (f *fileReader) Close() error {
	var errs []error
	if f.reader != nil {
		err := f.reader.Close()
		errs = append(errs, err)
	}
	var tempFile string
	if f.file != nil {
		tempFile = f.file.Name()
		err := f.file.Close()
		errs = append(errs, err)
	}
	if tempFile != "" {
		err := os.Remove(tempFile)
		errs = append(errs, err)
	}

	f.closed = true

	for _, err := range errs {
		if err != nil {
			return f.pathError("Close", err)
		}
	}
	return nil
}

func (f *fileReader) Read(p []byte) (n int, err error) {
	if f.closed {
		return 0, f.pathError("Read", errFileClosed)
	}
	if f.file != nil {
		return f.file.Read(p)
	}
	if f.reader == nil {
		f.reader, err = f.fileInfo.zipFile.Open()
		if err != nil {
			return 0, err
		}
	}
	return f.reader.Read(p)
}

func (f *fileReader) Seek(offset int64, whence int) (int64, error) {
	if f.closed {
		return 0, f.pathError("Seek", errFileClosed)
	}

	// The reader cannot seek, so close it.
	if f.reader != nil {
		if err := f.reader.Close(); err != nil {
			return 0, err
		}
	}

	// A special case for when there is no file created and the seek is
	// to the beginning of the file. Just open (or re-open) the reader
	// at the beginning of the file.
	if f.file == nil && offset == 0 && whence == 0 {
		var err error
		f.reader, err = f.fileInfo.zipFile.Open()
		return 0, err
	}

	if err := f.createTempFile(); err != nil {
		return 0, err
	}

	return f.file.Seek(offset, whence)
}

func (f *fileReader) Readdir(count int) ([]os.FileInfo, error) {
	var err error
	var osFileInfos []os.FileInfo

	if count > 0 {
		if f.readdir == nil {
			f.readdir, err = f.fileInfo.readdir()
			if err != nil {
				return nil, f.pathError("Readdir", err)
			}
		}
		if len(f.readdir) >= count {
			osFileInfos = f.readdir[0:count]
			f.readdir = f.readdir[count:]
		} else {
			osFileInfos = f.readdir
			f.readdir = nil
			err = io.EOF
		}
	} else {
		osFileInfos, err = f.fileInfo.readdir()
		if err != nil {
			return nil, f.pathError("Readdir", err)
		}
	}

	return osFileInfos, err
}

func (f *fileReader) Stat() (os.FileInfo, error) {
	return f.fileInfo, nil
}

func (f *fileReader) createTempFile() error {
	if f.reader != nil {
		if err := f.reader.Close(); err != nil {
			return err
		}
		f.reader = nil
	}
	if f.file == nil {
		// Open a file that contains the contents of the zip file.
		osFile, err := createTempFile(f.fileInfo.zipFile)
		if err != nil {
			return err
		}

		f.file = osFile
	}
	return nil
}

func (f *fileReader) pathError(op string, err error) error {
	return &os.PathError{
		Op:   op,
		Path: f.name,
		Err:  err,
	}
}

// createTempFile creates a temporary file with the contents of the
// zip file. Used to implement io.Seeker interface.
func createTempFile(f *zip.File) (*os.File, error) {
	reader, err := f.Open()
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	tempFile, err := ioutil.TempFile("", "zipfs")
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(tempFile, reader)
	if err != nil {
		tempFile.Close()
		os.Remove(tempFile.Name())
		return nil, err
	}
	_, err = tempFile.Seek(0, os.SEEK_SET)
	if err != nil {
		tempFile.Close()
		os.Remove(tempFile.Name())
		return nil, err
	}

	return tempFile, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"sort"
	"strings"

	"github.com/juju/errors"
	"github.com/juju/utils/set"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"
)

// machineRemovalDoc indicates that this machine needs to be removed
// and any necessary provider-level cleanup should now be done.
type machineRemovalDoc struct {
	DocID     string `bson:"_id"`
	MachineID string `bson:"machine-id"`
}

// MarkForRemoval requests that this machine be removed after any
// needed provider-level cleanup is done.
func (m *Machine) MarkForRemoval() (err error) {
	defer errors.DeferredAnnotatef(&err, "cannot remove machine %s", m.doc.Id)
	if m.doc.Life != Dead {
		return errors.Errorf("machine is not dead")
	}
	ops := []txn.Op{{
		C:      machinesC,
		Id:     m.doc.DocID,
		Assert: isDeadDoc,
	}, {
		C:      machineRemovalsC,
		Id:     m.globalKey(),
		Insert: &machineRemovalDoc{MachineID: m.Id()},
	}}
	return onAbort(m.st.runTransaction(ops), errors.Errorf("machine is not dead"))
}

// AllMachineRemovals returns (the ids of) all of the machines that
// need to be removed but need provider-level cleanup.
func (st *State) AllMachineRemovals() ([]string, error) {
	removals, close := st.getCollection(machineRemovalsC)
	defer close()

	var docs []machineRemovalDoc
	err := removals.Find(nil).All(&docs)
	if err != nil {
		return nil, errors.Trace(err)
	}
	results := make([]string, len(docs))
	for i := range docs {
		results[i] = docs[i].MachineID
	}
	return results, nil
}

func (st *State) allMachinesMatching(query bson.D) ([]*Machine, error) {
	machines, close := st.getCollection(machinesC)
	defer close()

	var docs []machineDoc
	err := machines.Find(query).All(&docs)
	if err != nil {
		return nil, errors.Trace(err)
	}
	results := make([]*Machine, len(docs))
	for i, doc := range docs {
		results[i] = newMachine(st, &doc)
	}
	return results, nil
}

func plural(count int) string {
	if count == 1 {
		return ""
	}
	return "s"
}

func collectMissingMachineIds(expectedIds []string, machines []*Machine) []string {
	expectedSet := set.NewStrings(expectedIds...)
	actualSet := set.NewStrings()
	for _, machine := range machines {
		actualSet.Add(machine.Id())
	}
	return expectedSet.Difference(actualSet).SortedValues()
}

// CompleteMachineRemovals finishes the removal of the specified
// machines. The machines must have been marked for removal
// previously. Unknown machine ids are ignored so that this is
// idempotent.
func (st *State) CompleteMachineRemovals(ids ...string) error {
	removals, err := st.AllMachineRemovals()
	if err != nil {
		return errors.Trace(err)
	}
	removalSet := set.NewStrings(removals...)
	query := bson.D{{"machineid", bson.D{{"$in", ids}}}}
	machinesToRemove, err := st.allMachinesMatching(query)
	if err != nil {
		return errors.Trace(err)
	}

	if len(machinesToRemove) < len(ids) {
		missingMachines := collectMissingMachineIds(ids, machinesToRemove)
		logger.Debugf("skipping nonexistent machine%s: %s",
			plural(len(missingMachines)),
			strings.Join(missingMachines, ", "),
		)
	}

	var ops []txn.Op
	var missingRemovals []string
	for _, machine := range machinesToRemove {
		if !removalSet.Contains(machine.Id()) {
			missingRemovals = append(missingRemovals, machine.Id())
			continue
		}

		ops = append(ops, txn.Op{
			C:      machineRemovalsC,
			Id:     machine.globalKey(),
			Remove: true,
		})
		removeMachineOps, err := machine.removeOps()
		if err != nil {
			return errors.Trace(err)
		}
		ops = append(ops, removeMachineOps...)
	}
	// We should complain about machines that still exist but haven't
	// been marked for removal.
	if len(missingRemovals) > 0 {
		sort.Strings(missingRemovals)
		return errors.Errorf(
			"cannot remove machine%s %s: not marked for removal",
			plural(len(missingRemovals)),
			strings.Join(missingRemovals, ", "),
		)
	}

	return st.runTransaction(ops)
}
<commit_msg>Explicatory comments for txn assertions<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"sort"
	"strings"

	"github.com/juju/errors"
	"github.com/juju/utils/set"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"
)

// machineRemovalDoc indicates that this machine needs to be removed
// and any necessary provider-level cleanup should now be done.
type machineRemovalDoc struct {
	DocID     string `bson:"_id"`
	MachineID string `bson:"machine-id"`
}

// MarkForRemoval requests that this machine be removed after any
// needed provider-level cleanup is done.
func (m *Machine) MarkForRemoval() (err error) {
	defer errors.DeferredAnnotatef(&err, "cannot remove machine %s", m.doc.Id)
	if m.doc.Life != Dead {
		return errors.Errorf("machine is not dead")
	}
	ops := []txn.Op{{
		C:  machinesC,
		Id: m.doc.DocID,
		// Check that the machine is still dead (and implicitly that
		// it still exists).
		Assert: isDeadDoc,
	}, {
		C:      machineRemovalsC,
		Id:     m.globalKey(),
		Insert: &machineRemovalDoc{MachineID: m.Id()},
		// No assert here - it's ok if the machine has already been
		// marked. The id will prevent duplicates.
	}}
	return onAbort(m.st.runTransaction(ops), errors.Errorf("machine is not dead"))
}

// AllMachineRemovals returns (the ids of) all of the machines that
// need to be removed but need provider-level cleanup.
func (st *State) AllMachineRemovals() ([]string, error) {
	removals, close := st.getCollection(machineRemovalsC)
	defer close()

	var docs []machineRemovalDoc
	err := removals.Find(nil).All(&docs)
	if err != nil {
		return nil, errors.Trace(err)
	}
	results := make([]string, len(docs))
	for i := range docs {
		results[i] = docs[i].MachineID
	}
	return results, nil
}

func (st *State) allMachinesMatching(query bson.D) ([]*Machine, error) {
	machines, close := st.getCollection(machinesC)
	defer close()

	var docs []machineDoc
	err := machines.Find(query).All(&docs)
	if err != nil {
		return nil, errors.Trace(err)
	}
	results := make([]*Machine, len(docs))
	for i, doc := range docs {
		results[i] = newMachine(st, &doc)
	}
	return results, nil
}

func plural(count int) string {
	if count == 1 {
		return ""
	}
	return "s"
}

func collectMissingMachineIds(expectedIds []string, machines []*Machine) []string {
	expectedSet := set.NewStrings(expectedIds...)
	actualSet := set.NewStrings()
	for _, machine := range machines {
		actualSet.Add(machine.Id())
	}
	return expectedSet.Difference(actualSet).SortedValues()
}

// CompleteMachineRemovals finishes the removal of the specified
// machines. The machines must have been marked for removal
// previously. Unknown machine ids are ignored so that this is
// idempotent.
func (st *State) CompleteMachineRemovals(ids ...string) error {
	removals, err := st.AllMachineRemovals()
	if err != nil {
		return errors.Trace(err)
	}
	removalSet := set.NewStrings(removals...)
	query := bson.D{{"machineid", bson.D{{"$in", ids}}}}
	machinesToRemove, err := st.allMachinesMatching(query)
	if err != nil {
		return errors.Trace(err)
	}

	if len(machinesToRemove) < len(ids) {
		missingMachines := collectMissingMachineIds(ids, machinesToRemove)
		logger.Debugf("skipping nonexistent machine%s: %s",
			plural(len(missingMachines)),
			strings.Join(missingMachines, ", "),
		)
	}

	var ops []txn.Op
	var missingRemovals []string
	for _, machine := range machinesToRemove {
		if !removalSet.Contains(machine.Id()) {
			missingRemovals = append(missingRemovals, machine.Id())
			continue
		}

		ops = append(ops, txn.Op{
			C:      machineRemovalsC,
			Id:     machine.globalKey(),
			Remove: true,
		})
		removeMachineOps, err := machine.removeOps()
		if err != nil {
			return errors.Trace(err)
		}
		ops = append(ops, removeMachineOps...)
	}
	// We should complain about machines that still exist but haven't
	// been marked for removal.
	if len(missingRemovals) > 0 {
		sort.Strings(missingRemovals)
		return errors.Errorf(
			"cannot remove machine%s %s: not marked for removal",
			plural(len(missingRemovals)),
			strings.Join(missingRemovals, ", "),
		)
	}

	return st.runTransaction(ops)
}
<|endoftext|>


<commit_before>package flake

import "math/big"

// Generator is the primary interface for flake ID generation
type Generator interface {
	// Epoch returns the epoch used for overt-flake identifers
	Epoch() int64
	// HardwareID returns the hardware identifier used for overt-flake identifiers
	HardwareID() HardwareID
	// ProcessID returns the process id hosting the overt-flake Generator
	ProcessID() int

	// Generate generates count overt-flake identifiers
	Generate(count int) ([]byte, error)
}

// HardwareID is an alias for []byte
type HardwareID []byte

// HardwareIDProvider is a provider that generates a hardware identifier for
// use by an overt-flake Generator
type HardwareIDProvider interface {
	GetHardwareID(byteSize int) ([]byte, error)
}

// OvertFlakeID is an interface that provides access to the components and
// alternate representations of an overt-flake identifier
type OvertFlakeID interface {
	// Timestamp is when the ID was generated, and is the # of milliseconds since
	// the generator Epoch
	Timestamp() uint64
	// Interval represents the Nth value created during a time interval (0 for the 1st)
	Interval() uint16

	// HardwareID is the HardwareID assigned by the generator
	HardwareID() HardwareID
	// ProcessID is the processID assigned by the generator
	ProcessID() uint16
	// MachineID is the uint64 representation of HardwareID and ProcessID and is == Lower()
	MachineID() uint64

	// Upper is the upper (most-signficant) bytes of the id represented as a uint64
	Upper() uint64
	// Lower is the lower (least-signficant) bytes of the id represented as a uint64
	Lower() uint64

	// Bytes is the []byte representation of the ID
	Bytes() []byte

	// ToBigInt converts the ID to a *big.Int
	ToBigInt() *big.Int

	// String returns the big.Int string representation of the ID
	String() string
}
<commit_msg>added Generator.LastAllocatedTime() OvertFlakeID.Interval() => SequenceID()<commit_after>package flake

import "math/big"

// Generator is the primary interface for flake ID generation
type Generator interface {
	// Epoch returns the epoch used for overt-flake identifers
	Epoch() int64
	// HardwareID returns the hardware identifier used for overt-flake identifiers
	HardwareID() HardwareID
	// ProcessID returns the process id hosting the overt-flake Generator
	ProcessID() int

	// LastAllocatedTime is the last Unix Epoch value that one or more ids
	// are known to have been generated
	LastAllocatedTime() int64

	// Generate generates count overt-flake identifiers
	Generate(count int) ([]byte, error)

	// GenerateAsStream allocates and returns ids in chunks (based on the size of buffer) via a callback
	GenerateAsStream(count int, buffer []byte, callback func(int, []byte) error) (totalAllocated int, err error)
}

// HardwareID is an alias for []byte
type HardwareID []byte

// HardwareIDProvider is a provider that generates a hardware identifier for
// use by an overt-flake Generator
type HardwareIDProvider interface {
	GetHardwareID(byteSize int) ([]byte, error)
}

// OvertFlakeID is an interface that provides access to the components and
// alternate representations of an overt-flake identifier
type OvertFlakeID interface {
	// Timestamp is when the ID was generated, and is the # of milliseconds since
	// the generator Epoch
	Timestamp() uint64
	// SequenceID represents the Nth value created during a time interval (0 for the 1st)
	SequenceID() uint16

	// HardwareID is the HardwareID assigned by the generator
	HardwareID() HardwareID
	// ProcessID is the processID assigned by the generator
	ProcessID() uint16
	// MachineID is the uint64 representation of HardwareID and ProcessID and is == Lower()
	MachineID() uint64

	// Upper is the upper (most-signficant) bytes of the id represented as a uint64
	Upper() uint64
	// Lower is the lower (least-signficant) bytes of the id represented as a uint64
	Lower() uint64

	// Bytes is the []byte representation of the ID
	Bytes() []byte

	// ToBigInt converts the ID to a *big.Int
	ToBigInt() *big.Int

	// String returns the big.Int string representation of the ID
	String() string
}
<|endoftext|>


<commit_before>// dsize.go -- sort in reverse size order.  IE, biggest is first.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	//
)

const lastCompiled = "22 Apr 17"

/*
Revision History
----------------
20 Apr 17 -- Started writing this rtn, based on dirlist.go
21 Apr 17 -- Now tweaking the output format.  And used flag package.  One as a pointer and one as a value, just to learn them.
22 Apr 17 -- Now to use the non flag commandline.
*/

// FIS is a FileInfo slice, as in os.FileInfo
type FISlice []os.FileInfo

func (f FISlice) Less(i, j int) bool {
	return f[i].Size() > f[j].Size() // I want a reverse sort
}

func (f FISlice) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISlice) Len() int {
	return len(f)
}

func main() {
	var files FISlice
	var err error

	var revflag = flag.Bool("r", false, "reverse the sort, ie, smallest is first") // Ptr

	var RevFlag bool
	flag.BoolVar(&RevFlag, "R", false, "Reverse the sort, ie, smallest is first") // Var

	flag.PrintDefaults()
	flag.Parse()
	Reverse := *revflag || RevFlag

	CleanDirName := "." + string(filepath.Separator)
	commandline := flag.Args()
	/*
		if len(os.Args) > 1 {
			commandline := getcommandline.GetCommandLineString()
			CleanDirName = filepath.Clean(commandline)
		}
	*/

	fmt.Println(" dsize will display a directory by size.  Written in Go.  lastCompiled ", lastCompiled)
	fmt.Println()
	fmt.Println(" Dirname is", CleanDirName)

	files, err = ioutil.ReadDir(CleanDirName)
	if err != nil {
		log.Fatal(err)
	}

	//	fmt.Println(" bool pointer reverse is ", *revflag, ".  bool var Reverse is ", RevFlag, ", Reverse is ", Reverse)

	if Reverse {
		sort.Sort(sort.Reverse(files))
	} else {
		sort.Sort(files)
	}

	for _, f := range files {
		s := f.ModTime().Format("Jan-02-2006 15:04:05")
		fmt.Printf("%10v %11d %s %s\n", f.Mode(), f.Size(), s, f.Name())
		//		fmt.Printf("%10v %11d %s %s\n", f.Mode(), f.Size(), f.ModTime().String(), f.Name())
	}

}

/*
package path
func Match

func Match(pattern, name string) (matched bool, err error)

Match reports whether name matches the shell file name pattern.  The pattern syntax is:

pattern:
	{ term }
term:
	'*'         matches any sequence of non-/ characters
	'?'         matches any single non-/ character
	'[' [ '^' ] { character-range } ']'
	            character class (must be non-empty)
	c           matches character c (c != '*', '?', '\\', '[')
	'\\' c      matches character c

character-range:
	c           matches character c (c != '\\', '-', ']')
	'\\' c      matches character c
	lo '-' hi   matches character c for lo <= c <= hi

Match requires pattern to match all of name, not just a substring.  The only possible returned error is ErrBadPattern, when pattern is malformed.

package os
type FileInfo

type FileInfo interface {
        Name() string       // base name of the file
        Size() int64        // length in bytes for regular files; system-dependent for others
        Mode() FileMode     // file mode bits
        ModTime() time.Time // modification time
        IsDir() bool        // abbreviation for Mode().IsDir()
        Sys() interface{}   // underlying data source (can return nil)
}

A FileInfo describes a file and is returned by Stat and Lstat.

func Lstat

func Lstat(name string) (FileInfo, error)

Lstat returns a FileInfo describing the named file.  If the file is a symbolic link, the returned FileInfo describes the symbolic link.  Lstat makes no attempt to follow the link.
If there is an error, it will be of type *PathError.

func Stat

func Stat(name string) (FileInfo, error)

Stat returns a FileInfo describing the named file.  If there is an error, it will be of type *PathError.

The insight I had with my append troubles that the 1 slice entries were empty, is that when I used append, it would do just that to the end of the slice, and ignore the empty slices.
I needed to make the slice as empty for this to work.  So I am directly assigning the DirEntries slice, and appending the FileNames slice, to make sure that these both are doing what I want.
This code is now doing exactly what I want.  I guess there is no substitute for playing with myself.  Wait, that didn't come out right.  Or did it.

*/
<commit_msg>modified:   dsize/dsize.go -- finished version<commit_after>// dsize.go -- sort in reverse size order.  IE, biggest is first.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
)

const lastCompiled = "22 Apr 17"

/*
Revision History
----------------
20 Apr 17 -- Started writing this rtn, based on dirlist.go
21 Apr 17 -- Now tweaking the output format.  And used flag package.  One as a pointer and one as a value, just to learn them.
22 Apr 17 -- Coded the use of the first non flag commandline param,  which is all I need.  Note that the flag must appear before the non-flag param, else the flag is ignored.
*/

// FIS is a FileInfo slice, as in os.FileInfo
type FISlice []os.FileInfo

func (f FISlice) Less(i, j int) bool {
	return f[i].Size() > f[j].Size() // I want a reverse sort
}

func (f FISlice) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISlice) Len() int {
	return len(f)
}

func main() {
	var files FISlice
	var err error

	var revflag = flag.Bool("r", false, "reverse the sort, ie, smallest is first") // Ptr

	var RevFlag bool
	flag.BoolVar(&RevFlag, "R", false, "Reverse the sort, ie, smallest is first") // Var

	flag.PrintDefaults()
	flag.Parse()
	Reverse := *revflag || RevFlag

	CleanDirName := "." + string(filepath.Separator)
	commandline := flag.Arg(0) // this only gets the first non flag argument.  That's all I want
	if len(commandline) > 0 {
		CleanDirName = filepath.Clean(commandline)
	}

	fmt.Println(" dsize will display a directory by size.  Written in Go.  lastCompiled ", lastCompiled)
	fmt.Println()
	fmt.Println(" Dirname is", CleanDirName)

	files, err = ioutil.ReadDir(CleanDirName)
	if err != nil {
		log.Fatal(err)
	}

	//	fmt.Println(" bool pointer reverse is ", *revflag, ".  bool var Reverse is ", RevFlag, ", Reverse is ", Reverse)

	if Reverse {
		sort.Sort(sort.Reverse(files))
	} else {
		sort.Sort(files)
	}

	for _, f := range files {
		s := f.ModTime().Format("Jan-02-2006 15:04:05")
		fmt.Printf("%10v %11d %s %s\n", f.Mode(), f.Size(), s, f.Name())
		//		fmt.Printf("%10v %11d %s %s\n", f.Mode(), f.Size(), f.ModTime().String(), f.Name())
	}

}

/*
package path
func Match

func Match(pattern, name string) (matched bool, err error)

Match reports whether name matches the shell file name pattern.  The pattern syntax is:

pattern:
	{ term }
term:
	'*'         matches any sequence of non-/ characters
	'?'         matches any single non-/ character
	'[' [ '^' ] { character-range } ']'
	            character class (must be non-empty)
	c           matches character c (c != '*', '?', '\\', '[')
	'\\' c      matches character c

character-range:
	c           matches character c (c != '\\', '-', ']')
	'\\' c      matches character c
	lo '-' hi   matches character c for lo <= c <= hi

Match requires pattern to match all of name, not just a substring.  The only possible returned error is ErrBadPattern, when pattern is malformed.

package os
type FileInfo

type FileInfo interface {
        Name() string       // base name of the file
        Size() int64        // length in bytes for regular files; system-dependent for others
        Mode() FileMode     // file mode bits
        ModTime() time.Time // modification time
        IsDir() bool        // abbreviation for Mode().IsDir()
        Sys() interface{}   // underlying data source (can return nil)
}

A FileInfo describes a file and is returned by Stat and Lstat.

func Lstat

func Lstat(name string) (FileInfo, error)

Lstat returns a FileInfo describing the named file.  If the file is a symbolic link, the returned FileInfo describes the symbolic link.  Lstat makes no attempt to follow the link.
If there is an error, it will be of type *PathError.

func Stat

func Stat(name string) (FileInfo, error)

Stat returns a FileInfo describing the named file.  If there is an error, it will be of type *PathError.

The insight I had with my append troubles that the 1 slice entries were empty, is that when I used append, it would do just that to the end of the slice, and ignore the empty slices.
I needed to make the slice as empty for this to work.  So I am directly assigning the DirEntries slice, and appending the FileNames slice, to make sure that these both are doing what I want.
This code is now doing exactly what I want.  I guess there is no substitute for playing with myself.  Wait, that didn't come out right.  Or did it.

*/
<|endoftext|>


<commit_before>package eventemitter

import (
	"fmt"
)

type Event struct {
	Name   string
	Argv   []interface{}
	Result interface{}
}

type EventListener func(event *Event)

type EventError struct {
	EventName string
	Message   string
}

func (self EventError) Error() string {
	return fmt.Sprintf("%s (Event: '%s')", self.Message, self.EventName)
}

type EventEmitter struct {
	Events map[string][]EventListener
}

func NewEventEmitter() *EventEmitter {
	e := new(EventEmitter)
	e.Init()

	return e
}

func (self *EventEmitter) Init() {
	self.Events = make(map[string][]EventListener)
}

func (self *EventEmitter) On(event string, listener EventListener) {
    self.AddListener(event, listener)
}

func (self *EventEmitter) AddListener(event string, listener EventListener) {
	// Check if the event exists, otherwise initialize the list
	// of handlers for this event.
	if _, exists := self.Events[event]; !exists {
		self.Events[event] = []EventListener{listener}
	} else {
		self.Events[event] = append(self.Events[event], listener)
	}
}

func (self *EventEmitter) RemoveListeners(event string) {
	delete(self.Events, event)
}

// TODO: Return "false" as second argument when no listeners
// where registered to enable the "Comma OK" idiom.
func (self *EventEmitter) Emit(event string, argv ...interface{}) (chan *Event) {
	listeners, exists := self.Events[event]

	if !exists {
		return nil
	}

	c := make(chan *Event)

	for _, listener := range listeners {
		go func() {
			e := &Event{Name: event, Argv: argv}
			listener(e)
			c <- e
		}()
	}

	return c
}
<commit_msg>Added some doc comments.<commit_after>package eventemitter

import (
	"fmt"
)

type Event struct {
	Name   string
	Argv   []interface{}
	Result interface{}
}

type EventListener func(event *Event)

type EventEmitter struct {
	Events map[string][]EventListener
}

func NewEventEmitter() *EventEmitter {
	e := new(EventEmitter)
	e.Init()

	return e
}

// Allocates the EventEmitters memory. Has to be called when
// embedding an EventEmitter in another Type.
func (self *EventEmitter) Init() {
	self.Events = make(map[string][]EventListener)
}

// Alias to AddListener.
func (self *EventEmitter) On(event string, listener EventListener) {
	self.AddListener(event, listener)
}

// AddListener adds an event listener on the given event name.
func (self *EventEmitter) AddListener(event string, listener EventListener) {
	// Check if the event exists, otherwise initialize the list
	// of handlers for this event.
	if _, exists := self.Events[event]; !exists {
		self.Events[event] = []EventListener{listener}
	} else {
		self.Events[event] = append(self.Events[event], listener)
	}
}

// Removes all listeners from the given event.
func (self *EventEmitter) RemoveListeners(event string) {
	delete(self.Events, event)
}

// Emits the given event. Puts all arguments following the event name
// into the Event's `Argv` member. Returns a channel if listeners were
// called, nil otherwise.
func (self *EventEmitter) Emit(event string, argv ...interface{}) chan *Event {
	listeners, exists := self.Events[event]

	if !exists {
		return nil
	}

	c := make(chan *Event)

	for _, listener := range listeners {
		go func() {
			e := &Event{Name: event, Argv: argv}
			listener(e)
			c <- e
		}()
	}

	return c
}
<|endoftext|>


<commit_before>package linux_backend

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sync"

	"github.com/cloudfoundry-incubator/garden/backend"
)

type Container interface {
	Snapshot(io.Writer) error
	Cleanup()

	backend.Container
}

type ContainerPool interface {
	Setup() error
	Create(backend.ContainerSpec) (Container, error)
	Restore(io.Reader) (Container, error)
	Destroy(Container) error
	Prune(keep map[string]bool) error
}

type LinuxBackend struct {
	containerPool ContainerPool
	snapshotsPath string

	containers      map[string]Container
	containersMutex *sync.RWMutex
}

type UnknownHandleError struct {
	Handle string
}

func (e UnknownHandleError) Error() string {
	return "unknown handle: " + e.Handle
}

type FailedToSnapshotError struct {
	OriginalError error
}

func (e FailedToSnapshotError) Error() string {
	return fmt.Sprintf("failed to save snapshot: %s", e.OriginalError)
}

func New(containerPool ContainerPool, snapshotsPath string) *LinuxBackend {
	return &LinuxBackend{
		containerPool: containerPool,
		snapshotsPath: snapshotsPath,

		containers:      make(map[string]Container),
		containersMutex: new(sync.RWMutex),
	}
}

func (b *LinuxBackend) Setup() error {
	return b.containerPool.Setup()
}

func (b *LinuxBackend) Start() error {
	if b.snapshotsPath != "" {
		_, err := os.Stat(b.snapshotsPath)
		if err == nil {
			err = b.restoreSnapshots()
			if err != nil {
				return err
			}

			os.RemoveAll(b.snapshotsPath)
		}

		err = os.MkdirAll(b.snapshotsPath, 0755)
		if err != nil {
			return err
		}
	}

	keep := map[string]bool{}

	containers, err := b.Containers()
	if err != nil {
		return err
	}

	for _, container := range containers {
		keep[container.ID()] = true
	}

	return b.containerPool.Prune(keep)
}

func (b *LinuxBackend) Create(spec backend.ContainerSpec) (backend.Container, error) {
	container, err := b.containerPool.Create(spec)
	if err != nil {
		return nil, err
	}

	err = container.Start()
	if err != nil {
		return nil, err
	}

	b.containersMutex.Lock()

	b.containers[container.Handle()] = container

	b.containersMutex.Unlock()

	return container, nil
}

func (b *LinuxBackend) Destroy(handle string) error {
	container, found := b.containers[handle]
	if !found {
		return UnknownHandleError{handle}
	}

	err := b.containerPool.Destroy(container)
	if err != nil {
		return err
	}

	b.containersMutex.Lock()

	delete(b.containers, container.Handle())

	b.containersMutex.Unlock()

	return nil
}

func (b *LinuxBackend) Containers() (containers []backend.Container, err error) {
	b.containersMutex.RLock()
	defer b.containersMutex.RUnlock()

	for _, container := range b.containers {
		containers = append(containers, container)
	}

	return containers, nil
}

func (b *LinuxBackend) Lookup(handle string) (backend.Container, error) {
	b.containersMutex.RLock()
	defer b.containersMutex.RUnlock()

	container, found := b.containers[handle]
	if !found {
		return nil, UnknownHandleError{handle}
	}

	return container, nil
}

func (b *LinuxBackend) Stop() {
	b.containersMutex.RLock()
	defer b.containersMutex.RUnlock()

	for _, container := range b.containers {
		container.Cleanup()
		b.saveSnapshot(container)
	}
}

func (b *LinuxBackend) restoreSnapshots() error {
	entries, err := ioutil.ReadDir(b.snapshotsPath)
	if err != nil {
		return err
	}

	for _, entry := range entries {
		snapshot := path.Join(b.snapshotsPath, entry.Name())

		log.Println("loading snapshot for", entry.Name())

		file, err := os.Open(snapshot)
		if err != nil {
			return err
		}

		_, err = b.restore(file)
		if err != nil {
			return err
		}
	}

	return nil
}

func (b *LinuxBackend) saveSnapshot(container Container) error {
	if b.snapshotsPath == "" {
		return nil
	}

	log.Println("saving snapshot for", container.ID())

	tmpfile, err := ioutil.TempFile(os.TempDir(), "snapshot-"+container.ID())
	if err != nil {
		return &FailedToSnapshotError{err}
	}

	err = container.Snapshot(tmpfile)
	if err != nil {
		return &FailedToSnapshotError{err}
	}

	snapshotPath := path.Join(b.snapshotsPath, container.ID())

	err = os.Rename(tmpfile.Name(), snapshotPath)
	if err != nil {
		return &FailedToSnapshotError{err}
	}

	return nil
}

func (b *LinuxBackend) restore(snapshot io.Reader) (backend.Container, error) {
	container, err := b.containerPool.Restore(snapshot)
	if err != nil {
		return nil, err
	}

	b.containersMutex.Lock()

	b.containers[container.Handle()] = container

	b.containersMutex.Unlock()

	return container, nil
}
<commit_msg>add Start() to linux backend container interface<commit_after>package linux_backend

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sync"

	"github.com/cloudfoundry-incubator/garden/backend"
)

type Container interface {
	Start() error

	Snapshot(io.Writer) error
	Cleanup()

	backend.Container
}

type ContainerPool interface {
	Setup() error
	Create(backend.ContainerSpec) (Container, error)
	Restore(io.Reader) (Container, error)
	Destroy(Container) error
	Prune(keep map[string]bool) error
}

type LinuxBackend struct {
	containerPool ContainerPool
	snapshotsPath string

	containers      map[string]Container
	containersMutex *sync.RWMutex
}

type UnknownHandleError struct {
	Handle string
}

func (e UnknownHandleError) Error() string {
	return "unknown handle: " + e.Handle
}

type FailedToSnapshotError struct {
	OriginalError error
}

func (e FailedToSnapshotError) Error() string {
	return fmt.Sprintf("failed to save snapshot: %s", e.OriginalError)
}

func New(containerPool ContainerPool, snapshotsPath string) *LinuxBackend {
	return &LinuxBackend{
		containerPool: containerPool,
		snapshotsPath: snapshotsPath,

		containers:      make(map[string]Container),
		containersMutex: new(sync.RWMutex),
	}
}

func (b *LinuxBackend) Setup() error {
	return b.containerPool.Setup()
}

func (b *LinuxBackend) Start() error {
	if b.snapshotsPath != "" {
		_, err := os.Stat(b.snapshotsPath)
		if err == nil {
			err = b.restoreSnapshots()
			if err != nil {
				return err
			}

			os.RemoveAll(b.snapshotsPath)
		}

		err = os.MkdirAll(b.snapshotsPath, 0755)
		if err != nil {
			return err
		}
	}

	keep := map[string]bool{}

	containers, err := b.Containers()
	if err != nil {
		return err
	}

	for _, container := range containers {
		keep[container.ID()] = true
	}

	return b.containerPool.Prune(keep)
}

func (b *LinuxBackend) Create(spec backend.ContainerSpec) (backend.Container, error) {
	container, err := b.containerPool.Create(spec)
	if err != nil {
		return nil, err
	}

	err = container.Start()
	if err != nil {
		return nil, err
	}

	b.containersMutex.Lock()

	b.containers[container.Handle()] = container

	b.containersMutex.Unlock()

	return container, nil
}

func (b *LinuxBackend) Destroy(handle string) error {
	container, found := b.containers[handle]
	if !found {
		return UnknownHandleError{handle}
	}

	err := b.containerPool.Destroy(container)
	if err != nil {
		return err
	}

	b.containersMutex.Lock()

	delete(b.containers, container.Handle())

	b.containersMutex.Unlock()

	return nil
}

func (b *LinuxBackend) Containers() (containers []backend.Container, err error) {
	b.containersMutex.RLock()
	defer b.containersMutex.RUnlock()

	for _, container := range b.containers {
		containers = append(containers, container)
	}

	return containers, nil
}

func (b *LinuxBackend) Lookup(handle string) (backend.Container, error) {
	b.containersMutex.RLock()
	defer b.containersMutex.RUnlock()

	container, found := b.containers[handle]
	if !found {
		return nil, UnknownHandleError{handle}
	}

	return container, nil
}

func (b *LinuxBackend) Stop() {
	b.containersMutex.RLock()
	defer b.containersMutex.RUnlock()

	for _, container := range b.containers {
		container.Cleanup()
		b.saveSnapshot(container)
	}
}

func (b *LinuxBackend) restoreSnapshots() error {
	entries, err := ioutil.ReadDir(b.snapshotsPath)
	if err != nil {
		return err
	}

	for _, entry := range entries {
		snapshot := path.Join(b.snapshotsPath, entry.Name())

		log.Println("loading snapshot for", entry.Name())

		file, err := os.Open(snapshot)
		if err != nil {
			return err
		}

		_, err = b.restore(file)
		if err != nil {
			return err
		}
	}

	return nil
}

func (b *LinuxBackend) saveSnapshot(container Container) error {
	if b.snapshotsPath == "" {
		return nil
	}

	log.Println("saving snapshot for", container.ID())

	tmpfile, err := ioutil.TempFile(os.TempDir(), "snapshot-"+container.ID())
	if err != nil {
		return &FailedToSnapshotError{err}
	}

	err = container.Snapshot(tmpfile)
	if err != nil {
		return &FailedToSnapshotError{err}
	}

	snapshotPath := path.Join(b.snapshotsPath, container.ID())

	err = os.Rename(tmpfile.Name(), snapshotPath)
	if err != nil {
		return &FailedToSnapshotError{err}
	}

	return nil
}

func (b *LinuxBackend) restore(snapshot io.Reader) (backend.Container, error) {
	container, err := b.containerPool.Restore(snapshot)
	if err != nil {
		return nil, err
	}

	b.containersMutex.Lock()

	b.containers[container.Handle()] = container

	b.containersMutex.Unlock()

	return container, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"

	"github.com/codegangsta/negroni"
	"github.com/xyproto/middleskel"
)

func main() {
	mux := http.NewServeMux()

	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		fmt.Fprintf(w, "Bafflesnark!")
	})

	n := negroni.Classic()

	// Moose status
	n.Use(moose.NewMiddleware())

	// Handler goes last
	n.UseHandler(mux)

	// Serve
	n.Run(":9030")
}
<commit_msg>Fix the example<commit_after>package main

import (
	"fmt"
	"net/http"

	"github.com/codegangsta/negroni"
	"github.com/xyproto/mooseware"
)

func main() {
	mux := http.NewServeMux()

	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		fmt.Fprintf(w, "Bafflesnark!")
	})

	n := negroni.Classic()

	// Moose status
	n.Use(moose.NewMiddleware())

	// Handler goes last
	n.UseHandler(mux)

	// Serve
	n.Run(":9030")
}
<|endoftext|>


<commit_before>package http

import (
	"encoding/json"
	"fmt"
	"github.com/getlantern/flashlight/client"
	"github.com/getlantern/flashlight/config"
	"github.com/getlantern/flashlight/util"
	"github.com/getlantern/golog"
	"github.com/getlantern/whitelist"
	"github.com/skratchdot/open-golang/open"
	"net/http"
	"reflect"
	"time"

	"github.com/getlantern/tarfs"
)

const (
	UiUrl = "http://%s"
	UiDir = "src/github.com/getlantern/ui/app"
)

var (
	log = golog.LoggerFor("http")
)

type JsonResponse struct {
	Error     string   `json:"Error, omitempty"`
	Whitelist []string `json:"Whitelist, omitempty"`
	Original  []string `json:"Original, omitempty"`
}

type WhitelistHandler struct {
	http.HandlerFunc
	whitelist *whitelist.Whitelist
	wlChan    chan *whitelist.Config
}

func sendJsonResponse(w http.ResponseWriter, response *JsonResponse, indent bool) {

	enc := json.NewEncoder(w)
	err := enc.Encode(response)
	if err != nil {
		log.Errorf("error sending json response %v", err)
	}
}

func setResponseHeaders(w http.ResponseWriter) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token")
	w.Header().Set("Access-Control-Allow-Credentials", "True")
	w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")

}

func (wlh WhitelistHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var response JsonResponse

	setResponseHeaders(w)

	switch r.Method {
	case "OPTIONS":
		// return if it's a preflight OPTIONS request
		// this is mainly for testing the UI when it's running on
		// a separate port
		return
	case "POST":
		// update whitelist
		decoder := json.NewDecoder(r.Body)
		var entries []string
		err := decoder.Decode(&entries)
		if err != nil {
			log.Error(err)
			response.Error = fmt.Sprintf("Error decoding whitelist: %q", err)
		} else {
			wl := wlh.whitelist.UpdateEntries(entries)
			copy := wlh.whitelist.Copy()
			log.Debug("Propagating whitelist changes..")
			wlh.wlChan <- copy
			response.Whitelist = wl
		}
	case "GET":
		response.Whitelist = wlh.whitelist.GetEntries()
	default:
		log.Debugf("Received %s", response.Error)
		response.Error = "Invalid whitelist HTTP request"
		response.Whitelist = nil
	}
	sendJsonResponse(w, &response, false)
}

func servePacFile(w http.ResponseWriter, r *http.Request) {
	pacFile := whitelist.GetPacFile()
	http.ServeFile(w, r, pacFile)
}

func UiHttpServer(cfg *client.ClientConfig, cfgChan chan *config.Config, wlChan chan *whitelist.Config) error {

	wlh := &WhitelistHandler{
		whitelist: whitelist.New(cfg.Whitelist),
		wlChan:    wlChan,
	}

	r := http.NewServeMux()
	r.Handle("/whitelist", wlh)

	// poll for config updates to the whitelist
	// with this immediately see flashlight.yaml
	// changes in the UI
	go func() {
		for {
			newCfg := <-cfgChan
			clientCfg := newCfg.Client
			if !reflect.DeepEqual(wlh.whitelist.GetConfig(), clientCfg.Whitelist) {
				log.Debugf("Whitelist changed in flashlight.yaml..")
				wlh.whitelist = whitelist.New(newCfg.Client.Whitelist)
				wlh.whitelist.RefreshEntries()
			}
		}
	}()
	r.HandleFunc("/proxy_on.pac", servePacFile)

	UiDirExists, err := util.DirExists(UiDir)
	if err != nil {
		log.Debugf("UI Directory does not exist %s", err)
	}

	if UiDirExists {
		// UI directory found--serve assets directly from it
		log.Debugf("Serving UI assets from directory %s", UiDir)
		r.Handle("/", http.FileServer(http.Dir(UiDir)))
	} else {
		start := time.Now()
		fs, err := tarfs.New(Resources, "../ui/app")
		if err != nil {
			panic(err)
		}
		delta := time.Now().Sub(start)
		log.Debugf("tarfs startup time: %v", delta)
		r.Handle("/", http.FileServer(fs))
	}

	httpServer := &http.Server{
		Addr:    cfg.UiAddr,
		Handler: r,
	}

	log.Debugf("Starting UI HTTP server at %s", cfg.UiAddr)
	uiAddr := fmt.Sprintf(UiUrl, cfg.UiAddr)

	if cfg.OpenUi {
		err = open.Run(uiAddr)
		if err != nil {
			log.Errorf("Could not open UI! %s", err)
			return err
		}
	}

	err = httpServer.ListenAndServe()
	if err != nil {
		log.Errorf("Could not start HTTP server! %s", err)
		return err
	}
	return err
}
<commit_msg>remove unused LoadTemplate function<commit_after>package http

import (
	"encoding/json"
	"fmt"
	"github.com/getlantern/flashlight/client"
	"github.com/getlantern/flashlight/config"
	"github.com/getlantern/flashlight/util"
	"github.com/getlantern/golog"
	"github.com/getlantern/whitelist"
	"github.com/skratchdot/open-golang/open"
	"net/http"
	"reflect"
	"time"

	"github.com/getlantern/tarfs"
)

const (
	UiUrl = "http://%s"
	UiDir = "src/github.com/getlantern/ui/app"
)

var (
	log = golog.LoggerFor("http")
)

type JsonResponse struct {
	Error     string   `json:"Error, omitempty"`
	Whitelist []string `json:"Whitelist, omitempty"`
	Global    []string `json:"Original, omitempty"`
}

type WhitelistHandler struct {
	http.HandlerFunc
	whitelist *whitelist.Whitelist
	wlChan    chan *whitelist.Config
}

func sendJsonResponse(w http.ResponseWriter, response *JsonResponse, indent bool) {

	enc := json.NewEncoder(w)
	err := enc.Encode(response)
	if err != nil {
		log.Errorf("error sending json response %v", err)
	}
}

func setResponseHeaders(w http.ResponseWriter) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token")
	w.Header().Set("Access-Control-Allow-Credentials", "True")
	w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")

}

func (wlh WhitelistHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var response JsonResponse

	setResponseHeaders(w)

	switch r.Method {
	case "OPTIONS":
		// return if it's a preflight OPTIONS request
		// this is mainly for testing the UI when it's running on
		// a separate port
		return
	case "POST":
		// update whitelist
		decoder := json.NewDecoder(r.Body)
		var entries []string
		err := decoder.Decode(&entries)
		if err != nil {
			log.Error(err)
			response.Error = fmt.Sprintf("Error decoding whitelist: %q", err)
		} else {
			wl := wlh.whitelist.UpdateEntries(entries)
			copy := wlh.whitelist.Copy()
			log.Debug("Propagating whitelist changes..")
			wlh.wlChan <- copy
			response.Whitelist = wl
		}
	case "GET":
		response.Whitelist = wlh.whitelist.GetEntries()
		response.Global = wlh.whitelist.GetGlobalList()
	default:
		log.Debugf("Received %s", response.Error)
		response.Error = "Invalid whitelist HTTP request"
		response.Whitelist = nil
	}
	sendJsonResponse(w, &response, false)
}

func servePacFile(w http.ResponseWriter, r *http.Request) {
	pacFile := whitelist.GetPacFile()
	http.ServeFile(w, r, pacFile)
}

func UiHttpServer(cfg *client.ClientConfig, cfgChan chan *config.Config, wlChan chan *whitelist.Config) error {

	wlh := &WhitelistHandler{
		whitelist: whitelist.New(cfg.Whitelist),
		wlChan:    wlChan,
	}

	r := http.NewServeMux()
	r.Handle("/whitelist", wlh)

	// poll for config updates to the whitelist
	// with this immediately see flashlight.yaml
	// changes in the UI
	go func() {
		for {
			newCfg := <-cfgChan
			clientCfg := newCfg.Client
			if !reflect.DeepEqual(wlh.whitelist.GetConfig(), clientCfg.Whitelist) {
				log.Debugf("Whitelist changed in flashlight.yaml..")
				wlh.whitelist = whitelist.New(newCfg.Client.Whitelist)
				wlh.whitelist.RefreshEntries()
			}
		}
	}()
	r.HandleFunc("/proxy_on.pac", servePacFile)

	UiDirExists, err := util.DirExists(UiDir)
	if err != nil {
		log.Debugf("UI Directory does not exist %s", err)
	}

	if UiDirExists {
		// UI directory found--serve assets directly from it
		log.Debugf("Serving UI assets from directory %s", UiDir)
		r.Handle("/", http.FileServer(http.Dir(UiDir)))
	} else {
		start := time.Now()
		fs, err := tarfs.New(Resources, "../ui/app")
		if err != nil {
			panic(err)
		}
		delta := time.Now().Sub(start)
		log.Debugf("tarfs startup time: %v", delta)
		r.Handle("/", http.FileServer(fs))
	}

	httpServer := &http.Server{
		Addr:    cfg.UiAddr,
		Handler: r,
	}

	log.Debugf("Starting UI HTTP server at %s", cfg.UiAddr)
	uiAddr := fmt.Sprintf(UiUrl, cfg.UiAddr)

	if cfg.OpenUi {
		err = open.Run(uiAddr)
		if err != nil {
			log.Errorf("Could not open UI! %s", err)
			return err
		}
	}

	err = httpServer.ListenAndServe()
	if err != nil {
		log.Errorf("Could not start HTTP server! %s", err)
		return err
	}
	return err
}
<|endoftext|>