<commit_before>package rpc

import (
	"io"
	"net/rpc"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/job"
	"github.com/weaveworks/flux/remote"
	"github.com/weaveworks/flux/update"
)

// RPCClient is the rpc-backed implementation of a platform, for
// talking to remote daemons.
type RPCClientV6 struct {
	*RPCClientV5
}

var _ remote.PlatformV6 = &RPCClientV6{}

// NewClient creates a new rpc-backed implementation of the platform.
func NewClientV6(conn io.ReadWriteCloser) *RPCClientV6 {
	return &RPCClientV6{NewClientV5(conn)}
}

// Export is used to get service configuration in platform-specific format
func (p *RPCClientV6) Export() ([]byte, error) {
	var config []byte
	err := p.client.Call("RPCServer.Export", struct{}{}, &config)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return nil, remote.FatalError{err}
	}
	return config, err
}

// Export is used to get service configuration in platform-specific format
func (p *RPCClientV6) ListServices(namespace string) ([]flux.ServiceStatus, error) {
	var services []flux.ServiceStatus
	err := p.client.Call("RPCServer.ListServices", namespace, &services)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return nil, remote.FatalError{err}
	}
	return services, err
}

func (p *RPCClientV6) ListImages(spec update.ServiceSpec) ([]flux.ImageStatus, error) {
	var images []flux.ImageStatus
	err := p.client.Call("RPCServer.ListImages", spec, &images)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return nil, remote.FatalError{err}
	}
	return images, err
}

func (p *RPCClientV6) UpdateManifests(u update.Spec) (job.ID, error) {
	var result job.ID
	err := p.client.Call("RPCServer.UpdateManifests", u, &result)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return result, remote.FatalError{err}
	}
	return result, err
}

func (p *RPCClientV6) SyncNotify() error {
	var result struct{}
	err := p.client.Call("RPCServer.SyncNotify", struct{}{}, &result)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return remote.FatalError{err}
	}
	return err
}

func (p *RPCClientV6) JobStatus(jobID job.ID) (job.Status, error) {
	var result job.Status
	err := p.client.Call("RPCServer.JobStatus", jobID, &result)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return job.Status{}, remote.FatalError{err}
	}
	return result, err
}

func (p *RPCClientV6) SyncStatus(ref string) ([]string, error) {
	var result []string
	err := p.client.Call("RPCServer.SyncStatus", ref, &result)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return nil, remote.FatalError{err}
	}
	return result, err
}

func (p *RPCClientV6) GitRepoConfig(regenerate bool) (flux.GitConfig, error) {
	var result flux.GitConfig
	err := p.client.Call("RPCServer.GitRepoConfig", regenerate, &result)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return flux.GitConfig{}, remote.FatalError{err}
	}
	return result, err
}
<commit_msg>Treat non-fatal v6 errors as application errors<commit_after>package rpc

import (
	"io"
	"net/rpc"

	"github.com/weaveworks/flux"
	fluxerr "github.com/weaveworks/flux/errors"
	"github.com/weaveworks/flux/job"
	"github.com/weaveworks/flux/remote"
	"github.com/weaveworks/flux/update"
)

// RPCClient is the rpc-backed implementation of a platform, for
// talking to remote daemons.
type RPCClientV6 struct {
	*RPCClientV5
}

// We don't get proper application error structs back from v6, but we
// do know that anything that's not considered a fatal error can be
// translated into an application error.
func remoteApplicationError(err error) error {
	return &fluxerr.Error{
		Type: fluxerr.User,
		Err:  err,
		Help: `Error from daemon

The daemon (fluxd, running in your cluster) reported this error when
attempting to fulfil your request:

    ` + err.Error() + `
`,
	}
}

var _ remote.PlatformV6 = &RPCClientV6{}

// NewClient creates a new rpc-backed implementation of the platform.
func NewClientV6(conn io.ReadWriteCloser) *RPCClientV6 {
	return &RPCClientV6{NewClientV5(conn)}
}

// Export is used to get service configuration in platform-specific format
func (p *RPCClientV6) Export() ([]byte, error) {
	var config []byte
	err := p.client.Call("RPCServer.Export", struct{}{}, &config)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return nil, remote.FatalError{err}
	}
	if err != nil {
		err = remoteApplicationError(err)
	}
	return config, err
}

// Export is used to get service configuration in platform-specific format
func (p *RPCClientV6) ListServices(namespace string) ([]flux.ServiceStatus, error) {
	var services []flux.ServiceStatus
	err := p.client.Call("RPCServer.ListServices", namespace, &services)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return nil, remote.FatalError{err}
	}
	if err != nil {
		err = remoteApplicationError(err)
	}
	return services, err
}

func (p *RPCClientV6) ListImages(spec update.ServiceSpec) ([]flux.ImageStatus, error) {
	var images []flux.ImageStatus
	err := p.client.Call("RPCServer.ListImages", spec, &images)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return nil, remote.FatalError{err}
	}
	if err != nil {
		err = remoteApplicationError(err)
	}
	return images, err
}

func (p *RPCClientV6) UpdateManifests(u update.Spec) (job.ID, error) {
	var result job.ID
	err := p.client.Call("RPCServer.UpdateManifests", u, &result)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return result, remote.FatalError{err}
	}
	if err != nil {
		err = remoteApplicationError(err)
	}
	return result, err
}

func (p *RPCClientV6) SyncNotify() error {
	var result struct{}
	err := p.client.Call("RPCServer.SyncNotify", struct{}{}, &result)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return remote.FatalError{err}
	}
	if err != nil {
		err = remoteApplicationError(err)
	}
	return err
}

func (p *RPCClientV6) JobStatus(jobID job.ID) (job.Status, error) {
	var result job.Status
	err := p.client.Call("RPCServer.JobStatus", jobID, &result)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return job.Status{}, remote.FatalError{err}
	}
	if err != nil {
		err = remoteApplicationError(err)
	}
	return result, err
}

func (p *RPCClientV6) SyncStatus(ref string) ([]string, error) {
	var result []string
	err := p.client.Call("RPCServer.SyncStatus", ref, &result)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return nil, remote.FatalError{err}
	}
	if err != nil {
		err = remoteApplicationError(err)
	}
	return result, err
}

func (p *RPCClientV6) GitRepoConfig(regenerate bool) (flux.GitConfig, error) {
	var result flux.GitConfig
	err := p.client.Call("RPCServer.GitRepoConfig", regenerate, &result)
	if _, ok := err.(rpc.ServerError); !ok && err != nil {
		return flux.GitConfig{}, remote.FatalError{err}
	}
	if err != nil {
		err = remoteApplicationError(err)
	}
	return result, err
}
<|endoftext|>


<commit_before>package expr

import (
	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/consolidation"
)

// Context describes a series timeframe and consolidator
type Context struct {
	from   uint32
	to     uint32
	consol consolidation.Consolidator // can be 0 to mean undefined
}

// GraphiteFunc defines a graphite processing function
type GraphiteFunc interface {
	// Signature declares input and output arguments (return values)
	// input args can be optional in which case they can be specified positionally or via keys if you want to specify params that come after un-specified optional params
	// the val pointers of each input Arg should point to a location accessible to the function,
	// so that the planner can set up the inputs for your function based on user input.
	// NewPlan() will only create the plan if the expressions it parsed correspond to the signatures provided by the function
	Signature() ([]Arg, []Arg)

	// Context allows a func to alter the context that will be passed down the expression tree.
	// this function will be called after validating and setting up all non-series and non-serieslist parameters.
	// (as typically, context alterations require integer/string/bool/etc parameters, and shall affect series[list] parameters)
	// examples:
	// * movingAverage(foo,5min) -> the 5min arg will be parsed, so we can request 5min of earlier data, which will affect the request for foo.
	// * consolidateBy(bar, "sum") -> the "sum" arg will be parsed, so we can pass on the fact that bar needs to be sum-consolidated
	Context(c Context) Context
	// Exec executes the function. the function should call any input functions, do its processing, and return output.
	// IMPORTANT: for performance and correctness, functions should
	// * not modify slices of points that they get from their inputs
	// * use the pool to get new slices in which to store any new/modified dat
	// * add the newly created slices into the cache so they can be reclaimed after the output is consumed
	Exec(map[Req][]models.Series) ([]models.Series, error)
}

type funcConstructor func() GraphiteFunc

type funcDef struct {
	constr funcConstructor
	stable bool
}

var funcs map[string]funcDef

func init() {
	// keys must be sorted alphabetically. but functions with aliases can go together, in which case they are sorted by the first of their aliases
	funcs = map[string]funcDef{
		"alias":                 {NewAlias, true},
		"aliasByTags":           {NewAliasByNode, true},
		"aliasByNode":           {NewAliasByNode, true},
		"aliasSub":              {NewAliasSub, true},
		"asPercent":             {NewAsPercent, true},
		"avg":                   {NewAggregateConstructor("average", crossSeriesAvg), true},
		"averageSeries":         {NewAggregateConstructor("average", crossSeriesAvg), true},
		"consolidateBy":         {NewConsolidateBy, true},
		"countSeries":           {NewCountSeries, true},
		"derivative":            {NewDerivative, true},
		"diffSeries":            {NewAggregateConstructor("diff", crossSeriesDiff), true},
		"divideSeries":          {NewDivideSeries, true},
		"divideSeriesLists":     {NewDivideSeriesLists, true},
		"exclude":               {NewExclude, true},
		"filterSeries":          {NewFilterSeries, true},
		"grep":                  {NewGrep, true},
		"groupByTags":           {NewGroupByTags, true},
		"highest":               {NewHighestLowestConstructor("", true), true},
		"highestAverage":        {NewHighestLowestConstructor("average", true), true},
		"highestCurrent":        {NewHighestLowestConstructor("current", true), true},
		"highestMax":            {NewHighestLowestConstructor("max", true), true},
		"isNonNull":             {NewIsNonNull, true},
		"lowest":                {NewHighestLowestConstructor("", false), true},
		"lowestAverage":         {NewHighestLowestConstructor("average", false), true},
		"lowestCurrent":         {NewHighestLowestConstructor("current", false), true},
		"max":                   {NewAggregateConstructor("max", crossSeriesMax), true},
		"maxSeries":             {NewAggregateConstructor("max", crossSeriesMax), true},
		"min":                   {NewAggregateConstructor("min", crossSeriesMin), true},
		"minSeries":             {NewAggregateConstructor("min", crossSeriesMin), true},
		"multiplySeries":        {NewAggregateConstructor("multiply", crossSeriesMultiply), true},
		"movingAverage":         {NewMovingAverage, false},
		"nonNegativeDerivative": {NewNonNegativeDerivative, true},
		"perSecond":             {NewPerSecond, true},
		"rangeOfSeries":         {NewAggregateConstructor("rangeOf", crossSeriesRange), true},
		"removeAboveValue":      {NewRemoveAboveBelowValueConstructor(true), true},
		"removeBelowValue":      {NewRemoveAboveBelowValueConstructor(false), true},
		"scale":                 {NewScale, true},
		"scaleToSeconds":        {NewScaleToSeconds, true},
		"smartSummarize":        {NewSmartSummarize, false},
    "sortBy":                {NewSortByConstructor("", false), true},
		"sortByMaxima":          {NewSortByConstructor("max", true), true},
		"sortByName":            {NewSortByName, true},
		"sortByTotal":           {NewSortByConstructor("sum", true), true},
		"stddevSeries":          {NewAggregateConstructor("stddev", crossSeriesStddev), true},
		"sum":                   {NewAggregateConstructor("sum", crossSeriesSum), true},
		"sumSeries":             {NewAggregateConstructor("sum", crossSeriesSum), true},
		"summarize":             {NewSummarize, true},
		"transformNull":         {NewTransformNull, true},
	}
}

// summarizeCons returns the first explicitly specified Consolidator, QueryCons for the given set of input series,
// or the first one, otherwise.
func summarizeCons(series []models.Series) (consolidation.Consolidator, consolidation.Consolidator) {
	for _, serie := range series {
		if serie.QueryCons != 0 {
			return serie.Consolidator, serie.QueryCons
		}
	}
	return series[0].Consolidator, series[0].QueryCons
}

func consumeFuncs(cache map[Req][]models.Series, fns []GraphiteFunc) ([]models.Series, []string, error) {
	var series []models.Series
	var queryPatts []string
	for i := range fns {
		in, err := fns[i].Exec(cache)
		if err != nil {
			return nil, nil, err
		}
		if len(in) != 0 {
			series = append(series, in...)
			queryPatts = append(queryPatts, in[0].QueryPatt)
		}
	}
	return series, queryPatts, nil
}
<commit_msg>gofmt fix<commit_after>package expr

import (
	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/consolidation"
)

// Context describes a series timeframe and consolidator
type Context struct {
	from   uint32
	to     uint32
	consol consolidation.Consolidator // can be 0 to mean undefined
}

// GraphiteFunc defines a graphite processing function
type GraphiteFunc interface {
	// Signature declares input and output arguments (return values)
	// input args can be optional in which case they can be specified positionally or via keys if you want to specify params that come after un-specified optional params
	// the val pointers of each input Arg should point to a location accessible to the function,
	// so that the planner can set up the inputs for your function based on user input.
	// NewPlan() will only create the plan if the expressions it parsed correspond to the signatures provided by the function
	Signature() ([]Arg, []Arg)

	// Context allows a func to alter the context that will be passed down the expression tree.
	// this function will be called after validating and setting up all non-series and non-serieslist parameters.
	// (as typically, context alterations require integer/string/bool/etc parameters, and shall affect series[list] parameters)
	// examples:
	// * movingAverage(foo,5min) -> the 5min arg will be parsed, so we can request 5min of earlier data, which will affect the request for foo.
	// * consolidateBy(bar, "sum") -> the "sum" arg will be parsed, so we can pass on the fact that bar needs to be sum-consolidated
	Context(c Context) Context
	// Exec executes the function. the function should call any input functions, do its processing, and return output.
	// IMPORTANT: for performance and correctness, functions should
	// * not modify slices of points that they get from their inputs
	// * use the pool to get new slices in which to store any new/modified dat
	// * add the newly created slices into the cache so they can be reclaimed after the output is consumed
	Exec(map[Req][]models.Series) ([]models.Series, error)
}

type funcConstructor func() GraphiteFunc

type funcDef struct {
	constr funcConstructor
	stable bool
}

var funcs map[string]funcDef

func init() {
	// keys must be sorted alphabetically. but functions with aliases can go together, in which case they are sorted by the first of their aliases
	funcs = map[string]funcDef{
		"alias":                 {NewAlias, true},
		"aliasByTags":           {NewAliasByNode, true},
		"aliasByNode":           {NewAliasByNode, true},
		"aliasSub":              {NewAliasSub, true},
		"asPercent":             {NewAsPercent, true},
		"avg":                   {NewAggregateConstructor("average", crossSeriesAvg), true},
		"averageSeries":         {NewAggregateConstructor("average", crossSeriesAvg), true},
		"consolidateBy":         {NewConsolidateBy, true},
		"countSeries":           {NewCountSeries, true},
		"derivative":            {NewDerivative, true},
		"diffSeries":            {NewAggregateConstructor("diff", crossSeriesDiff), true},
		"divideSeries":          {NewDivideSeries, true},
		"divideSeriesLists":     {NewDivideSeriesLists, true},
		"exclude":               {NewExclude, true},
		"filterSeries":          {NewFilterSeries, true},
		"grep":                  {NewGrep, true},
		"groupByTags":           {NewGroupByTags, true},
		"highest":               {NewHighestLowestConstructor("", true), true},
		"highestAverage":        {NewHighestLowestConstructor("average", true), true},
		"highestCurrent":        {NewHighestLowestConstructor("current", true), true},
		"highestMax":            {NewHighestLowestConstructor("max", true), true},
		"isNonNull":             {NewIsNonNull, true},
		"lowest":                {NewHighestLowestConstructor("", false), true},
		"lowestAverage":         {NewHighestLowestConstructor("average", false), true},
		"lowestCurrent":         {NewHighestLowestConstructor("current", false), true},
		"max":                   {NewAggregateConstructor("max", crossSeriesMax), true},
		"maxSeries":             {NewAggregateConstructor("max", crossSeriesMax), true},
		"min":                   {NewAggregateConstructor("min", crossSeriesMin), true},
		"minSeries":             {NewAggregateConstructor("min", crossSeriesMin), true},
		"multiplySeries":        {NewAggregateConstructor("multiply", crossSeriesMultiply), true},
		"movingAverage":         {NewMovingAverage, false},
		"nonNegativeDerivative": {NewNonNegativeDerivative, true},
		"perSecond":             {NewPerSecond, true},
		"rangeOfSeries":         {NewAggregateConstructor("rangeOf", crossSeriesRange), true},
		"removeAboveValue":      {NewRemoveAboveBelowValueConstructor(true), true},
		"removeBelowValue":      {NewRemoveAboveBelowValueConstructor(false), true},
		"scale":                 {NewScale, true},
		"scaleToSeconds":        {NewScaleToSeconds, true},
		"smartSummarize":        {NewSmartSummarize, false},
		"sortBy":                {NewSortByConstructor("", false), true},
		"sortByMaxima":          {NewSortByConstructor("max", true), true},
		"sortByName":            {NewSortByName, true},
		"sortByTotal":           {NewSortByConstructor("sum", true), true},
		"stddevSeries":          {NewAggregateConstructor("stddev", crossSeriesStddev), true},
		"sum":                   {NewAggregateConstructor("sum", crossSeriesSum), true},
		"sumSeries":             {NewAggregateConstructor("sum", crossSeriesSum), true},
		"summarize":             {NewSummarize, true},
		"transformNull":         {NewTransformNull, true},
	}
}

// summarizeCons returns the first explicitly specified Consolidator, QueryCons for the given set of input series,
// or the first one, otherwise.
func summarizeCons(series []models.Series) (consolidation.Consolidator, consolidation.Consolidator) {
	for _, serie := range series {
		if serie.QueryCons != 0 {
			return serie.Consolidator, serie.QueryCons
		}
	}
	return series[0].Consolidator, series[0].QueryCons
}

func consumeFuncs(cache map[Req][]models.Series, fns []GraphiteFunc) ([]models.Series, []string, error) {
	var series []models.Series
	var queryPatts []string
	for i := range fns {
		in, err := fns[i].Exec(cache)
		if err != nil {
			return nil, nil, err
		}
		if len(in) != 0 {
			series = append(series, in...)
			queryPatts = append(queryPatts, in[0].QueryPatt)
		}
	}
	return series, queryPatts, nil
}
<|endoftext|>


<commit_before>package expr

import (
	"fmt"
	"math"

	"github.com/StackExchange/tcollector/opentsdb"
	"github.com/StackExchange/tsaf/expr/parse"
)

const (
	DefDuration = "1h"
	DefPeriod   = "1w"
	DefNum      = 8
)

var Builtins = map[string]parse.Func{
	"avg": {
		[]parse.FuncType{parse.TYPE_SERIES, parse.TYPE_STRING},
		parse.TYPE_NUMBER,
		[]interface{}{DefDuration},
		Avg,
	},
	"band": {
		[]parse.FuncType{parse.TYPE_QUERY, parse.TYPE_STRING, parse.TYPE_STRING, parse.TYPE_NUMBER},
		parse.TYPE_SERIES,
		[]interface{}{DefDuration, DefPeriod, DefNum},
		band,
	},
	"dev": {
		[]parse.FuncType{parse.TYPE_SERIES, parse.TYPE_STRING},
		parse.TYPE_NUMBER,
		[]interface{}{DefDuration},
		Dev,
	},
	"recent": {
		[]parse.FuncType{parse.TYPE_SERIES, parse.TYPE_STRING},
		parse.TYPE_NUMBER,
		[]interface{}{DefDuration},
		recent,
	},
}

func queryDuration(query, duration string, F func([]float64) float64) (r []*Result, err error) {
	q, err := opentsdb.ParseQuery(query)
	if err != nil {
		return
	}
	d, err := ParseDuration(duration)
	if err != nil {
		return
	}
	req := opentsdb.Request{
		Queries: []*opentsdb.Query{q},
		Start:   fmt.Sprintf("%dms-ago", d.Nanoseconds()/1e6),
	}
	s, err := req.Query("ny-devtsdb04:4242")
	if err != nil {
		return
	} else if len(s) == 0 {
		err = fmt.Errorf("expr: no results returned: %s", query)
		return
	}
	for _, res := range s {
		if len(res.DPS) == 0 {
			// do something here?
			continue
		}
		var f []float64
		for _, p := range res.DPS {
			f = append(f, float64(p))
		}
		r = append(r, &Result{
			Value: Value(F(f)),
			Group: res.Tags,
		})
	}
	return
}

func Avg(query, duration string) ([]*Result, error) {
	return queryDuration(query, duration, avg)
}

// avg returns the mean of x.
func avg(x []float64) (a float64) {
	for _, v := range x {
		a += v
	}
	a /= float64(len(x))
	return
}

func Dev(query, duration string) ([]*Result, error) {
	return queryDuration(query, duration, dev)
}

// dev returns the sample standard deviation of x.
func dev(x []float64) (d float64) {
	a := avg(x)
	for _, v := range x {
		d += math.Pow(v-a, 2)
	}
	// how should we handle len(x) == 1?
	d /= float64(len(x) - 1)
	return math.Sqrt(d)
}

func recent() {}
func band()   {}
<commit_msg>No results isn't an error<commit_after>package expr

import (
	"fmt"
	"math"

	"github.com/StackExchange/tcollector/opentsdb"
	"github.com/StackExchange/tsaf/expr/parse"
)

const (
	DefDuration = "1h"
	DefPeriod   = "1w"
	DefNum      = 8
)

var Builtins = map[string]parse.Func{
	"avg": {
		[]parse.FuncType{parse.TYPE_SERIES, parse.TYPE_STRING},
		parse.TYPE_NUMBER,
		[]interface{}{DefDuration},
		Avg,
	},
	"band": {
		[]parse.FuncType{parse.TYPE_QUERY, parse.TYPE_STRING, parse.TYPE_STRING, parse.TYPE_NUMBER},
		parse.TYPE_SERIES,
		[]interface{}{DefDuration, DefPeriod, DefNum},
		band,
	},
	"dev": {
		[]parse.FuncType{parse.TYPE_SERIES, parse.TYPE_STRING},
		parse.TYPE_NUMBER,
		[]interface{}{DefDuration},
		Dev,
	},
	"recent": {
		[]parse.FuncType{parse.TYPE_SERIES, parse.TYPE_STRING},
		parse.TYPE_NUMBER,
		[]interface{}{DefDuration},
		recent,
	},
}

func queryDuration(query, duration string, F func([]float64) float64) (r []*Result, err error) {
	q, err := opentsdb.ParseQuery(query)
	if err != nil {
		return
	}
	d, err := ParseDuration(duration)
	if err != nil {
		return
	}
	req := opentsdb.Request{
		Queries: []*opentsdb.Query{q},
		Start:   fmt.Sprintf("%dms-ago", d.Nanoseconds()/1e6),
	}
	s, err := req.Query("ny-devtsdb04:4242")
	if err != nil {
		return
	} else if len(s) == 0 {
		return
	}
	for _, res := range s {
		if len(res.DPS) == 0 {
			// do something here?
			continue
		}
		var f []float64
		for _, p := range res.DPS {
			f = append(f, float64(p))
		}
		r = append(r, &Result{
			Value: Value(F(f)),
			Group: res.Tags,
		})
	}
	return
}

func Avg(query, duration string) ([]*Result, error) {
	return queryDuration(query, duration, avg)
}

// avg returns the mean of x.
func avg(x []float64) (a float64) {
	for _, v := range x {
		a += v
	}
	a /= float64(len(x))
	return
}

func Dev(query, duration string) ([]*Result, error) {
	return queryDuration(query, duration, dev)
}

// dev returns the sample standard deviation of x.
func dev(x []float64) (d float64) {
	a := avg(x)
	for _, v := range x {
		d += math.Pow(v-a, 2)
	}
	// how should we handle len(x) == 1?
	d /= float64(len(x) - 1)
	return math.Sqrt(d)
}

func recent() {}
func band()   {}
<|endoftext|>


<commit_before>package extra

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"strconv"
	"time"

	"github.com/pkg/errors"

	"github.com/jmoiron/sqlx"

	"github.com/elpinal/coco3/extra/ast"
	"github.com/elpinal/coco3/extra/parser" // Only for ParseError.
	"github.com/elpinal/coco3/extra/typed"
	"github.com/elpinal/coco3/extra/types"
)

type Env struct {
	cmds map[string]typed.Command
	Option
}

type Option struct {
	DB *sqlx.DB
}

func New(opt Option) Env {
	return Env{
		Option: opt,
		cmds: map[string]typed.Command{
			"exec":    execCommand,
			"cd":      cdCommand,
			"exit":    exitCommand,
			"free":    freeCommand,
			"history": historyCommand,

			"remove": removeCommand,

			"ls":  lsCommand,
			"man": manCommand,

			"git":   gitCommand,
			"cargo": cargoCommand,
			"go":    goCommand,
			"stack": stackCommand,

			"vim":    vimCommand,
			"emacs":  emacsCommand,
			"screen": screenCommand,
		},
	}
}

func WithoutDefault() Env {
	return Env{cmds: make(map[string]typed.Command)}
}

func (e *Env) Bind(name string, c typed.Command) {
	e.cmds[name] = c
}

func (e *Env) Eval(command *ast.Command) error {
	if command == nil {
		return nil
	}
	tc, found := e.cmds[command.Name.Lit]
	if !found {
		return &parser.ParseError{
			Msg:    fmt.Sprintf("no such typed command: %q", command.Name.Lit),
			Line:   command.Name.Line,
			Column: command.Name.Column,
		}
	}
	if len(command.Args) != len(tc.Params) {
		return &parser.ParseError{
			Msg:    fmt.Sprintf("the length of args (%d) != the one of params (%d)", len(command.Args), len(tc.Params)),
			Line:   command.Name.Line,
			Column: command.Name.Column,
		}
	}
	for i, arg := range command.Args {
		if arg.Type() != tc.Params[i] {
			return &parser.ParseError{
				Msg:    fmt.Sprintf("type mismatch: (%v) (type of %v) does not match with (%v) (expected type)", arg.Type(), arg, tc.Params[i]),
				Line:   command.Name.Line,
				Column: command.Name.Column,
			}
		}
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	defer close(c)
	defer signal.Stop(c)

	return tc.Fn(command.Args, e.DB)
}

func toSlice(list ast.List) ([]string, error) {
	ret := make([]string, 0, list.Length())
	for {
		switch x := list.(type) {
		case *ast.Cons:
			ret = append(ret, x.Head)
			list = x.Tail
		case *ast.Empty:
			return ret, nil
		default:
			return nil, fmt.Errorf("unexpected list type: %T", x)
		}
	}
}

var execCommand = typed.Command{
	Params: []types.Type{types.String, types.StringList},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, "exec")
		}
		cmd := stdCmd(args[0].(*ast.String).Lit, cmdArgs...)
		return cmd.Run()
	},
}

var cdCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		return os.Chdir(args[0].(*ast.String).Lit)
	},
}

var exitCommand = typed.Command{
	Params: []types.Type{types.Int},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		n, err := strconv.Atoi(args[0].(*ast.Int).Lit)
		if err != nil {
			return err
		}
		os.Exit(n)
		return nil
	},
}

var freeCommand = typed.Command{
	Params: []types.Type{types.String, types.StringList},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, "free")
		}
		name := args[0].(*ast.String).Lit
		cmd := exec.Cmd{Path: name, Args: append([]string{name}, cmdArgs...)}
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Stdin = os.Stdin
		return cmd.Run()
	},
}

func commandsInCommand(name string) func([]ast.Expr, *sqlx.DB) error {
	return func(args []ast.Expr, _ *sqlx.DB) error {
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, name)
		}
		var cmd *exec.Cmd
		switch lit := args[0].(*ast.Ident).Lit; lit {
		case "command":
			cmd = stdCmd(name, cmdArgs...)
		default:
			cmd = stdCmd(name, append([]string{lit}, cmdArgs...)...)
		}
		return cmd.Run()
	}
}

func goCommand1() func([]ast.Expr, *sqlx.DB) error {
	return func(args []ast.Expr, _ *sqlx.DB) error {
		name := "go"
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, name)
		}
		var cmd *exec.Cmd
		switch lit := args[0].(*ast.Ident).Lit; lit {
		case "command":
			cmd = stdCmd(name, cmdArgs...)
		case "testall":
			// I can't be confident in using such
			// a subcommand-specific way.  Another suggestion might
			// be like `go test all`, where 'all' is a postfix
			// operator of './...'.
			cmd = stdCmd(name, append([]string{"test"}, append(cmdArgs, "./...")...)...)
		default:
			cmd = stdCmd(name, append([]string{lit}, cmdArgs...)...)
		}
		return cmd.Run()
	}
}

var gitCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("git"),
}

var cargoCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("cargo"),
}

var goCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     goCommand1(),
}

var stackCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("stack"),
}

func stdCmd(name string, args ...string) *exec.Cmd {
	cmd := exec.Command(name, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	return cmd
}

func stdExec(name string, args ...string) func([]ast.Expr, *sqlx.DB) error {
	return func(_ []ast.Expr, _ *sqlx.DB) error {
		return stdCmd(name, args...).Run()
	}
}

var vimCommand = typed.Command{
	Params: []types.Type{},
	Fn:     stdExec("vim"),
}

var emacsCommand = typed.Command{
	Params: []types.Type{},
	Fn:     stdExec("emacs"),
}

func withEnv(s string, cmd *exec.Cmd) *exec.Cmd {
	if cmd.Env == nil {
		cmd.Env = os.Environ()
	}
	cmd.Env = append(cmd.Env, s)
	return cmd
}

var screenCommand = typed.Command{
	Params: []types.Type{},
	Fn: func(_ []ast.Expr, _ *sqlx.DB) error {
		return withEnv("LANG=en_US.UTF-8", stdCmd("screen")).Run()
	},
}

type execution struct {
	Time time.Time
	Line string
}

var historyCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(e []ast.Expr, db *sqlx.DB) error {
		var jsonFormat bool
		var enc *json.Encoder
		switch format := e[0].(*ast.String).Lit; format {
		case "json":
			jsonFormat = true
		case "lines":
		default:
			return fmt.Errorf("history: format %q is not supported", format)
		}

		buf := bufio.NewWriter(os.Stdout)
		if jsonFormat {
			enc = json.NewEncoder(buf)
		}
		rows, err := db.Queryx("select * from command_info")
		if err != nil {
			return err
		}
		data := execution{}
		for rows.Next() {
			err := rows.StructScan(&data)
			if err != nil {
				return err
			}
			if jsonFormat {
				err := enc.Encode(data)
				if err != nil {
					return err
				}
			} else {
				buf.WriteString(data.Time.Format("Mon, 02 Jan 2006 15:04:05"))
				buf.Write([]byte("  "))
				buf.WriteString(data.Line)
				buf.WriteByte('\n')
			}
		}
		return buf.Flush()
	},
}

var lsCommand = typed.Command{
	Params: []types.Type{},
	Fn:     stdExec("ls", "--show-control-chars", "--color=auto"),
}

var manCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(e []ast.Expr, _ *sqlx.DB) error {
		lit := e[0].(*ast.String).Lit
		return stdCmd("man", lit).Run()
	},
}

var removeCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(e []ast.Expr, _ *sqlx.DB) error {
		lit := e[0].(*ast.String).Lit
		return stdCmd("rm", "-i", lit).Run()
	},
}
<commit_msg>Add create-new-project typed command<commit_after>package extra

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"strconv"
	"time"

	"github.com/pkg/errors"

	"github.com/jmoiron/sqlx"

	"github.com/elpinal/coco3/extra/ast"
	"github.com/elpinal/coco3/extra/parser" // Only for ParseError.
	"github.com/elpinal/coco3/extra/typed"
	"github.com/elpinal/coco3/extra/types"
)

type Env struct {
	cmds map[string]typed.Command
	Option
}

type Option struct {
	DB *sqlx.DB
}

func New(opt Option) Env {
	return Env{
		Option: opt,
		cmds: map[string]typed.Command{
			"exec":    execCommand,
			"cd":      cdCommand,
			"exit":    exitCommand,
			"free":    freeCommand,
			"history": historyCommand,

			"remove": removeCommand,

			"ls":  lsCommand,
			"man": manCommand,

			"git":   gitCommand,
			"cargo": cargoCommand,
			"go":    goCommand,
			"stack": stackCommand,

			"vim":    vimCommand,
			"emacs":  emacsCommand,
			"screen": screenCommand,

			"create-new-project": cnpCommand,
		},
	}
}

func WithoutDefault() Env {
	return Env{cmds: make(map[string]typed.Command)}
}

func (e *Env) Bind(name string, c typed.Command) {
	e.cmds[name] = c
}

func (e *Env) Eval(command *ast.Command) error {
	if command == nil {
		return nil
	}
	tc, found := e.cmds[command.Name.Lit]
	if !found {
		return &parser.ParseError{
			Msg:    fmt.Sprintf("no such typed command: %q", command.Name.Lit),
			Line:   command.Name.Line,
			Column: command.Name.Column,
		}
	}
	if len(command.Args) != len(tc.Params) {
		return &parser.ParseError{
			Msg:    fmt.Sprintf("the length of args (%d) != the one of params (%d)", len(command.Args), len(tc.Params)),
			Line:   command.Name.Line,
			Column: command.Name.Column,
		}
	}
	for i, arg := range command.Args {
		if arg.Type() != tc.Params[i] {
			return &parser.ParseError{
				Msg:    fmt.Sprintf("type mismatch: (%v) (type of %v) does not match with (%v) (expected type)", arg.Type(), arg, tc.Params[i]),
				Line:   command.Name.Line,
				Column: command.Name.Column,
			}
		}
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	defer close(c)
	defer signal.Stop(c)

	return tc.Fn(command.Args, e.DB)
}

func toSlice(list ast.List) ([]string, error) {
	ret := make([]string, 0, list.Length())
	for {
		switch x := list.(type) {
		case *ast.Cons:
			ret = append(ret, x.Head)
			list = x.Tail
		case *ast.Empty:
			return ret, nil
		default:
			return nil, fmt.Errorf("unexpected list type: %T", x)
		}
	}
}

var execCommand = typed.Command{
	Params: []types.Type{types.String, types.StringList},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, "exec")
		}
		cmd := stdCmd(args[0].(*ast.String).Lit, cmdArgs...)
		return cmd.Run()
	},
}

var cdCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		return os.Chdir(args[0].(*ast.String).Lit)
	},
}

var exitCommand = typed.Command{
	Params: []types.Type{types.Int},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		n, err := strconv.Atoi(args[0].(*ast.Int).Lit)
		if err != nil {
			return err
		}
		os.Exit(n)
		return nil
	},
}

var freeCommand = typed.Command{
	Params: []types.Type{types.String, types.StringList},
	Fn: func(args []ast.Expr, _ *sqlx.DB) error {
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, "free")
		}
		name := args[0].(*ast.String).Lit
		cmd := exec.Cmd{Path: name, Args: append([]string{name}, cmdArgs...)}
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Stdin = os.Stdin
		return cmd.Run()
	},
}

func commandsInCommand(name string) func([]ast.Expr, *sqlx.DB) error {
	return func(args []ast.Expr, _ *sqlx.DB) error {
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, name)
		}
		var cmd *exec.Cmd
		switch lit := args[0].(*ast.Ident).Lit; lit {
		case "command":
			cmd = stdCmd(name, cmdArgs...)
		default:
			cmd = stdCmd(name, append([]string{lit}, cmdArgs...)...)
		}
		return cmd.Run()
	}
}

func goCommand1() func([]ast.Expr, *sqlx.DB) error {
	return func(args []ast.Expr, _ *sqlx.DB) error {
		name := "go"
		cmdArgs, err := toSlice(args[1].(ast.List))
		if err != nil {
			return errors.Wrap(err, name)
		}
		var cmd *exec.Cmd
		switch lit := args[0].(*ast.Ident).Lit; lit {
		case "command":
			cmd = stdCmd(name, cmdArgs...)
		case "testall":
			// I can't be confident in using such
			// a subcommand-specific way.  Another suggestion might
			// be like `go test all`, where 'all' is a postfix
			// operator of './...'.
			cmd = stdCmd(name, append([]string{"test"}, append(cmdArgs, "./...")...)...)
		default:
			cmd = stdCmd(name, append([]string{lit}, cmdArgs...)...)
		}
		return cmd.Run()
	}
}

var gitCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("git"),
}

var cargoCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("cargo"),
}

var goCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     goCommand1(),
}

var stackCommand = typed.Command{
	Params: []types.Type{types.Ident, types.StringList},
	Fn:     commandsInCommand("stack"),
}

func stdCmd(name string, args ...string) *exec.Cmd {
	cmd := exec.Command(name, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	return cmd
}

func stdExec(name string, args ...string) func([]ast.Expr, *sqlx.DB) error {
	return func(_ []ast.Expr, _ *sqlx.DB) error {
		return stdCmd(name, args...).Run()
	}
}

var vimCommand = typed.Command{
	Params: []types.Type{},
	Fn:     stdExec("vim"),
}

var emacsCommand = typed.Command{
	Params: []types.Type{},
	Fn:     stdExec("emacs"),
}

func withEnv(s string, cmd *exec.Cmd) *exec.Cmd {
	if cmd.Env == nil {
		cmd.Env = os.Environ()
	}
	cmd.Env = append(cmd.Env, s)
	return cmd
}

var screenCommand = typed.Command{
	Params: []types.Type{},
	Fn: func(_ []ast.Expr, _ *sqlx.DB) error {
		return withEnv("LANG=en_US.UTF-8", stdCmd("screen")).Run()
	},
}

type execution struct {
	Time time.Time
	Line string
}

var historyCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(e []ast.Expr, db *sqlx.DB) error {
		var jsonFormat bool
		var enc *json.Encoder
		switch format := e[0].(*ast.String).Lit; format {
		case "json":
			jsonFormat = true
		case "lines":
		default:
			return fmt.Errorf("history: format %q is not supported", format)
		}

		buf := bufio.NewWriter(os.Stdout)
		if jsonFormat {
			enc = json.NewEncoder(buf)
		}
		rows, err := db.Queryx("select * from command_info")
		if err != nil {
			return err
		}
		data := execution{}
		for rows.Next() {
			err := rows.StructScan(&data)
			if err != nil {
				return err
			}
			if jsonFormat {
				err := enc.Encode(data)
				if err != nil {
					return err
				}
			} else {
				buf.WriteString(data.Time.Format("Mon, 02 Jan 2006 15:04:05"))
				buf.Write([]byte("  "))
				buf.WriteString(data.Line)
				buf.WriteByte('\n')
			}
		}
		return buf.Flush()
	},
}

var lsCommand = typed.Command{
	Params: []types.Type{},
	Fn:     stdExec("ls", "--show-control-chars", "--color=auto"),
}

var manCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(e []ast.Expr, _ *sqlx.DB) error {
		lit := e[0].(*ast.String).Lit
		return stdCmd("man", lit).Run()
	},
}

var removeCommand = typed.Command{
	Params: []types.Type{types.String},
	Fn: func(e []ast.Expr, _ *sqlx.DB) error {
		lit := e[0].(*ast.String).Lit
		return stdCmd("rm", "-i", lit).Run()
	},
}

var cnpCommand = typed.Command{
	Params: []types.Type{},
	Fn: func(e []ast.Expr, _ *sqlx.DB) error {
		lit := e[0].(*ast.String).Lit
		return stdCmd("create-new-project", lit).Run()
	},
}
<|endoftext|>


<commit_before>// Package fastentity provides text sequence identification in documents.
package fastentity

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"sync"
	"unicode"
)

var (
	// Maximum entity length.  NB: currently entities can be added which are longer
	// but they will be ignored in the search.
	// TODO: Maybe also error/warn on import?
	MaxEntityLen = 30
	// Number of entities to initially allocate when creating a Group.
	DefaultGroupSize = 1000
)

const (
	left  = 0
	right = 1
)

type pair [2]int

type Store struct {
	groups map[string]*Group
	sync.RWMutex
}

type Group struct {
	Name     string
	Entities map[string][][]rune
	MaxLen   int
	sync.RWMutex
}

// Pops the last element and adds the new element
// to the front of stack.
func shift(n pair, s []pair) (pair, []pair) {
	if len(s) == 0 {
		return pair{}, append(s, n)
	}
	if len(s) == cap(s) {
		return s[0], append(s[1:], n)
	}
	return s[0], append(s, n)
}

// Create a new entity group structure
func New(groups ...string) *Store {
	s := &Store{
		groups: make(map[string]*Group, len(groups)),
	}
	for _, name := range groups {
		g := &Group{
			Name:     name,
			Entities: make(map[string][][]rune, DefaultGroupSize),
		}
		s.groups[name] = g
	}
	return s
}

// Add a new entity to a particular group
func (s *Store) Add(name string, entities ...[]rune) {
	s.Lock()
	g, ok := s.groups[name]
	if !ok {
		g = &Group{
			Name:     name,
			Entities: make(map[string][][]rune, DefaultGroupSize),
		}
		s.groups[name] = g
	}
	s.Unlock()

	g.Lock()
	for _, e := range entities {
		h := hash([]rune(e))
		g.Entities[h] = append(g.Entities[h], e)
		if len(e) > g.MaxLen {
			g.MaxLen = len(e)
		}
	}
	g.Unlock()
}

func hash(rs []rune) string {
	if len(rs) > 2 {
		return fmt.Sprintf("%s%s%s%03d", string(unicode.ToLower(rs[0])), string(unicode.ToLower(rs[1])), string(unicode.ToLower(rs[2])), len(rs))
	}
	if len(rs) > 1 {
		return fmt.Sprintf("%s%s%03d", string(unicode.ToLower(rs[0])), string(unicode.ToLower(rs[1])), len(rs))
	}
	return fmt.Sprintf("%s%03d", string(unicode.ToLower(rs[0])), len(rs))
}

// Find all entities for all type keys
func (s *Store) FindAll(rs []rune) map[string][][]rune {
	result := make(map[string][][]rune, len(s.groups))
	for name, g := range s.groups {
		result[name] = g.Find(rs)
	}
	return result
}

// Find only the entities of a given type = "key"
func (g *Group) Find(rs []rune) [][]rune {
	g.RLock()
	ents := find(rs, []*Group{g})
	g.RUnlock()
	return ents[g.Name]
}

// Lock free find for use internally
func find(rs []rune, groups []*Group) map[string][][]rune {
	results := make(map[string][][]rune, len(groups))
	pairs := make([]pair, 0, 20)
	start := 0
	prevSpace := true // First char of sequence is legit
	space := false

	for off, r := range rs {
		// What are we looking at?
		space = unicode.IsPunct(r) || unicode.IsSpace(r)

		if prevSpace && !space {
			// Word is beginning at this rune
			start = off
		} else if space && !prevSpace {
			// Word is ending, shift the pairs stack
			_, pairs = shift(pair{start, off}, pairs)

			// Run the stack, check for entities working backwards from the current position
			if len(pairs) > 1 {
				p2 := pairs[len(pairs)-1]
				for i := len(pairs) - 1; i >= 0; i-- {
					p1 := pairs[i]
					if p2[right]-p1[left] > MaxEntityLen {
						break // Too long or short, can ignore it
					}
					for _, group := range groups {
						if p2[right]-p1[left] > group.MaxLen {
							continue
						}
						if ents, ok := group.Entities[hash(rs[p1[left]:p2[right]])]; ok {
							// We have at least one entity with this key
							for _, ent := range ents {
								if len(ent) != p2[right]-p1[left] {
									break
								}
								match := true
								for i, r := range ent {
									if unicode.ToLower(r) != unicode.ToLower(rs[p1[left]+i]) {
										match = false
										break
									}
								}
								if match {
									results[group.Name] = append(results[group.Name], rs[p1[left]:p2[right]])
								}
							}
						}
					}
				}
			}
		}

		// Mark prevSpace for the next loop
		if space {
			prevSpace = true
		} else {
			prevSpace = false
		}
	}
	return results
}

type incr struct {
	sync.Mutex
	n int
}

func (i *incr) incr() {
	i.Lock()
	i.n++
	i.Unlock()
}

var entityFileSuffix = ".entities.csv"

// Create a new store by loading entity files from a given directory. The format
// expected has the format "<GROUP>.entities.csv"
func Load(dir string) (*Store, error) {
	dir = strings.TrimRight(dir, "/")
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}

	s := New()
	var wg sync.WaitGroup
	count := &incr{}
	for _, fileInfo := range files {
		if strings.HasSuffix(fileInfo.Name(), entityFileSuffix) {
			wg.Add(1)
			go func(filename string, group string) {
				defer wg.Done()
				f, err := os.Open(filename)
				if err != nil {
					// TODO: Remove this, return an error instead?
					fmt.Printf("Unable to load \"%s\" entity file: %s: %s\n", group, filename, err.Error())
					return
				}
				defer f.Close()

				count.incr()
				r := bufio.NewScanner(f)
				for r.Scan() {
					s.Add(group, []rune(r.Text()))
				}
			}(fmt.Sprintf("%s/%s", dir, fileInfo.Name()), strings.TrimSuffix(fileInfo.Name(), entityFileSuffix))
		}
	}
	wg.Wait()
	if count.n == 0 {
		return s, errors.New("There are no entity files")
	}
	return s, nil
}

// Save the existing entities to disk. Each group becomes a file with the format
// the format "<GROUP>.entities.csv" in the dir specified.
func (s *Store) Save(dir string) error {
	s.RLock()
	defer s.RUnlock()
	dir = strings.TrimRight(dir, "/")
	for name, group := range s.groups {
		filename := fmt.Sprintf("%s/%s", dir, strings.Replace(name, "/", "_", -1)+entityFileSuffix)
		f, err := os.Create(filename)
		if err != nil {
			return err
		}
		w := bufio.NewWriter(f)
		for _, entities := range group.Entities {
			for _, e := range entities {
				w.WriteString(string(e) + "\n")
			}
		}
		w.Flush()
		f.Close()
	}
	return nil
}
<commit_msg>Add and improve godoc comments.<commit_after>// Package fastentity provides text sequence identification in documents.
package fastentity

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"sync"
	"unicode"
)

var (
	// Maximum entity length.  NB: currently entities can be added which are longer
	// but they will be ignored in the search.
	// TODO: Maybe also error/warn on import?
	MaxEntityLen = 30
	// Number of entities to initially allocate when creating a Group.
	DefaultGroupSize = 1000
)

const (
	left  = 0
	right = 1
)

type pair [2]int

// Store is a collection of groups of entities.
type Store struct {
	groups map[string]*Group
	sync.RWMutex
}

type Group struct {
	Name     string
	Entities map[string][][]rune
	MaxLen   int
	sync.RWMutex
}

// Pops the last element and adds the new element to the front of stack.
func shift(n pair, s []pair) (pair, []pair) {
	if len(s) == 0 {
		return pair{}, append(s, n)
	}
	if len(s) == cap(s) {
		return s[0], append(s[1:], n)
	}
	return s[0], append(s, n)
}

// New creates a new Store of entity groups using the provided names.
func New(groups ...string) *Store {
	s := &Store{
		groups: make(map[string]*Group, len(groups)),
	}
	for _, name := range groups {
		g := &Group{
			Name:     name,
			Entities: make(map[string][][]rune, DefaultGroupSize),
		}
		s.groups[name] = g
	}
	return s
}

// Add ajoins the entities to the group identified by name.
func (s *Store) Add(name string, entities ...[]rune) {
	s.Lock()
	g, ok := s.groups[name]
	if !ok {
		g = &Group{
			Name:     name,
			Entities: make(map[string][][]rune, DefaultGroupSize),
		}
		s.groups[name] = g
	}
	s.Unlock()

	g.Lock()
	for _, e := range entities {
		h := hash([]rune(e))
		g.Entities[h] = append(g.Entities[h], e)
		if len(e) > g.MaxLen {
			g.MaxLen = len(e)
		}
	}
	g.Unlock()
}

func hash(rs []rune) string {
	if len(rs) > 2 {
		return fmt.Sprintf("%s%s%s%03d", string(unicode.ToLower(rs[0])), string(unicode.ToLower(rs[1])), string(unicode.ToLower(rs[2])), len(rs))
	}
	if len(rs) > 1 {
		return fmt.Sprintf("%s%s%03d", string(unicode.ToLower(rs[0])), string(unicode.ToLower(rs[1])), len(rs))
	}
	return fmt.Sprintf("%s%03d", string(unicode.ToLower(rs[0])), len(rs))
}

// FindAll searches the input returning a maping group name -> found entities.
func (s *Store) FindAll(rs []rune) map[string][][]rune {
	result := make(map[string][][]rune, len(s.groups))
	for name, g := range s.groups {
		result[name] = g.Find(rs)
	}
	return result
}

// Find only the entities of a given type = "key"
func (g *Group) Find(rs []rune) [][]rune {
	g.RLock()
	ents := find(rs, []*Group{g})
	g.RUnlock()
	return ents[g.Name]
}

// Lock free find for use internally
func find(rs []rune, groups []*Group) map[string][][]rune {
	results := make(map[string][][]rune, len(groups))
	pairs := make([]pair, 0, 20)
	start := 0
	prevSpace := true // First char of sequence is legit
	space := false

	for off, r := range rs {
		// What are we looking at?
		space = unicode.IsPunct(r) || unicode.IsSpace(r)

		if prevSpace && !space {
			// Word is beginning at this rune
			start = off
		} else if space && !prevSpace {
			// Word is ending, shift the pairs stack
			_, pairs = shift(pair{start, off}, pairs)

			// Run the stack, check for entities working backwards from the current position
			if len(pairs) > 1 {
				p2 := pairs[len(pairs)-1]
				for i := len(pairs) - 1; i >= 0; i-- {
					p1 := pairs[i]
					if p2[right]-p1[left] > MaxEntityLen {
						break // Too long or short, can ignore it
					}
					for _, group := range groups {
						if p2[right]-p1[left] > group.MaxLen {
							continue
						}
						if ents, ok := group.Entities[hash(rs[p1[left]:p2[right]])]; ok {
							// We have at least one entity with this key
							for _, ent := range ents {
								if len(ent) != p2[right]-p1[left] {
									break
								}
								match := true
								for i, r := range ent {
									if unicode.ToLower(r) != unicode.ToLower(rs[p1[left]+i]) {
										match = false
										break
									}
								}
								if match {
									results[group.Name] = append(results[group.Name], rs[p1[left]:p2[right]])
								}
							}
						}
					}
				}
			}
		}

		// Mark prevSpace for the next loop
		if space {
			prevSpace = true
		} else {
			prevSpace = false
		}
	}
	return results
}

type incr struct {
	sync.Mutex
	n int
}

func (i *incr) incr() {
	i.Lock()
	i.n++
	i.Unlock()
}

var entityFileSuffix = ".entities.csv"

// Load creates a new Store by loading entity files from a given directory path. Any files
// contained in the directory with names matching <group>.entities.csv will be imported,
// and the entities added to the group <group>.
func Load(dir string) (*Store, error) {
	dir = strings.TrimRight(dir, "/")
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}

	s := New()
	var wg sync.WaitGroup
	count := &incr{}
	for _, fileInfo := range files {
		if strings.HasSuffix(fileInfo.Name(), entityFileSuffix) {
			wg.Add(1)
			go func(filename string, group string) {
				defer wg.Done()
				f, err := os.Open(filename)
				if err != nil {
					// TODO: Remove this, return an error instead?
					fmt.Printf("Unable to load \"%s\" entity file: %s: %s\n", group, filename, err.Error())
					return
				}
				defer f.Close()

				count.incr()
				r := bufio.NewScanner(f)
				for r.Scan() {
					s.Add(group, []rune(r.Text()))
				}
			}(fmt.Sprintf("%s/%s", dir, fileInfo.Name()), strings.TrimSuffix(fileInfo.Name(), entityFileSuffix))
		}
	}
	wg.Wait()
	if count.n == 0 {
		return s, errors.New("There are no entity files")
	}
	return s, nil
}

// Save writes the existing entities to disk under the given directory path (assumed
// to already exist). Each entity group becomes a file <group>.entities.csv.
func (s *Store) Save(dir string) error {
	s.RLock()
	defer s.RUnlock()
	dir = strings.TrimRight(dir, "/")
	for name, group := range s.groups {
		filename := fmt.Sprintf("%s/%s", dir, strings.Replace(name, "/", "_", -1)+entityFileSuffix)
		f, err := os.Create(filename)
		if err != nil {
			return err
		}
		w := bufio.NewWriter(f)
		for _, entities := range group.Entities {
			for _, e := range entities {
				w.WriteString(string(e) + "\n")
			}
		}
		w.Flush()
		f.Close()
	}
	return nil
}
<|endoftext|>


<commit_before>package cacheddownloader

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"

	"code.cloudfoundry.org/archiver/compressor"
	"code.cloudfoundry.org/archiver/extractor"
	"code.cloudfoundry.org/lager"
)

var (
	lock          = &sync.Mutex{}
	EntryNotFound = errors.New("Entry Not Found")
	AlreadyClosed = errors.New("Already closed directory")
	NotCacheable  = errors.New("Not cacheable directory")
)

type FileCache struct {
	CachedPath     string
	maxSizeInBytes int64
	Entries        map[string]*FileCacheEntry
	OldEntries     map[string]*FileCacheEntry
	Seq            uint64
}

type FileCacheEntry struct {
	Size                  int64
	Access                time.Time
	CachingInfo           CachingInfoType
	FilePath              string
	ExpandedDirectoryPath string
	directoryInUseCount   int
	fileInUseCount        int
}

func NewCache(dir string, maxSizeInBytes int64) *FileCache {
	return &FileCache{
		CachedPath:     dir,
		maxSizeInBytes: maxSizeInBytes,
		Entries:        map[string]*FileCacheEntry{},
		OldEntries:     map[string]*FileCacheEntry{},
		Seq:            0,
	}
}

func newFileCacheEntry(cachePath string, size int64, cachingInfo CachingInfoType) *FileCacheEntry {
	return &FileCacheEntry{
		Size:                  size,
		FilePath:              cachePath,
		Access:                time.Now(),
		CachingInfo:           cachingInfo,
		ExpandedDirectoryPath: "",
	}
}

func (e *FileCacheEntry) inUse() bool {
	return e.directoryInUseCount > 0 || e.fileInUseCount > 0
}

func (e *FileCacheEntry) decrementUse() {
	e.decrementFileInUseCount()
	e.decrementDirectoryInUseCount()
}

func (e *FileCacheEntry) incrementDirectoryInUseCount() {
	e.directoryInUseCount++
}

func (e *FileCacheEntry) decrementDirectoryInUseCount() {
	e.directoryInUseCount--

	// Delete the directory if the tarball is the only asset
	// being used or if the directory has been removed (in use count -1)
	if e.directoryInUseCount < 0 || (e.directoryInUseCount == 0 && e.fileInUseCount > 0) {
		err := os.RemoveAll(e.ExpandedDirectoryPath)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Unable to delete cached directory", err)
		}
		e.ExpandedDirectoryPath = ""

		if e.fileInUseCount > 0 {
			e.Size = e.Size / 2
		}
	}
}

func (e *FileCacheEntry) incrementFileInUseCount() {
	e.fileInUseCount++
}

func (e *FileCacheEntry) decrementFileInUseCount() {
	e.fileInUseCount--

	// Delete the file if the file is not being used and there is
	// a directory of if the file has been removed (in use count -1)
	if e.fileInUseCount < 0 || (e.fileInUseCount == 0 && e.directoryInUseCount > 0) {
		err := os.RemoveAll(e.FilePath)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Unable to delete cached file", err)
		}

		if e.directoryInUseCount > 0 {
			e.Size = e.Size / 2
		}
	}
}

func (e *FileCacheEntry) fileDoesNotExist() bool {
	_, err := os.Stat(e.FilePath)
	return os.IsNotExist(err)
}

func (e *FileCacheEntry) dirDoesNotExist() bool {
	if e.ExpandedDirectoryPath == "" {
		return true
	}
	_, err := os.Stat(e.ExpandedDirectoryPath)
	return os.IsNotExist(err)
}

// Can we change this to be an io.ReadCloser return
func (e *FileCacheEntry) readCloser() (*CachedFile, error) {
	var f *os.File
	var err error

	if e.fileDoesNotExist() {
		f, err = os.Create(e.FilePath)
		if err != nil {
			return nil, err
		}

		err = compressor.WriteTar(e.ExpandedDirectoryPath+"/", f)
		if err != nil {
			return nil, err
		}

		// If the directory is not used remove it
		if e.directoryInUseCount == 0 {
			err = os.RemoveAll(e.ExpandedDirectoryPath)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Unable to remove cached directory", err)
			}
		} else {
			// Double the size to account for both assets
			e.Size = e.Size * 2
		}
	} else {
		f, err = os.Open(e.FilePath)
		if err != nil {
			return nil, err
		}
	}

	e.incrementFileInUseCount()

	if _, err := f.Seek(0, os.SEEK_SET); err != nil {
		return nil, err
	}

	readCloser := NewFileCloser(f, func(filePath string) {
		lock.Lock()
		e.decrementFileInUseCount()
		lock.Unlock()
	})

	return readCloser, nil
}

func (e *FileCacheEntry) expandedDirectory() (string, error) {
	// if it has not been extracted before expand it!
	if e.dirDoesNotExist() {
		e.ExpandedDirectoryPath = e.FilePath + ".d"
		err := extractTarToDirectory(e.FilePath, e.ExpandedDirectoryPath)
		if err != nil {
			return "", err
		}

		// If the file is not in use, we can delete it
		if e.fileInUseCount == 0 {
			err = os.RemoveAll(e.FilePath)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Unable to delete the cached file", err)
			}
		}
	}

	e.incrementDirectoryInUseCount()

	return e.ExpandedDirectoryPath, nil
}

func (c *FileCache) CloseDirectory(logger lager.Logger, cacheKey, dirPath string) error {
	logger = logger.Session("file-cache.close-directory", lager.Data{"cache_key": cacheKey, "dir_path": dirPath})
	lock.Lock()
	defer lock.Unlock()

	logger.Info("starting")
	defer logger.Info("finished")

	entry := c.Entries[cacheKey]
	if entry != nil && entry.ExpandedDirectoryPath == dirPath {
		if entry.directoryInUseCount == 0 {
			// We don't think anybody is using this so throw an error
			return AlreadyClosed
		}

		entry.decrementDirectoryInUseCount()
		return nil
	}

	// Key didn't match anything in the current cache, so
	// check and clean up old entries
	entry = c.OldEntries[cacheKey+dirPath]
	if entry == nil {
		return EntryNotFound
	}

	entry.decrementDirectoryInUseCount()
	if !entry.inUse() {
		// done with this old entry, so clean it up
		delete(c.OldEntries, cacheKey+dirPath)
	}
	return nil
}

func (c *FileCache) Add(logger lager.Logger, cacheKey, sourcePath string, size int64, cachingInfo CachingInfoType) (*CachedFile, error) {
	logger = logger.Session("file-cache.add", lager.Data{"cache_key": cacheKey, "source_path": sourcePath, "size": size})
	lock.Lock()
	defer lock.Unlock()

	logger.Info("starting")
	defer logger.Info("finished")

	oldEntry := c.Entries[cacheKey]

	c.makeRoom(logger, size, "")

	c.Seq++
	uniqueName := fmt.Sprintf("%s-%d-%d", cacheKey, time.Now().UnixNano(), c.Seq)
	cachePath := filepath.Join(c.CachedPath, uniqueName)

	err := os.Rename(sourcePath, cachePath)
	if err != nil {
		return nil, err
	}

	newEntry := newFileCacheEntry(cachePath, size, cachingInfo)
	c.Entries[cacheKey] = newEntry
	if oldEntry != nil {
		oldEntry.decrementUse()
		c.updateOldEntries(logger, cacheKey, oldEntry)
	}
	return newEntry.readCloser()
}

func (c *FileCache) AddDirectory(logger lager.Logger, cacheKey, sourcePath string, size int64, cachingInfo CachingInfoType) (string, error) {
	logger = logger.Session("file-cache.add-directory", lager.Data{"cache_key": cacheKey, "source_path": sourcePath, "size": size})
	lock.Lock()
	defer lock.Unlock()

	logger.Info("starting")
	defer logger.Info("finished")

	oldEntry := c.Entries[cacheKey]

	c.makeRoom(logger, size, "")

	c.Seq++
	uniqueName := fmt.Sprintf("%s-%d-%d", cacheKey, time.Now().UnixNano(), c.Seq)
	cachePath := filepath.Join(c.CachedPath, uniqueName)

	err := os.Rename(sourcePath, cachePath)
	if err != nil {
		return "", err
	}

	newEntry := newFileCacheEntry(cachePath, size, cachingInfo)
	c.Entries[cacheKey] = newEntry
	if oldEntry != nil {
		oldEntry.decrementUse()
		c.updateOldEntries(logger, cacheKey, oldEntry)
	}
	return newEntry.expandedDirectory()
}

func (c *FileCache) Get(logger lager.Logger, cacheKey string) (*CachedFile, CachingInfoType, error) {
	logger = logger.Session("file-cache.get", lager.Data{"cache_key": cacheKey})
	lock.Lock()
	defer lock.Unlock()

	logger.Info("starting")
	defer logger.Info("finished")

	entry := c.Entries[cacheKey]
	if entry == nil {
		return nil, CachingInfoType{}, EntryNotFound
	}

	if entry.fileDoesNotExist() {
		c.makeRoom(logger, entry.Size, cacheKey)
	}

	entry.Access = time.Now()
	readCloser, err := entry.readCloser()
	if err != nil {
		return nil, CachingInfoType{}, err
	}

	return readCloser, entry.CachingInfo, nil
}

func (c *FileCache) GetDirectory(logger lager.Logger, cacheKey string) (string, CachingInfoType, error) {
	logger = logger.Session("file-cache.get-directory", lager.Data{"cache_key": cacheKey})
	lock.Lock()
	defer lock.Unlock()

	logger.Info("starting")
	defer logger.Info("finished")

	entry := c.Entries[cacheKey]
	if entry == nil {
		return "", CachingInfoType{}, EntryNotFound
	}

	// Was it expanded before
	if entry.dirDoesNotExist() {
		// Do we have enough room to double the size?
		c.makeRoom(logger, entry.Size, cacheKey)
		entry.Size = entry.Size * 2
	}

	entry.Access = time.Now()
	dir, err := entry.expandedDirectory()
	if err != nil {
		return "", CachingInfoType{}, err
	}

	return dir, entry.CachingInfo, nil
}

func (c *FileCache) Remove(logger lager.Logger, cacheKey string) {
	logger = logger.Session("file-cache.get-directory", lager.Data{"cache_key": cacheKey})

	lock.Lock()
	logger.Info("starting")
	c.remove(logger, cacheKey)
	lock.Unlock()
	logger.Info("finished")
}

func (c *FileCache) remove(logger lager.Logger, cacheKey string) {
	entry := c.Entries[cacheKey]
	if entry != nil {
		entry.decrementUse()
		c.updateOldEntries(logger, cacheKey, entry)
		delete(c.Entries, cacheKey)
	}
}

func (c *FileCache) updateOldEntries(logger lager.Logger, cacheKey string, entry *FileCacheEntry) {
	if entry != nil {
		if !entry.inUse() && entry.ExpandedDirectoryPath != "" {
			// put it in the oldEntries Cache since somebody may still be using the directory
			c.OldEntries[cacheKey+entry.ExpandedDirectoryPath] = entry
		} else {
			// We need to remove it from oldEntries
			delete(c.OldEntries, cacheKey+entry.ExpandedDirectoryPath)
		}
	}
}

func (c *FileCache) makeRoom(logger lager.Logger, size int64, excludedCacheKey string) {
	usedSpace := c.usedSpace(logger)
	for c.maxSizeInBytes < usedSpace+size {
		var oldestEntry *FileCacheEntry
		oldestAccessTime, oldestCacheKey := maxTime(), ""
		for ck, f := range c.Entries {
			if f.Access.Before(oldestAccessTime) && ck != excludedCacheKey && !f.inUse() {
				oldestAccessTime = f.Access
				oldestEntry = f
				oldestCacheKey = ck
			}
		}

		if oldestEntry == nil {
			// could not find anything we could remove
			return
		}

		usedSpace -= oldestEntry.Size
		c.remove(logger, oldestCacheKey)
	}

	return
}

func (c *FileCache) usedSpace(logger lager.Logger) int64 {
	space := int64(0)
	for _, f := range c.Entries {
		space += f.Size
	}
	return space
}

func extractTarToDirectory(sourcePath, destinationDir string) error {
	e := extractor.NewTar()
	return e.Extract(sourcePath, destinationDir)
}

func maxTime() time.Time {
	unixToInternal := int64((1969*365 + 1969/4 - 1969/100 + 1969/400) * 24 * 60 * 60)
	return time.Unix(1<<63-1-unixToInternal, 999999999)
}
<commit_msg>fix log session<commit_after>package cacheddownloader

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"

	"code.cloudfoundry.org/archiver/compressor"
	"code.cloudfoundry.org/archiver/extractor"
	"code.cloudfoundry.org/lager"
)

var (
	lock          = &sync.Mutex{}
	EntryNotFound = errors.New("Entry Not Found")
	AlreadyClosed = errors.New("Already closed directory")
	NotCacheable  = errors.New("Not cacheable directory")
)

type FileCache struct {
	CachedPath     string
	maxSizeInBytes int64
	Entries        map[string]*FileCacheEntry
	OldEntries     map[string]*FileCacheEntry
	Seq            uint64
}

type FileCacheEntry struct {
	Size                  int64
	Access                time.Time
	CachingInfo           CachingInfoType
	FilePath              string
	ExpandedDirectoryPath string
	directoryInUseCount   int
	fileInUseCount        int
}

func NewCache(dir string, maxSizeInBytes int64) *FileCache {
	return &FileCache{
		CachedPath:     dir,
		maxSizeInBytes: maxSizeInBytes,
		Entries:        map[string]*FileCacheEntry{},
		OldEntries:     map[string]*FileCacheEntry{},
		Seq:            0,
	}
}

func newFileCacheEntry(cachePath string, size int64, cachingInfo CachingInfoType) *FileCacheEntry {
	return &FileCacheEntry{
		Size:                  size,
		FilePath:              cachePath,
		Access:                time.Now(),
		CachingInfo:           cachingInfo,
		ExpandedDirectoryPath: "",
	}
}

func (e *FileCacheEntry) inUse() bool {
	return e.directoryInUseCount > 0 || e.fileInUseCount > 0
}

func (e *FileCacheEntry) decrementUse() {
	e.decrementFileInUseCount()
	e.decrementDirectoryInUseCount()
}

func (e *FileCacheEntry) incrementDirectoryInUseCount() {
	e.directoryInUseCount++
}

func (e *FileCacheEntry) decrementDirectoryInUseCount() {
	e.directoryInUseCount--

	// Delete the directory if the tarball is the only asset
	// being used or if the directory has been removed (in use count -1)
	if e.directoryInUseCount < 0 || (e.directoryInUseCount == 0 && e.fileInUseCount > 0) {
		err := os.RemoveAll(e.ExpandedDirectoryPath)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Unable to delete cached directory", err)
		}
		e.ExpandedDirectoryPath = ""

		if e.fileInUseCount > 0 {
			e.Size = e.Size / 2
		}
	}
}

func (e *FileCacheEntry) incrementFileInUseCount() {
	e.fileInUseCount++
}

func (e *FileCacheEntry) decrementFileInUseCount() {
	e.fileInUseCount--

	// Delete the file if the file is not being used and there is
	// a directory of if the file has been removed (in use count -1)
	if e.fileInUseCount < 0 || (e.fileInUseCount == 0 && e.directoryInUseCount > 0) {
		err := os.RemoveAll(e.FilePath)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Unable to delete cached file", err)
		}

		if e.directoryInUseCount > 0 {
			e.Size = e.Size / 2
		}
	}
}

func (e *FileCacheEntry) fileDoesNotExist() bool {
	_, err := os.Stat(e.FilePath)
	return os.IsNotExist(err)
}

func (e *FileCacheEntry) dirDoesNotExist() bool {
	if e.ExpandedDirectoryPath == "" {
		return true
	}
	_, err := os.Stat(e.ExpandedDirectoryPath)
	return os.IsNotExist(err)
}

// Can we change this to be an io.ReadCloser return
func (e *FileCacheEntry) readCloser() (*CachedFile, error) {
	var f *os.File
	var err error

	if e.fileDoesNotExist() {
		f, err = os.Create(e.FilePath)
		if err != nil {
			return nil, err
		}

		err = compressor.WriteTar(e.ExpandedDirectoryPath+"/", f)
		if err != nil {
			return nil, err
		}

		// If the directory is not used remove it
		if e.directoryInUseCount == 0 {
			err = os.RemoveAll(e.ExpandedDirectoryPath)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Unable to remove cached directory", err)
			}
		} else {
			// Double the size to account for both assets
			e.Size = e.Size * 2
		}
	} else {
		f, err = os.Open(e.FilePath)
		if err != nil {
			return nil, err
		}
	}

	e.incrementFileInUseCount()

	if _, err := f.Seek(0, os.SEEK_SET); err != nil {
		return nil, err
	}

	readCloser := NewFileCloser(f, func(filePath string) {
		lock.Lock()
		e.decrementFileInUseCount()
		lock.Unlock()
	})

	return readCloser, nil
}

func (e *FileCacheEntry) expandedDirectory() (string, error) {
	// if it has not been extracted before expand it!
	if e.dirDoesNotExist() {
		e.ExpandedDirectoryPath = e.FilePath + ".d"
		err := extractTarToDirectory(e.FilePath, e.ExpandedDirectoryPath)
		if err != nil {
			return "", err
		}

		// If the file is not in use, we can delete it
		if e.fileInUseCount == 0 {
			err = os.RemoveAll(e.FilePath)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Unable to delete the cached file", err)
			}
		}
	}

	e.incrementDirectoryInUseCount()

	return e.ExpandedDirectoryPath, nil
}

func (c *FileCache) CloseDirectory(logger lager.Logger, cacheKey, dirPath string) error {
	logger = logger.Session("file-cache.close-directory", lager.Data{"cache_key": cacheKey, "dir_path": dirPath})
	lock.Lock()
	defer lock.Unlock()

	logger.Info("starting")
	defer logger.Info("finished")

	entry := c.Entries[cacheKey]
	if entry != nil && entry.ExpandedDirectoryPath == dirPath {
		if entry.directoryInUseCount == 0 {
			// We don't think anybody is using this so throw an error
			return AlreadyClosed
		}

		entry.decrementDirectoryInUseCount()
		return nil
	}

	// Key didn't match anything in the current cache, so
	// check and clean up old entries
	entry = c.OldEntries[cacheKey+dirPath]
	if entry == nil {
		return EntryNotFound
	}

	entry.decrementDirectoryInUseCount()
	if !entry.inUse() {
		// done with this old entry, so clean it up
		delete(c.OldEntries, cacheKey+dirPath)
	}
	return nil
}

func (c *FileCache) Add(logger lager.Logger, cacheKey, sourcePath string, size int64, cachingInfo CachingInfoType) (*CachedFile, error) {
	logger = logger.Session("file-cache.add", lager.Data{"cache_key": cacheKey, "source_path": sourcePath, "size": size})
	lock.Lock()
	defer lock.Unlock()

	logger.Info("starting")
	defer logger.Info("finished")

	oldEntry := c.Entries[cacheKey]

	c.makeRoom(logger, size, "")

	c.Seq++
	uniqueName := fmt.Sprintf("%s-%d-%d", cacheKey, time.Now().UnixNano(), c.Seq)
	cachePath := filepath.Join(c.CachedPath, uniqueName)

	err := os.Rename(sourcePath, cachePath)
	if err != nil {
		return nil, err
	}

	newEntry := newFileCacheEntry(cachePath, size, cachingInfo)
	c.Entries[cacheKey] = newEntry
	if oldEntry != nil {
		oldEntry.decrementUse()
		c.updateOldEntries(logger, cacheKey, oldEntry)
	}
	return newEntry.readCloser()
}

func (c *FileCache) AddDirectory(logger lager.Logger, cacheKey, sourcePath string, size int64, cachingInfo CachingInfoType) (string, error) {
	logger = logger.Session("file-cache.add-directory", lager.Data{"cache_key": cacheKey, "source_path": sourcePath, "size": size})
	lock.Lock()
	defer lock.Unlock()

	logger.Info("starting")
	defer logger.Info("finished")

	oldEntry := c.Entries[cacheKey]

	c.makeRoom(logger, size, "")

	c.Seq++
	uniqueName := fmt.Sprintf("%s-%d-%d", cacheKey, time.Now().UnixNano(), c.Seq)
	cachePath := filepath.Join(c.CachedPath, uniqueName)

	err := os.Rename(sourcePath, cachePath)
	if err != nil {
		return "", err
	}

	newEntry := newFileCacheEntry(cachePath, size, cachingInfo)
	c.Entries[cacheKey] = newEntry
	if oldEntry != nil {
		oldEntry.decrementUse()
		c.updateOldEntries(logger, cacheKey, oldEntry)
	}
	return newEntry.expandedDirectory()
}

func (c *FileCache) Get(logger lager.Logger, cacheKey string) (*CachedFile, CachingInfoType, error) {
	logger = logger.Session("file-cache.get", lager.Data{"cache_key": cacheKey})
	lock.Lock()
	defer lock.Unlock()

	logger.Info("starting")
	defer logger.Info("finished")

	entry := c.Entries[cacheKey]
	if entry == nil {
		return nil, CachingInfoType{}, EntryNotFound
	}

	if entry.fileDoesNotExist() {
		c.makeRoom(logger, entry.Size, cacheKey)
	}

	entry.Access = time.Now()
	readCloser, err := entry.readCloser()
	if err != nil {
		return nil, CachingInfoType{}, err
	}

	return readCloser, entry.CachingInfo, nil
}

func (c *FileCache) GetDirectory(logger lager.Logger, cacheKey string) (string, CachingInfoType, error) {
	logger = logger.Session("file-cache.get-directory", lager.Data{"cache_key": cacheKey})
	lock.Lock()
	defer lock.Unlock()

	logger.Info("starting")
	defer logger.Info("finished")

	entry := c.Entries[cacheKey]
	if entry == nil {
		return "", CachingInfoType{}, EntryNotFound
	}

	// Was it expanded before
	if entry.dirDoesNotExist() {
		// Do we have enough room to double the size?
		c.makeRoom(logger, entry.Size, cacheKey)
		entry.Size = entry.Size * 2
	}

	entry.Access = time.Now()
	dir, err := entry.expandedDirectory()
	if err != nil {
		return "", CachingInfoType{}, err
	}

	return dir, entry.CachingInfo, nil
}

func (c *FileCache) Remove(logger lager.Logger, cacheKey string) {
	logger = logger.Session("file-cache.remove", lager.Data{"cache_key": cacheKey})

	lock.Lock()
	logger.Info("starting")
	c.remove(logger, cacheKey)
	lock.Unlock()
	logger.Info("finished")
}

func (c *FileCache) remove(logger lager.Logger, cacheKey string) {
	entry := c.Entries[cacheKey]
	if entry != nil {
		entry.decrementUse()
		c.updateOldEntries(logger, cacheKey, entry)
		delete(c.Entries, cacheKey)
	}
}

func (c *FileCache) updateOldEntries(logger lager.Logger, cacheKey string, entry *FileCacheEntry) {
	if entry != nil {
		if !entry.inUse() && entry.ExpandedDirectoryPath != "" {
			// put it in the oldEntries Cache since somebody may still be using the directory
			c.OldEntries[cacheKey+entry.ExpandedDirectoryPath] = entry
		} else {
			// We need to remove it from oldEntries
			delete(c.OldEntries, cacheKey+entry.ExpandedDirectoryPath)
		}
	}
}

func (c *FileCache) makeRoom(logger lager.Logger, size int64, excludedCacheKey string) {
	usedSpace := c.usedSpace(logger)
	for c.maxSizeInBytes < usedSpace+size {
		var oldestEntry *FileCacheEntry
		oldestAccessTime, oldestCacheKey := maxTime(), ""
		for ck, f := range c.Entries {
			if f.Access.Before(oldestAccessTime) && ck != excludedCacheKey && !f.inUse() {
				oldestAccessTime = f.Access
				oldestEntry = f
				oldestCacheKey = ck
			}
		}

		if oldestEntry == nil {
			// could not find anything we could remove
			return
		}

		usedSpace -= oldestEntry.Size
		c.remove(logger, oldestCacheKey)
	}

	return
}

func (c *FileCache) usedSpace(logger lager.Logger) int64 {
	space := int64(0)
	for _, f := range c.Entries {
		space += f.Size
	}
	return space
}

func extractTarToDirectory(sourcePath, destinationDir string) error {
	e := extractor.NewTar()
	return e.Extract(sourcePath, destinationDir)
}

func maxTime() time.Time {
	unixToInternal := int64((1969*365 + 1969/4 - 1969/100 + 1969/400) * 24 * 60 * 60)
	return time.Unix(1<<63-1-unixToInternal, 999999999)
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package virtcontainers

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
)

// podResource is an int representing a pod resource type
type podResource int

const (
	// configFileType represents a configuration file type
	configFileType podResource = iota

	// stateFileType represents a state file type
	stateFileType

	// networkFileType represents a network file type
	networkFileType

	// processFileType represents a process file type
	processFileType

	// lockFileType represents a lock file type
	lockFileType
)

// configFile is the file name used for every JSON pod configuration.
const configFile = "config.json"

// stateFile is the file name storing a pod state.
const stateFile = "state.json"

// networkFile is the file name storing a pod network.
const networkFile = "network.json"

// processFile is the file name storing a container process.
const processFile = "process.json"

// lockFile is the file name locking the usage of a pod.
const lockFileName = "lock"

// dirMode is the permission bits used for creating a directory
const dirMode = os.FileMode(0750)

// storagePathSuffix is the suffix used for all storage paths
const storagePathSuffix = "/virtcontainers/pods"

// configStoragePath is the pod configuration directory.
// It will contain one config.json file for each created pod.
var configStoragePath = filepath.Join("/var/lib", storagePathSuffix)

// runStoragePath is the pod runtime directory.
// It will contain one state.json and one lock file for each created pod.
var runStoragePath = filepath.Join("/run", storagePathSuffix)

// resourceStorage is the virtcontainers resources (configuration, state, etc...)
// storage interface.
// The default resource storage implementation is filesystem.
type resourceStorage interface {
	// Create all resources for a pod
	createAllResources(pod Pod) error

	// Resources URIs functions return both the URI
	// for the actual resource and the URI base.
	containerURI(podID, containerID string, resource podResource) (string, string, error)
	podURI(podID string, resource podResource) (string, string, error)

	// Pod resources
	storePodResource(podID string, resource podResource, data interface{}) error
	deletePodResources(podID string, resources []podResource) error
	fetchPodConfig(podID string) (PodConfig, error)
	fetchPodState(podID string) (State, error)
	fetchPodNetwork(podID string) (NetworkNamespace, error)
	storePodNetwork(podID string, networkNS NetworkNamespace) error

	// Container resources
	storeContainerResource(podID, containerID string, resource podResource, data interface{}) error
	deleteContainerResources(podID, containerID string, resources []podResource) error
	fetchContainerConfig(podID, containerID string) (ContainerConfig, error)
	fetchContainerState(podID, containerID string) (State, error)
	fetchContainerProcess(podID, containerID string) (Process, error)
	storeContainerProcess(podID, containerID string, process Process) error
}

// filesystem is a resourceStorage interface implementation for a local filesystem.
type filesystem struct {
}

func (fs *filesystem) createAllResources(pod Pod) error {
	_, path, _ := fs.podURI(pod.id, stateFileType)
	err := os.MkdirAll(path, dirMode)
	if err != nil {
		return err
	}

	_, path, _ = fs.podURI(pod.id, configFileType)
	err = os.MkdirAll(path, dirMode)
	if err != nil {
		return err
	}

	for _, container := range pod.containers {
		_, path, _ = fs.containerURI(pod.id, container.id, configFileType)
		err = os.MkdirAll(path, dirMode)
		if err != nil {
			fs.deletePodResources(pod.id, nil)
			return err
		}

		_, path, _ = fs.containerURI(pod.id, container.id, stateFileType)
		err = os.MkdirAll(path, dirMode)
		if err != nil {
			fs.deletePodResources(pod.id, nil)
			return err
		}
	}

	podlockFile, _, err := fs.podURI(pod.id, lockFileType)
	if err != nil {
		fs.deletePodResources(pod.id, nil)
		return err
	}

	_, err = os.Stat(podlockFile)
	if err != nil {
		lockFile, err := os.Create(podlockFile)
		if err != nil {
			fs.deletePodResources(pod.id, nil)
			return err
		}
		lockFile.Close()
	}

	return nil
}

func (fs *filesystem) storeFile(path string, data interface{}) error {
	_, err := os.Stat(path)
	if err == nil {
		os.Remove(path)
	}

	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer f.Close()

	jsonOut, err := json.Marshal(data)
	if err != nil {
		return fmt.Errorf("Could not marshall data: %s", err)
	}
	f.Write(jsonOut)

	return nil
}

func (fs *filesystem) fetchFile(path string, data interface{}) error {
	_, err := os.Stat(path)
	if err != nil {
		return err
	}

	fileData, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	err = json.Unmarshal([]byte(string(fileData)), data)
	if err != nil {
		return err
	}

	return nil
}

func resourceDir(podID, containerID string, resource podResource) (string, error) {
	var path string

	if podID == "" {
		return "", fmt.Errorf("PodID cannot be empty")
	}

	switch resource {
	case configFileType:
		path = configStoragePath
		break
	case stateFileType, networkFileType, processFileType, lockFileType:
		path = runStoragePath
		break
	default:
		return "", fmt.Errorf("Invalid pod resource")
	}

	dirPath := filepath.Join(path, podID, containerID)

	return dirPath, nil
}

func (fs *filesystem) resourceURI(podID, containerID string, resource podResource) (string, string, error) {
	var filename string

	if podID == "" {
		return "", "", fmt.Errorf("Pod ID cannot be empty")
	}

	dirPath, err := resourceDir(podID, containerID, resource)
	if err != nil {
		return "", "", err
	}

	switch resource {
	case configFileType:
		filename = configFile
		break
	case stateFileType:
		filename = stateFile
	case networkFileType:
		filename = networkFile
	case processFileType:
		filename = processFile
	case lockFileType:
		filename = lockFileName
		break
	default:
		return "", "", fmt.Errorf("Invalid pod resource")
	}

	filePath := filepath.Join(dirPath, filename)

	return filePath, dirPath, nil
}

func (fs *filesystem) containerURI(podID, containerID string, resource podResource) (string, string, error) {
	if containerID == "" {
		return "", "", fmt.Errorf("Container ID cannot be empty")
	}

	return fs.resourceURI(podID, containerID, resource)
}

func (fs *filesystem) podURI(podID string, resource podResource) (string, string, error) {
	return fs.resourceURI(podID, "", resource)
}

func (fs *filesystem) storeResource(podID, containerID string, resource podResource, data interface{}) error {
	switch file := data.(type) {
	case PodConfig, ContainerConfig:
		if resource != configFileType {
			return fmt.Errorf("Invalid pod resource")
		}

		configFile, _, err := fs.resourceURI(podID, containerID, configFileType)
		if err != nil {
			return err
		}

		return fs.storeFile(configFile, file)

	case State:
		if resource != stateFileType {
			return fmt.Errorf("Invalid pod resource")
		}

		stateFile, _, err := fs.resourceURI(podID, containerID, stateFileType)
		if err != nil {
			return err
		}

		return fs.storeFile(stateFile, file)

	case NetworkNamespace:
		if resource != networkFileType {
			return fmt.Errorf("Invalid pod resource")
		}

		networkFile, _, err := fs.resourceURI(podID, containerID, networkFileType)
		if err != nil {
			return err
		}

		return fs.storeFile(networkFile, file)

	case Process:
		if resource != processFileType {
			return fmt.Errorf("Invalid pod resource")
		}

		processFile, _, err := fs.resourceURI(podID, containerID, processFileType)
		if err != nil {
			return err
		}

		return fs.storeFile(processFile, file)

	default:
		return fmt.Errorf("Invalid resource data type")
	}
}

func (fs *filesystem) fetchResource(podID, containerID string, resource podResource) (interface{}, error) {
	path, _, err := fs.resourceURI(podID, containerID, resource)
	if err != nil {
		return nil, err
	}

	switch resource {
	case configFileType:
		if containerID == "" {
			config := PodConfig{}
			err = fs.fetchFile(path, &config)
			if err != nil {
				return nil, err
			}

			return config, nil
		}

		config := ContainerConfig{}
		err = fs.fetchFile(path, &config)
		if err != nil {
			return nil, err
		}

		return config, nil

	case stateFileType:
		state := State{}
		err = fs.fetchFile(path, &state)
		if err != nil {
			return nil, err
		}

		return state, nil

	case networkFileType:
		networkNS := NetworkNamespace{}
		err = fs.fetchFile(path, &networkNS)
		if err != nil {
			return nil, err
		}

		return networkNS, nil

	case processFileType:
		process := Process{}
		err = fs.fetchFile(path, &process)
		if err != nil {
			return nil, err
		}

		return process, nil
	}

	return nil, fmt.Errorf("Invalid pod resource")
}

func (fs *filesystem) storePodResource(podID string, resource podResource, data interface{}) error {
	return fs.storeResource(podID, "", resource, data)
}

func (fs *filesystem) fetchPodConfig(podID string) (PodConfig, error) {
	data, err := fs.fetchResource(podID, "", configFileType)
	if err != nil {
		return PodConfig{}, err
	}

	switch config := data.(type) {
	case PodConfig:
		return config, nil
	}

	return PodConfig{}, fmt.Errorf("Unknown config type")
}

func (fs *filesystem) fetchPodState(podID string) (State, error) {
	data, err := fs.fetchResource(podID, "", stateFileType)
	if err != nil {
		return State{}, err
	}

	switch state := data.(type) {
	case State:
		return state, nil
	}

	return State{}, fmt.Errorf("Unknown state type")
}

func (fs *filesystem) fetchPodNetwork(podID string) (NetworkNamespace, error) {
	data, err := fs.fetchResource(podID, "", networkFileType)
	if err != nil {
		return NetworkNamespace{}, err
	}

	switch networkNS := data.(type) {
	case NetworkNamespace:
		return networkNS, nil
	}

	return NetworkNamespace{}, fmt.Errorf("Unknown network type")
}

func (fs *filesystem) storePodNetwork(podID string, networkNS NetworkNamespace) error {
	return fs.storePodResource(podID, networkFileType, networkNS)
}

func (fs *filesystem) deletePodResources(podID string, resources []podResource) error {
	if resources == nil {
		resources = []podResource{configFileType, stateFileType}
	}

	for _, resource := range resources {
		_, dir, err := fs.podURI(podID, resource)
		if err != nil {
			return err
		}

		err = os.RemoveAll(dir)
		if err != nil {
			return err
		}
	}

	return nil
}

func (fs *filesystem) storeContainerResource(podID, containerID string, resource podResource, data interface{}) error {
	if containerID == "" {
		return fmt.Errorf("Container ID cannot be empty")
	}

	return fs.storeResource(podID, containerID, resource, data)
}

func (fs *filesystem) fetchContainerConfig(podID, containerID string) (ContainerConfig, error) {
	if containerID == "" {
		return ContainerConfig{}, fmt.Errorf("Container ID cannot be empty")
	}

	data, err := fs.fetchResource(podID, containerID, configFileType)
	if err != nil {
		return ContainerConfig{}, err
	}

	switch config := data.(type) {
	case ContainerConfig:
		return config, nil
	}

	return ContainerConfig{}, fmt.Errorf("Unknown config type")
}

func (fs *filesystem) fetchContainerState(podID, containerID string) (State, error) {
	if containerID == "" {
		return State{}, fmt.Errorf("Container ID cannot be empty")
	}

	data, err := fs.fetchResource(podID, containerID, stateFileType)
	if err != nil {
		return State{}, err
	}

	switch state := data.(type) {
	case State:
		return state, nil
	}

	return State{}, fmt.Errorf("Unknown state type")
}

func (fs *filesystem) fetchContainerProcess(podID, containerID string) (Process, error) {
	if containerID == "" {
		return Process{}, fmt.Errorf("Container ID cannot be empty")
	}

	data, err := fs.fetchResource(podID, containerID, processFileType)
	if err != nil {
		return Process{}, err
	}

	switch process := data.(type) {
	case Process:
		return process, nil
	}

	return Process{}, fmt.Errorf("Unknown process type")
}

func (fs *filesystem) storeContainerProcess(podID, containerID string, process Process) error {
	return fs.storeContainerResource(podID, containerID, processFileType, process)
}

func (fs *filesystem) deleteContainerResources(podID, containerID string, resources []podResource) error {
	if resources == nil {
		resources = []podResource{configFileType, stateFileType}
	}

	for _, resource := range resources {
		_, dir, err := fs.podURI(podID, resource)
		if err != nil {
			return err
		}

		containerDir := filepath.Join(dir, containerID, "/")

		err = os.RemoveAll(containerDir)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>filesystem: Remove duplication.<commit_after>//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package virtcontainers

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
)

// podResource is an int representing a pod resource type
type podResource int

const (
	// configFileType represents a configuration file type
	configFileType podResource = iota

	// stateFileType represents a state file type
	stateFileType

	// networkFileType represents a network file type
	networkFileType

	// processFileType represents a process file type
	processFileType

	// lockFileType represents a lock file type
	lockFileType
)

// configFile is the file name used for every JSON pod configuration.
const configFile = "config.json"

// stateFile is the file name storing a pod state.
const stateFile = "state.json"

// networkFile is the file name storing a pod network.
const networkFile = "network.json"

// processFile is the file name storing a container process.
const processFile = "process.json"

// lockFile is the file name locking the usage of a pod.
const lockFileName = "lock"

// dirMode is the permission bits used for creating a directory
const dirMode = os.FileMode(0750)

// storagePathSuffix is the suffix used for all storage paths
const storagePathSuffix = "/virtcontainers/pods"

// configStoragePath is the pod configuration directory.
// It will contain one config.json file for each created pod.
var configStoragePath = filepath.Join("/var/lib", storagePathSuffix)

// runStoragePath is the pod runtime directory.
// It will contain one state.json and one lock file for each created pod.
var runStoragePath = filepath.Join("/run", storagePathSuffix)

// resourceStorage is the virtcontainers resources (configuration, state, etc...)
// storage interface.
// The default resource storage implementation is filesystem.
type resourceStorage interface {
	// Create all resources for a pod
	createAllResources(pod Pod) error

	// Resources URIs functions return both the URI
	// for the actual resource and the URI base.
	containerURI(podID, containerID string, resource podResource) (string, string, error)
	podURI(podID string, resource podResource) (string, string, error)

	// Pod resources
	storePodResource(podID string, resource podResource, data interface{}) error
	deletePodResources(podID string, resources []podResource) error
	fetchPodConfig(podID string) (PodConfig, error)
	fetchPodState(podID string) (State, error)
	fetchPodNetwork(podID string) (NetworkNamespace, error)
	storePodNetwork(podID string, networkNS NetworkNamespace) error

	// Container resources
	storeContainerResource(podID, containerID string, resource podResource, data interface{}) error
	deleteContainerResources(podID, containerID string, resources []podResource) error
	fetchContainerConfig(podID, containerID string) (ContainerConfig, error)
	fetchContainerState(podID, containerID string) (State, error)
	fetchContainerProcess(podID, containerID string) (Process, error)
	storeContainerProcess(podID, containerID string, process Process) error
}

// filesystem is a resourceStorage interface implementation for a local filesystem.
type filesystem struct {
}

func (fs *filesystem) createAllResources(pod Pod) (err error) {
	for _, resource := range []podResource{stateFileType, configFileType} {
		_, path, _ := fs.podURI(pod.id, resource)
		err = os.MkdirAll(path, os.ModeDir)
		if err != nil {
			return err
		}
	}

	for _, container := range pod.containers {
		for _, resource := range []podResource{stateFileType, configFileType} {
			_, path, _ := fs.containerURI(pod.id, container.id, resource)
			err = os.MkdirAll(path, os.ModeDir)
			if err != nil {
				fs.deletePodResources(pod.id, nil)
				return err
			}
		}
	}

	podlockFile, _, err := fs.podURI(pod.id, lockFileType)
	if err != nil {
		fs.deletePodResources(pod.id, nil)
		return err
	}

	_, err = os.Stat(podlockFile)
	if err != nil {
		lockFile, err := os.Create(podlockFile)
		if err != nil {
			fs.deletePodResources(pod.id, nil)
			return err
		}
		lockFile.Close()
	}

	return nil
}

func (fs *filesystem) storeFile(path string, data interface{}) error {
	_, err := os.Stat(path)
	if err == nil {
		os.Remove(path)
	}

	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer f.Close()

	jsonOut, err := json.Marshal(data)
	if err != nil {
		return fmt.Errorf("Could not marshall data: %s", err)
	}
	f.Write(jsonOut)

	return nil
}

func (fs *filesystem) fetchFile(path string, data interface{}) error {
	_, err := os.Stat(path)
	if err != nil {
		return err
	}

	fileData, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	err = json.Unmarshal([]byte(string(fileData)), data)
	if err != nil {
		return err
	}

	return nil
}

func resourceDir(podID, containerID string, resource podResource) (string, error) {
	var path string

	if podID == "" {
		return "", fmt.Errorf("PodID cannot be empty")
	}

	switch resource {
	case configFileType:
		path = configStoragePath
		break
	case stateFileType, networkFileType, processFileType, lockFileType:
		path = runStoragePath
		break
	default:
		return "", fmt.Errorf("Invalid pod resource")
	}

	dirPath := filepath.Join(path, podID, containerID)

	return dirPath, nil
}

func (fs *filesystem) resourceURI(podID, containerID string, resource podResource) (string, string, error) {
	var filename string

	if podID == "" {
		return "", "", fmt.Errorf("Pod ID cannot be empty")
	}

	dirPath, err := resourceDir(podID, containerID, resource)
	if err != nil {
		return "", "", err
	}

	switch resource {
	case configFileType:
		filename = configFile
		break
	case stateFileType:
		filename = stateFile
	case networkFileType:
		filename = networkFile
	case processFileType:
		filename = processFile
	case lockFileType:
		filename = lockFileName
		break
	default:
		return "", "", fmt.Errorf("Invalid pod resource")
	}

	filePath := filepath.Join(dirPath, filename)

	return filePath, dirPath, nil
}

func (fs *filesystem) containerURI(podID, containerID string, resource podResource) (string, string, error) {
	if containerID == "" {
		return "", "", fmt.Errorf("Container ID cannot be empty")
	}

	return fs.resourceURI(podID, containerID, resource)
}

func (fs *filesystem) podURI(podID string, resource podResource) (string, string, error) {
	return fs.resourceURI(podID, "", resource)
}

func (fs *filesystem) storeResource(podID, containerID string, resource podResource, data interface{}) error {
	switch file := data.(type) {
	case PodConfig, ContainerConfig:
		if resource != configFileType {
			return fmt.Errorf("Invalid pod resource")
		}

		configFile, _, err := fs.resourceURI(podID, containerID, configFileType)
		if err != nil {
			return err
		}

		return fs.storeFile(configFile, file)

	case State:
		if resource != stateFileType {
			return fmt.Errorf("Invalid pod resource")
		}

		stateFile, _, err := fs.resourceURI(podID, containerID, stateFileType)
		if err != nil {
			return err
		}

		return fs.storeFile(stateFile, file)

	case NetworkNamespace:
		if resource != networkFileType {
			return fmt.Errorf("Invalid pod resource")
		}

		networkFile, _, err := fs.resourceURI(podID, containerID, networkFileType)
		if err != nil {
			return err
		}

		return fs.storeFile(networkFile, file)

	case Process:
		if resource != processFileType {
			return fmt.Errorf("Invalid pod resource")
		}

		processFile, _, err := fs.resourceURI(podID, containerID, processFileType)
		if err != nil {
			return err
		}

		return fs.storeFile(processFile, file)

	default:
		return fmt.Errorf("Invalid resource data type")
	}
}

func (fs *filesystem) fetchResource(podID, containerID string, resource podResource) (interface{}, error) {
	path, _, err := fs.resourceURI(podID, containerID, resource)
	if err != nil {
		return nil, err
	}

	switch resource {
	case configFileType:
		if containerID == "" {
			config := PodConfig{}
			err = fs.fetchFile(path, &config)
			if err != nil {
				return nil, err
			}

			return config, nil
		}

		config := ContainerConfig{}
		err = fs.fetchFile(path, &config)
		if err != nil {
			return nil, err
		}

		return config, nil

	case stateFileType:
		state := State{}
		err = fs.fetchFile(path, &state)
		if err != nil {
			return nil, err
		}

		return state, nil

	case networkFileType:
		networkNS := NetworkNamespace{}
		err = fs.fetchFile(path, &networkNS)
		if err != nil {
			return nil, err
		}

		return networkNS, nil

	case processFileType:
		process := Process{}
		err = fs.fetchFile(path, &process)
		if err != nil {
			return nil, err
		}

		return process, nil
	}

	return nil, fmt.Errorf("Invalid pod resource")
}

func (fs *filesystem) storePodResource(podID string, resource podResource, data interface{}) error {
	return fs.storeResource(podID, "", resource, data)
}

func (fs *filesystem) fetchPodConfig(podID string) (PodConfig, error) {
	data, err := fs.fetchResource(podID, "", configFileType)
	if err != nil {
		return PodConfig{}, err
	}

	switch config := data.(type) {
	case PodConfig:
		return config, nil
	}

	return PodConfig{}, fmt.Errorf("Unknown config type")
}

func (fs *filesystem) fetchPodState(podID string) (State, error) {
	data, err := fs.fetchResource(podID, "", stateFileType)
	if err != nil {
		return State{}, err
	}

	switch state := data.(type) {
	case State:
		return state, nil
	}

	return State{}, fmt.Errorf("Unknown state type")
}

func (fs *filesystem) fetchPodNetwork(podID string) (NetworkNamespace, error) {
	data, err := fs.fetchResource(podID, "", networkFileType)
	if err != nil {
		return NetworkNamespace{}, err
	}

	switch networkNS := data.(type) {
	case NetworkNamespace:
		return networkNS, nil
	}

	return NetworkNamespace{}, fmt.Errorf("Unknown network type")
}

func (fs *filesystem) storePodNetwork(podID string, networkNS NetworkNamespace) error {
	return fs.storePodResource(podID, networkFileType, networkNS)
}

func (fs *filesystem) deletePodResources(podID string, resources []podResource) error {
	if resources == nil {
		resources = []podResource{configFileType, stateFileType}
	}

	for _, resource := range resources {
		_, dir, err := fs.podURI(podID, resource)
		if err != nil {
			return err
		}

		err = os.RemoveAll(dir)
		if err != nil {
			return err
		}
	}

	return nil
}

func (fs *filesystem) storeContainerResource(podID, containerID string, resource podResource, data interface{}) error {
	if containerID == "" {
		return fmt.Errorf("Container ID cannot be empty")
	}

	return fs.storeResource(podID, containerID, resource, data)
}

func (fs *filesystem) fetchContainerConfig(podID, containerID string) (ContainerConfig, error) {
	if containerID == "" {
		return ContainerConfig{}, fmt.Errorf("Container ID cannot be empty")
	}

	data, err := fs.fetchResource(podID, containerID, configFileType)
	if err != nil {
		return ContainerConfig{}, err
	}

	switch config := data.(type) {
	case ContainerConfig:
		return config, nil
	}

	return ContainerConfig{}, fmt.Errorf("Unknown config type")
}

func (fs *filesystem) fetchContainerState(podID, containerID string) (State, error) {
	if containerID == "" {
		return State{}, fmt.Errorf("Container ID cannot be empty")
	}

	data, err := fs.fetchResource(podID, containerID, stateFileType)
	if err != nil {
		return State{}, err
	}

	switch state := data.(type) {
	case State:
		return state, nil
	}

	return State{}, fmt.Errorf("Unknown state type")
}

func (fs *filesystem) fetchContainerProcess(podID, containerID string) (Process, error) {
	if containerID == "" {
		return Process{}, fmt.Errorf("Container ID cannot be empty")
	}

	data, err := fs.fetchResource(podID, containerID, processFileType)
	if err != nil {
		return Process{}, err
	}

	switch process := data.(type) {
	case Process:
		return process, nil
	}

	return Process{}, fmt.Errorf("Unknown process type")
}

func (fs *filesystem) storeContainerProcess(podID, containerID string, process Process) error {
	return fs.storeContainerResource(podID, containerID, processFileType, process)
}

func (fs *filesystem) deleteContainerResources(podID, containerID string, resources []podResource) error {
	if resources == nil {
		resources = []podResource{configFileType, stateFileType}
	}

	for _, resource := range resources {
		_, dir, err := fs.podURI(podID, resource)
		if err != nil {
			return err
		}

		containerDir := filepath.Join(dir, containerID, "/")

		err = os.RemoveAll(containerDir)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>