<commit_before>package mgorm

import (
	"errors"
	// "fmt"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

var session *mgo.Session
var db *mgo.Database

func InitDB(connectString, dbName string) error {
	session, err := mgo.Dial(connectString)
	if err != nil {
		return err
	}

	db = session.DB(dbName)
	return nil
}

func DB() *mgo.Database {
	return db
}

func FindAll(model IModel, criteria *Criteria) *Query {
	if !model.HasInited() {
		model.Init()
	}

	q := model.Collection().Find(criteria.GetConditions())
	criteriaSelects := criteria.GetSelect()
	if 0 < len(criteriaSelects) {
		selects := map[string]bool{}
		for _, field := range criteriaSelects {
			selects[field] = true
		}

		q.Select(selects)
	}

	if 0 < criteria.GetLimit() {
		q.Limit(criteria.GetLimit())
	}

	if 0 < criteria.GetOffset() {
		q.Skip(criteria.GetOffset())
	}

	if nil != criteria.GetSort() {
		sort := criteria.GetSort()
		sortStr := []string{}
		for key, value := range sort {
			if 0 < value {
				sortStr = append(sortStr, key)
			} else {
				sortStr = append(sortStr, "-"+key)
			}
		}

		q.Sort(sortStr...)
	}

	query := new(Query)
	query.SetQuery(q)
	return query
}

func Find(model IModel, criteria *Criteria) error {
	criteria.SetLimit(1)
	iter := FindAll(model, criteria).Iter()
	defer iter.Close()
	if iter.Next(model) {
		return nil
	}

	return errors.New("Not found")
}

func FindById(model IModel, id string) error {
	criteria := NewCriteria()
	criteria.AddCond("_id", "==", bson.ObjectIdHex(id))
	criteria.SetLimit(1)
	err := FindAll(model, criteria).GetQuery().One(model)

	if nil == err {
		model.Init()
		model.AfterFind()
	}

	return err
}

func Update(model IModel) bool {
	if model.IsNew() {
		model.AddError("the model is a new record")
		return false
	}

	if "" == model.GetId().Hex() {
		model.AddError("the id is empty")
		return false
	}

	err := model.Collection().UpdateId(model.GetId(), model)
	if nil != err {
		model.AddError(err.Error())
		return false
	}

	return true
}

func Insert(model IModel) bool {
	if !model.IsNew() {
		model.AddError("the model is not a new record")
		return false
	}

	err := model.Collection().Insert(model)
	if nil != err {
		model.AddError(err.Error())
		return false
	}

	return true
}

func Save(model IModel) bool {

	err := model.BeforeSave()
	if nil != err {
		model.AddError(err.Error())
		return false
	}

	res := false
	if model.IsNew() {
		res = Insert(model)
	} else {
		res = Update(model)
	}

	if res {
		model.AfterSave()
	}

	return res
}
<commit_msg>Fix the missing validate in save method.<commit_after>package mgorm

import (
	"errors"
	// "fmt"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

var session *mgo.Session
var db *mgo.Database

func InitDB(connectString, dbName string) error {
	session, err := mgo.Dial(connectString)
	if err != nil {
		return err
	}

	db = session.DB(dbName)
	return nil
}

func DB() *mgo.Database {
	return db
}

func FindAll(model IModel, criteria *Criteria) *Query {
	if !model.HasInited() {
		model.Init()
	}

	q := model.Collection().Find(criteria.GetConditions())
	criteriaSelects := criteria.GetSelect()
	if 0 < len(criteriaSelects) {
		selects := map[string]bool{}
		for _, field := range criteriaSelects {
			selects[field] = true
		}

		q.Select(selects)
	}

	if 0 < criteria.GetLimit() {
		q.Limit(criteria.GetLimit())
	}

	if 0 < criteria.GetOffset() {
		q.Skip(criteria.GetOffset())
	}

	if nil != criteria.GetSort() {
		sort := criteria.GetSort()
		sortStr := []string{}
		for key, value := range sort {
			if 0 < value {
				sortStr = append(sortStr, key)
			} else {
				sortStr = append(sortStr, "-"+key)
			}
		}

		q.Sort(sortStr...)
	}

	query := new(Query)
	query.SetQuery(q)
	return query
}

func Find(model IModel, criteria *Criteria) error {
	criteria.SetLimit(1)
	iter := FindAll(model, criteria).Iter()
	defer iter.Close()
	if iter.Next(model) {
		return nil
	}

	return errors.New("Not found")
}

func FindById(model IModel, id string) error {
	criteria := NewCriteria()
	criteria.AddCond("_id", "==", bson.ObjectIdHex(id))
	criteria.SetLimit(1)
	err := FindAll(model, criteria).GetQuery().One(model)

	if nil == err {
		model.Init()
		model.AfterFind()
	}

	return err
}

func Update(model IModel) bool {
	if model.IsNew() {
		model.AddError("the model is a new record")
		return false
	}

	if "" == model.GetId().Hex() {
		model.AddError("the id is empty")
		return false
	}

	err := model.Collection().UpdateId(model.GetId(), model)
	if nil != err {
		model.AddError(err.Error())
		return false
	}

	return true
}

func Insert(model IModel) bool {
	if !model.IsNew() {
		model.AddError("the model is not a new record")
		return false
	}

	err := model.Collection().Insert(model)
	if nil != err {
		model.AddError(err.Error())
		return false
	}

	return true
}

func Save(model IModel) bool {

	if !model.Validate() {
		return false
	}

	err := model.BeforeSave()
	if nil != err {
		model.AddError(err.Error())
		return false
	}

	res := false
	if model.IsNew() {
		res = Insert(model)
	} else {
		res = Update(model)
	}

	if res {
		model.AfterSave()
	}

	return res
}
<|endoftext|>


<commit_before>package game

import (
	"fmt"
	"net/http"
	"text/template"

	"github.com/ghthor/engine/rpg2d"
	"github.com/ghthor/engine/rpg2d/coord"
	"github.com/ghthor/engine/rpg2d/quad"
	"github.com/ghthor/engine/sim/stime"
)

type inputPhase struct{}
type narrowPhase struct{}

func (inputPhase) ApplyInputsIn(c quad.Chunk, now stime.Time) quad.Chunk {
	for _, e := range c.Entities {
		switch a := e.(type) {
		case actor:
			input := a.ReadInput()
			fmt.Println(input)

			// Naively apply input to actor
		}
	}
	return c
}

func (narrowPhase) ResolveCollisions(c quad.CollisionGroup, now stime.Time) quad.CollisionGroup {
	return c
}

type serveIndex struct {
	tmpl     *template.Template
	settings clientSettings
}

type clientSettings struct {
	JsMain       string
	WebsocketURL string
	Simulation   simulationSettings
}

type simulationSettings struct {
	Width, Height int
}

func (html serveIndex) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	err := html.tmpl.Execute(w, html.settings)

	if err != nil {
		http.Error(w, fmt.Sprint("template error:", err), http.StatusInternalServerError)
	}
}

type ShardConfig struct {
	// Is the resultant http server going to be
	// run using TLS or just standard HTTP.
	IsHTTPS bool

	// LAddr for the http server
	LAddr,

	// Path to javascript directory
	JsDir,

	// The javascript module that require.js should
	// call as the javascript main.
	JsMain string

	// A template for the index page. The template
	// will be executed with a game.ClientSettings{} struct.
	IndexTmpl *template.Template

	// A mux that http server will use. Is provided so the
	// user can extend the server with additional routes.
	Mux *http.ServeMux
}

func NewSimShard(c ShardConfig) (*http.Server, error) {
	// TODO pull this information from a datastore
	quadTree, err := quad.New(coord.Bounds{
		coord.Cell{-1024, 1024},
		coord.Cell{1023, -1023},
	}, 40, nil)

	if err != nil {
		return nil, err
	}

	now := stime.Time(0)

	simDef := rpg2d.SimulationDef{
		FPS: 40,

		// Initial World State
		QuadTree: quadTree,
		Now:      now,

		InputPhaseHandler:  inputPhase{},
		NarrowPhaseHandler: narrowPhase{},
	}

	runningSim, err := simDef.Begin()
	if err != nil {
		return nil, err
	}

	wsRoute := "/actor/socket"
	var wsUrl string
	if c.IsHTTPS {
		wsUrl = "wss://"
	} else {
		wsUrl = "ws://"
	}

	wsUrl += c.LAddr + wsRoute

	indexHandler := serveIndex{
		c.IndexTmpl,
		clientSettings{
			c.JsMain,
			wsUrl,
			simulationSettings{
				Width:  quadTree.Bounds().Width(),
				Height: quadTree.Bounds().Height(),
			},
		},
	}

	mux := c.Mux

	mux.Handle("/", indexHandler)
	mux.Handle("/js/", http.StripPrefix("/js/", http.FileServer(http.Dir(c.JsDir))))
	mux.Handle(wsRoute, newWebsocketActorHandler(runningSim))

	return &http.Server{
		Addr:    c.LAddr,
		Handler: mux,
	}, nil
}
<commit_msg>Export and document the struct provided to the index template<commit_after>package game

import (
	"fmt"
	"net/http"
	"text/template"

	"github.com/ghthor/engine/rpg2d"
	"github.com/ghthor/engine/rpg2d/coord"
	"github.com/ghthor/engine/rpg2d/quad"
	"github.com/ghthor/engine/sim/stime"
)

type inputPhase struct{}
type narrowPhase struct{}

func (inputPhase) ApplyInputsIn(c quad.Chunk, now stime.Time) quad.Chunk {
	for _, e := range c.Entities {
		switch a := e.(type) {
		case actor:
			input := a.ReadInput()
			fmt.Println(input)

			// Naively apply input to actor
		}
	}
	return c
}

func (narrowPhase) ResolveCollisions(c quad.CollisionGroup, now stime.Time) quad.CollisionGroup {
	return c
}

type serveIndex struct {
	tmpl     *template.Template
	settings ClientSettings
}

// A set of data that the index page template will
// have access to when it is executed.
type ClientSettings struct {
	JsMain       string
	WebsocketURL string
	Simulation   SimulationSettings
}

// Part of the ClientSettings that are rpg2d.Simulation specific
type SimulationSettings struct {
	Width, Height int
}

func (html serveIndex) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	err := html.tmpl.Execute(w, html.settings)

	if err != nil {
		http.Error(w, fmt.Sprint("template error:", err), http.StatusInternalServerError)
	}
}

type ShardConfig struct {
	// Is the resultant http server going to be
	// run using TLS or just standard HTTP.
	IsHTTPS bool

	// LAddr for the http server
	LAddr,

	// Path to javascript directory
	JsDir,

	// The javascript module that require.js should
	// call as the javascript main.
	JsMain string

	// A template for the index page. The template
	// will be executed with a game.ClientSettings{} struct.
	IndexTmpl *template.Template

	// A mux that http server will use. Is provided so the
	// user can extend the server with additional routes.
	Mux *http.ServeMux
}

func NewSimShard(c ShardConfig) (*http.Server, error) {
	// TODO pull this information from a datastore
	quadTree, err := quad.New(coord.Bounds{
		coord.Cell{-1024, 1024},
		coord.Cell{1023, -1023},
	}, 40, nil)

	if err != nil {
		return nil, err
	}

	now := stime.Time(0)

	simDef := rpg2d.SimulationDef{
		FPS: 40,

		// Initial World State
		QuadTree: quadTree,
		Now:      now,

		InputPhaseHandler:  inputPhase{},
		NarrowPhaseHandler: narrowPhase{},
	}

	runningSim, err := simDef.Begin()
	if err != nil {
		return nil, err
	}

	wsRoute := "/actor/socket"
	var wsUrl string
	if c.IsHTTPS {
		wsUrl = "wss://"
	} else {
		wsUrl = "ws://"
	}

	wsUrl += c.LAddr + wsRoute

	indexHandler := serveIndex{
		c.IndexTmpl,
		ClientSettings{
			c.JsMain,
			wsUrl,
			SimulationSettings{
				Width:  quadTree.Bounds().Width(),
				Height: quadTree.Bounds().Height(),
			},
		},
	}

	mux := c.Mux

	mux.Handle("/", indexHandler)
	mux.Handle("/js/", http.StripPrefix("/js/", http.FileServer(http.Dir(c.JsDir))))
	mux.Handle(wsRoute, newWebsocketActorHandler(runningSim))

	return &http.Server{
		Addr:    c.LAddr,
		Handler: mux,
	}, nil
}
<|endoftext|>


<commit_before>package crdt

import (
	"strconv"

	"github.com/koding/redis"
)

// GCounter is a grow-only counter (inspired by vector clocks) in which only
// increment and merge are possible. Divergent histories are resolved by taking
// the maximum count for the counter.  The value of the counter is the sum of
// all counts.
//
// TODO implement merge!!
type GCounter struct {
	ccrdt *CRDT
	key   string
}

// NewGCounter creates a new GCounter
func (c *CRDT) NewGCounter(key string) *GCounter {
	return &GCounter{
		ccrdt: c,
		key:   key,
	}
}

// Add adds item to the GCounter with a given delta
func (g *GCounter) Add(delta int64) error {

	errCount := 0
	var lastErr error

	// add goroutine support
	for _, c := range g.ccrdt.sessions.All() {

		// TODO we can do read-repair here
		// redis returns lastest value
		_, err := c.IncrBy(g.key, delta)
		if err != nil {
			lastErr = err
			errCount++
		}
	}

	// at least we have one success
	if errCount != g.ccrdt.sessions.Count() {
		return nil
	}

	return lastErr
}

// Merge returns the sum of all the actors
func (g *GCounter) Merge() (int64, error) {
	var res int64
	var repairNeeded bool
	values := make(map[*redis.RedisSession]int64)

	for i, c := range g.ccrdt.sessions.All() {
		val, err := c.Get(g.key)
		if err != nil && err != redis.ErrNil {
			// ignore the errors
			// return 0, err
		}

		if val == "" {
			val = "0"
		}

		d, err := strconv.ParseInt(val, 10, 64)
		if err != nil {
			// ignore the errors
			// return 0, err
		}

		// add data to a temp cache
		values[c] = d

		// if the `res`is smaller than the current value, previous ones should
		// be repaired
		if res < d {
			// if this is the first operation, ignore the case
			if i != 0 {
				repairNeeded = true
			}

			res = d
		}
	}

	if repairNeeded {
		for ses, per := range values {
			if res == per {
				continue
			}
			ses.IncrBy(g.key, res-per)
		}
	}

	return res, nil
}
<commit_msg>CRDT: remove TODO<commit_after>package crdt

import (
	"strconv"

	"github.com/koding/redis"
)

// GCounter is a grow-only counter (inspired by vector clocks) in which only
// increment and merge are possible. Divergent histories are resolved by taking
// the maximum count for the counter.  The value of the counter is the sum of
// all counts.
type GCounter struct {
	ccrdt *CRDT
	key   string
}

// NewGCounter creates a new GCounter
func (c *CRDT) NewGCounter(key string) *GCounter {
	return &GCounter{
		ccrdt: c,
		key:   key,
	}
}

// Add adds item to the GCounter with a given delta
func (g *GCounter) Add(delta int64) error {

	errCount := 0
	var lastErr error

	// add goroutine support
	for _, c := range g.ccrdt.sessions.All() {

		// TODO we can do read-repair here
		// redis returns lastest value
		_, err := c.IncrBy(g.key, delta)
		if err != nil {
			lastErr = err
			errCount++
		}
	}

	// at least we have one success
	if errCount != g.ccrdt.sessions.Count() {
		return nil
	}

	return lastErr
}

// Merge returns the sum of all the actors
func (g *GCounter) Merge() (int64, error) {
	var res int64
	var repairNeeded bool
	values := make(map[*redis.RedisSession]int64)

	for i, c := range g.ccrdt.sessions.All() {
		val, err := c.Get(g.key)
		if err != nil && err != redis.ErrNil {
			// ignore the errors
			// return 0, err
		}

		if val == "" {
			val = "0"
		}

		d, err := strconv.ParseInt(val, 10, 64)
		if err != nil {
			// ignore the errors
			// return 0, err
		}

		// add data to a temp cache
		values[c] = d

		// if the `res`is smaller than the current value, previous ones should
		// be repaired
		if res < d {
			// if this is the first operation, ignore the case
			if i != 0 {
				repairNeeded = true
			}

			res = d
		}
	}

	if repairNeeded {
		for ses, per := range values {
			if res == per {
				continue
			}
			ses.IncrBy(g.key, res-per)
		}
	}

	return res, nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"text/template"
)

type dim struct {
	Z string
	M string
}

var dims = []dim{
	{Z: "", M: ""},
	{Z: "Z", M: ""},
	{Z: "", M: "M"},
	{Z: "Z", M: "M"},
}

const point = `package geom
{{range .Instances}}
type Point{{.Z}}{{.M}} struct {
	X float64
	Y float64{{with .Z}}
	Z float64{{end}}{{with .M}}
	M float64{{end}}
}

func (point{{.Z}}{{.M}} Point{{.Z}}{{.M}}) Bounds() *Bounds {
	return NewBoundsPoint{{.Z}}{{.M}}(point{{.Z}}{{.M}})
}
{{end}}`

const lineString = `package geom
{{range .Instances}}
type LineString{{.Z}}{{.M}} struct {
	Points []Point{{.Z}}{{.M}}
}

func (lineString{{.Z}}{{.M}} LineString{{.Z}}{{.M}}) Bounds() *Bounds {
	return NewBounds().ExtendPoint{{.Z}}{{.M}}s(lineString{{.Z}}{{.M}}.Points)
}
{{end}}`

var types = []struct {
	filename  string
	name      string
	template  string
	Instances []dim
}{
	{"geom/point.go", "Point", point, dims},
	{"geom/linestring.go", "LineString", lineString, dims},
}

func main() {

	for _, typ := range types {
		t := template.Must(template.New(typ.name).Parse(typ.template))
		if w, err := os.Create(typ.filename); err == nil {
			if err = t.Execute(w, typ); err != nil {
				log.Println("executing template:", err)
			}
		} else {
			log.Println("creating:", err)
		}
	}

}
<commit_msg>Generate polygon.go automatically<commit_after>package main

import (
	"log"
	"os"
	"text/template"
)

type dim struct {
	Z string
	M string
}

var dims = []dim{
	{Z: "", M: ""},
	{Z: "Z", M: ""},
	{Z: "", M: "M"},
	{Z: "Z", M: "M"},
}

const point = `package geom
{{range .Instances}}
type Point{{.Z}}{{.M}} struct {
	X float64
	Y float64{{with .Z}}
	Z float64{{end}}{{with .M}}
	M float64{{end}}
}

func (point{{.Z}}{{.M}} Point{{.Z}}{{.M}}) Bounds() *Bounds {
	return NewBoundsPoint{{.Z}}{{.M}}(point{{.Z}}{{.M}})
}
{{end}}`

const lineString = `package geom
{{range .Instances}}
type LineString{{.Z}}{{.M}} struct {
	Points []Point{{.Z}}{{.M}}
}

func (lineString{{.Z}}{{.M}} LineString{{.Z}}{{.M}}) Bounds() *Bounds {
	return NewBounds().ExtendPoint{{.Z}}{{.M}}s(lineString{{.Z}}{{.M}}.Points)
}
{{end}}`

const polygon = `package geom
{{range .Instances}}
type Polygon{{.Z}}{{.M}} struct {
	Rings [][]Point{{.Z}}{{.M}}
}

func (polygon{{.Z}}{{.M}} Polygon{{.Z}}{{.M}}) Bounds() *Bounds {
	return NewBounds().ExtendPoint{{.Z}}{{.M}}ss(polygon{{.Z}}{{.M}}.Rings)
}
{{end}}`

var types = []struct {
	filename  string
	name      string
	template  string
	Instances []dim
}{
	{"geom/point.go", "Point", point, dims},
	{"geom/linestring.go", "LineString", lineString, dims},
	{"geom/polygon.go", "Polygon", polygon, dims},
}

func main() {

	for _, typ := range types {
		t := template.Must(template.New(typ.name).Parse(typ.template))
		if w, err := os.Create(typ.filename); err == nil {
			if err = t.Execute(w, typ); err != nil {
				log.Println("executing template:", err)
			}
		} else {
			log.Println("creating:", err)
		}
	}

}
<|endoftext|>


<commit_before>package assets

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	"go/format"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"
)

type file struct {
	info os.FileInfo
	path string
}

// An asset generator. The generator can be used to generate an asset go file
// with all the assets that were added to the generator embedded into it.
// The generated assets are made available by the specified go variable
// VariableName which is of type assets.FileSystem.
type Generator struct {
	// The package name to generate assets in,
	PackageName string

	// The variable name containing the asset filesystem (defaults to Assets),
	VariableName string

	// Strip the specified prefix from all paths,
	StripPrefix string

	fsDirsMap  map[string][]string
	fsFilesMap map[string]file
}

func (x *Generator) addPath(parent string, prefix string, info os.FileInfo) error {
	p := path.Join(parent, info.Name())

	f := file{
		info: info,
		path: path.Join(prefix, p),
	}

	x.fsFilesMap[p] = f

	if info.IsDir() {
		f, err := os.Open(f.path)
		fi, err := f.Readdir(-1)
		f.Close()
		if err != nil {
			return err
		}

		x.fsDirsMap[p] = make([]string, 0, len(fi))

		for _, f := range fi {
			if err := x.addPath(p, prefix, f); err != nil {
				return err
			}
		}
	} else {
		x.appendFileInDir(parent, info.Name())
	}

	return nil
}

func (x *Generator) appendFileInDir(dir string, file string) {
	for _, v := range x.fsDirsMap[dir] {
		if v == file {
			return
		}
	}

	x.fsDirsMap[dir] = append(x.fsDirsMap[dir], file)
}

func (x *Generator) addParents(p string, prefix string) error {
	dname, fname := path.Split(p)

	if len(dname) == 0 {
		return nil
	}

	wosep := dname[0 : len(dname)-1]

	if err := x.addParents(wosep, prefix); err != nil {
		return err
	}

	if len(wosep) == 0 {
		wosep = "/"
	}

	x.appendFileInDir(wosep, fname)

	if _, ok := x.fsFilesMap[wosep]; !ok {
		pp := path.Join(prefix, wosep)
		s, err := os.Stat(pp)

		if err != nil {
			return err
		}

		x.fsFilesMap[wosep] = file{
			info: s,
			path: pp,
		}
	}

	return nil
}

func (x *Generator) splitRelPrefix(p string) (string, string) {
	i := 0
	relp := "../"

	for strings.HasPrefix(p[i:], relp) {
		i += len(relp)
	}

	return path.Join(p[0:i], "."), path.Join("/", p[i:])
}

// Add a file or directory asset to the generator. Added directories will be
// recursed automatically.
func (x *Generator) Add(p string) error {
	if x.fsFilesMap == nil {
		x.fsFilesMap = make(map[string]file)
	}

	if x.fsDirsMap == nil {
		x.fsDirsMap = make(map[string][]string)
	}

	p = path.Clean(p)

	info, err := os.Stat(p)

	if err != nil {
		return err
	}

	prefix, p := x.splitRelPrefix(p)

	if err := x.addParents(p, prefix); err != nil {
		return err
	}

	return x.addPath(path.Dir(p), prefix, info)
}

func (x *Generator) stripPrefix(p string) (string, bool) {
	if len(x.StripPrefix) == 0 {
		return p, true
	}

	if strings.HasPrefix(p, x.StripPrefix) {
		return p[len(x.StripPrefix):], true
	} else {
		return p, false
	}
}

// Write the asset tree specified in the generator to the given writer. The
// written asset tree is a valid, standalone go file with the assets
// embedded into it.
func (x *Generator) Write(wr io.Writer) error {
	p := x.PackageName

	if len(p) == 0 {
		p = "main"
	}

	variableName := x.VariableName

	if len(variableName) == 0 {
		variableName = "Assets"
	}

	writer := &bytes.Buffer{}

	// Write package and import
	fmt.Fprintf(writer, "package %s\n\n", p)
	fmt.Fprintln(writer, "import (")
	fmt.Fprintln(writer, "\t\"time\"")
	fmt.Fprintln(writer)
	fmt.Fprintln(writer, "\t\"github.com/jessevdk/go-assets\"")
	fmt.Fprintln(writer, ")")
	fmt.Fprintln(writer)

	vnames := make(map[string]string)

	// Write file contents as const strings
	if x.fsFilesMap != nil {
		// Create mapping from full file path to asset variable name.
		// This also reads the file and writes the contents as a const
		// string
		for k, v := range x.fsFilesMap {
			if v.info.IsDir() {
				continue
			}

			f, err := os.Open(v.path)

			if err != nil {
				return err
			}

			data, err := ioutil.ReadAll(f)

			f.Close()

			if err != nil {
				return err
			}

			s := sha1.New()
			io.WriteString(s, k)

			vname := fmt.Sprintf("__%s%x", variableName, s.Sum(nil))
			vnames[k] = vname

			fmt.Fprintf(writer, "var %s = %#v\n", vname, string(data))
		}

		fmt.Fprintln(writer)
	}

	fmt.Fprintf(writer, "var %s = assets.NewFileSystem(", variableName)

	if x.fsDirsMap == nil {
		x.fsDirsMap = make(map[string][]string)
	}

	if x.fsFilesMap == nil {
		x.fsFilesMap = make(map[string]file)
	}

	dirmap := make(map[string][]string)

	for k, v := range x.fsDirsMap {
		if kk, ok := x.stripPrefix(k); ok {
			if len(kk) == 0 {
				kk = "/"
			}

			dirmap[kk] = v
		}
	}

	fmt.Fprintf(writer, "%#v, ", dirmap)
	fmt.Fprintf(writer, "map[string]*assets.File{\n")

	// Write files
	for k, v := range x.fsFilesMap {
		kk, ok := x.stripPrefix(k)

		if !ok {
			continue
		}

		if len(kk) == 0 {
			kk = "/"
		}

		mt := v.info.ModTime()

		var dt string

		if !v.info.IsDir() {
			dt = "[]byte(" + vnames[k] + ")"
		} else {
			dt = "nil"
		}

		fmt.Fprintf(writer, "\t\t%#v: &assets.File{\n", kk)
		fmt.Fprintf(writer, "\t\t\tPath: %#v,\n", kk)
		fmt.Fprintf(writer, "\t\t\tFileMode: %#v,\n", v.info.Mode())
		fmt.Fprintf(writer, "\t\t\tMtime: time.Unix(%#v, %#v),\n", mt.Unix(), mt.UnixNano())
		fmt.Fprintf(writer, "\t\t\tData: %s,\n", dt)
		fmt.Fprintf(writer, "\t\t},")
	}

	fmt.Fprintln(writer, "\t}, \"\")")

	ret, err := format.Source(writer.Bytes())

	if err != nil {
		return err
	}

	wr.Write(ret)
	return nil
}
<commit_msg>avoid warnings if generated code test by `go vet`<commit_after>package assets

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	"go/format"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"
)

type file struct {
	info os.FileInfo
	path string
}

// An asset generator. The generator can be used to generate an asset go file
// with all the assets that were added to the generator embedded into it.
// The generated assets are made available by the specified go variable
// VariableName which is of type assets.FileSystem.
type Generator struct {
	// The package name to generate assets in,
	PackageName string

	// The variable name containing the asset filesystem (defaults to Assets),
	VariableName string

	// Strip the specified prefix from all paths,
	StripPrefix string

	fsDirsMap  map[string][]string
	fsFilesMap map[string]file
}

func (x *Generator) addPath(parent string, prefix string, info os.FileInfo) error {
	p := path.Join(parent, info.Name())

	f := file{
		info: info,
		path: path.Join(prefix, p),
	}

	x.fsFilesMap[p] = f

	if info.IsDir() {
		f, err := os.Open(f.path)
		fi, err := f.Readdir(-1)
		f.Close()
		if err != nil {
			return err
		}

		x.fsDirsMap[p] = make([]string, 0, len(fi))

		for _, f := range fi {
			if err := x.addPath(p, prefix, f); err != nil {
				return err
			}
		}
	} else {
		x.appendFileInDir(parent, info.Name())
	}

	return nil
}

func (x *Generator) appendFileInDir(dir string, file string) {
	for _, v := range x.fsDirsMap[dir] {
		if v == file {
			return
		}
	}

	x.fsDirsMap[dir] = append(x.fsDirsMap[dir], file)
}

func (x *Generator) addParents(p string, prefix string) error {
	dname, fname := path.Split(p)

	if len(dname) == 0 {
		return nil
	}

	wosep := dname[0 : len(dname)-1]

	if err := x.addParents(wosep, prefix); err != nil {
		return err
	}

	if len(wosep) == 0 {
		wosep = "/"
	}

	x.appendFileInDir(wosep, fname)

	if _, ok := x.fsFilesMap[wosep]; !ok {
		pp := path.Join(prefix, wosep)
		s, err := os.Stat(pp)

		if err != nil {
			return err
		}

		x.fsFilesMap[wosep] = file{
			info: s,
			path: pp,
		}
	}

	return nil
}

func (x *Generator) splitRelPrefix(p string) (string, string) {
	i := 0
	relp := "../"

	for strings.HasPrefix(p[i:], relp) {
		i += len(relp)
	}

	return path.Join(p[0:i], "."), path.Join("/", p[i:])
}

// Add a file or directory asset to the generator. Added directories will be
// recursed automatically.
func (x *Generator) Add(p string) error {
	if x.fsFilesMap == nil {
		x.fsFilesMap = make(map[string]file)
	}

	if x.fsDirsMap == nil {
		x.fsDirsMap = make(map[string][]string)
	}

	p = path.Clean(p)

	info, err := os.Stat(p)

	if err != nil {
		return err
	}

	prefix, p := x.splitRelPrefix(p)

	if err := x.addParents(p, prefix); err != nil {
		return err
	}

	return x.addPath(path.Dir(p), prefix, info)
}

func (x *Generator) stripPrefix(p string) (string, bool) {
	if len(x.StripPrefix) == 0 {
		return p, true
	}

	if strings.HasPrefix(p, x.StripPrefix) {
		return p[len(x.StripPrefix):], true
	} else {
		return p, false
	}
}

// Write the asset tree specified in the generator to the given writer. The
// written asset tree is a valid, standalone go file with the assets
// embedded into it.
func (x *Generator) Write(wr io.Writer) error {
	p := x.PackageName

	if len(p) == 0 {
		p = "main"
	}

	variableName := x.VariableName

	if len(variableName) == 0 {
		variableName = "Assets"
	}

	writer := &bytes.Buffer{}

	// Write package and import
	fmt.Fprintf(writer, "package %s\n\n", p)
	fmt.Fprintln(writer, "import (")
	fmt.Fprintln(writer, "\t\"time\"")
	fmt.Fprintln(writer)
	fmt.Fprintln(writer, "\t\"github.com/jessevdk/go-assets\"")
	fmt.Fprintln(writer, ")")
	fmt.Fprintln(writer)

	vnames := make(map[string]string)

	// Write file contents as const strings
	if x.fsFilesMap != nil {
		// Create mapping from full file path to asset variable name.
		// This also reads the file and writes the contents as a const
		// string
		for k, v := range x.fsFilesMap {
			if v.info.IsDir() {
				continue
			}

			f, err := os.Open(v.path)

			if err != nil {
				return err
			}

			data, err := ioutil.ReadAll(f)

			f.Close()

			if err != nil {
				return err
			}

			s := sha1.New()
			io.WriteString(s, k)

			vname := fmt.Sprintf("_%s%x", variableName, s.Sum(nil))
			vnames[k] = vname

			fmt.Fprintf(writer, "var %s = %#v\n", vname, string(data))
		}

		fmt.Fprintln(writer)
	}

	fmt.Fprintf(writer, "// %s returns go-assets FileSystem\n", variableName)
	fmt.Fprintf(writer, "var %s = assets.NewFileSystem(", variableName)

	if x.fsDirsMap == nil {
		x.fsDirsMap = make(map[string][]string)
	}

	if x.fsFilesMap == nil {
		x.fsFilesMap = make(map[string]file)
	}

	dirmap := make(map[string][]string)

	for k, v := range x.fsDirsMap {
		if kk, ok := x.stripPrefix(k); ok {
			if len(kk) == 0 {
				kk = "/"
			}

			dirmap[kk] = v
		}
	}

	fmt.Fprintf(writer, "%#v, ", dirmap)
	fmt.Fprintf(writer, "map[string]*assets.File{\n")

	// Write files
	for k, v := range x.fsFilesMap {
		kk, ok := x.stripPrefix(k)

		if !ok {
			continue
		}

		if len(kk) == 0 {
			kk = "/"
		}

		mt := v.info.ModTime()

		var dt string

		if !v.info.IsDir() {
			dt = "[]byte(" + vnames[k] + ")"
		} else {
			dt = "nil"
		}

		fmt.Fprintf(writer, "\t\t%#v: &assets.File{\n", kk)
		fmt.Fprintf(writer, "\t\t\tPath: %#v,\n", kk)
		fmt.Fprintf(writer, "\t\t\tFileMode: %#v,\n", v.info.Mode())
		fmt.Fprintf(writer, "\t\t\tMtime: time.Unix(%#v, %#v),\n", mt.Unix(), mt.UnixNano())
		fmt.Fprintf(writer, "\t\t\tData: %s,\n", dt)
		fmt.Fprintf(writer, "\t\t},")
	}

	fmt.Fprintln(writer, "\t}, \"\")")

	ret, err := format.Source(writer.Bytes())

	if err != nil {
		return err
	}

	wr.Write(ret)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"encoding/base32"
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"strconv"
	"time"
)

var totalPallets *uint64

const PALLET_NTIN_ID = 1462

var palletIndex *uint64
var palletNumber *uint64

const CASE_NTIN_ID = 1420

var caseIndex uint64 = 0
var casesInPallet uint64 = 4

const BUNDLE_NTIN_ID = 1419

var bundleIndex uint64 = 0
var bundlesInCase uint64 = 8

const PACKAGE_NTIN_ID = 1397

var packageIndex uint64 = 0
var packagesInBundle uint64 = 100

var ntins = make(map[int]string)

var c = make([]byte, 5)
var digits = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"

func getEncodedTimeStamp() string {
	time.Sleep(50 * time.Millisecond)
	l := uint64(time.Now().UnixNano() / 1000 / 1000 / 20)
	c[0] = digits[l&63]
	l = l >> 6
	c[1] = digits[l&63]
	l = l >> 6
	c[2] = digits[l&63]
	l = l >> 6
	c[3] = digits[l&63]
	l = l >> 6
	c[4] = digits[l&63]
	return base32.StdEncoding.EncodeToString(c)
}

func main() {
	palletNumber = flag.Uint64("numberOfPallets", 1, "number of pallets to produce")
	palletIndex = flag.Uint64("palletIndex", 0, "starting pallet index")
	totalPallets = flag.Uint64("totalPallets", 10, "total number of pallets")
	flag.Parse()

	rand.Seed(time.Now().UnixNano())

	itemsFile, err := os.Create("items_header.csv.gz")
	checkError("Cannot create file", err)
	defer itemsFile.Close()
	itemRelationFile, err := os.Create("itemrelations_header.csv.gz")
	checkError("Cannot create file", err)
	defer itemRelationFile.Close()
	lotRelationFile, err := os.Create("lotrelations_header.csv.gz")
	checkError("Cannot create file", err)
	defer lotRelationFile.Close()

	itemsGzip := gzip.NewWriter(itemsFile)
	defer itemsGzip.Close()
	defer itemsGzip.Flush()
	itemRelationGzip := gzip.NewWriter(itemRelationFile)
	defer itemRelationGzip.Close()
	defer itemRelationGzip.Flush()
	lotRelationGzip := gzip.NewWriter(lotRelationFile)
	defer lotRelationGzip.Close()
	defer lotRelationGzip.Flush()

	itemsWriter := csv.NewWriter(itemsGzip)
	defer itemsWriter.Flush()
	itemRelationWriter := csv.NewWriter(itemRelationGzip)
	defer itemRelationWriter.Flush()
	lotRelationWriter := csv.NewWriter(lotRelationGzip)
	defer lotRelationWriter.Flush()

	itemsWriter.Write([]string{"DbKey:ID(Item)", "Type:int", "Status:int", "Sequence:long", "Flags:string", "HelperCode:string"})
	itemRelationWriter.Write([]string{":START_ID(Item)", ":END_ID(Item)"})
	lotRelationWriter.Write([]string{":START_ID(Item)", ":END_ID(Lot)"})

	log.Println("Exporting NTINs")
	exportNtins()
	log.Println("done")

	log.Println("Exporting LOTs")
	exportLots()
	log.Println("done")

	createLot("BENCHMARK", (*palletNumber))
	createLot("W_BRA_AGG_ALLBOUND_01_001", (*totalPallets)-(*palletNumber))
}

func createLot(lot string, palletsForThisLot uint64) {

	itemsFile, err := os.Create("items" + lot + ".csv.gz")
	checkError("Cannot create file", err)
	defer itemsFile.Close()
	itemRelationFile, err := os.Create("itemrelations" + lot + ".csv.gz")
	checkError("Cannot create file", err)
	defer itemRelationFile.Close()
	lotRelationFile, err := os.Create("lotrelations" + lot + ".csv.gz")
	checkError("Cannot create file", err)
	defer lotRelationFile.Close()

	itemsGzip := gzip.NewWriter(itemsFile)
	defer itemsGzip.Close()
	defer itemsGzip.Flush()
	itemRelationGzip := gzip.NewWriter(itemRelationFile)
	defer itemRelationGzip.Close()
	defer itemRelationGzip.Flush()
	lotRelationGzip := gzip.NewWriter(lotRelationFile)
	defer lotRelationGzip.Close()
	defer lotRelationGzip.Flush()

	itemsWriter := csv.NewWriter(itemsGzip)
	defer itemsWriter.Flush()
	itemRelationWriter := csv.NewWriter(itemRelationGzip)
	defer itemRelationWriter.Flush()
	lotRelationWriter := csv.NewWriter(lotRelationGzip)
	defer lotRelationWriter.Flush()

	log.Println("Starting lot", lot)

	caseIndex = casesInPallet * (*palletIndex)
	casesForThisLot := palletsForThisLot * casesInPallet

	bundleIndex = bundlesInCase * caseIndex
	bundlesForThisLot := casesForThisLot * bundlesInCase

	packageIndex = packagesInBundle * bundleIndex
	packagesForThisLot := bundlesForThisLot * packagesInBundle

	log.Println("Aggregating", palletsForThisLot, "pallets starting from", (*palletIndex))
	log.Println("Aggregating", casesForThisLot, "cases starting from", caseIndex)
	log.Println("Aggregating", bundlesForThisLot, "bundles starting from", bundleIndex)
	log.Println("Creating", packagesForThisLot, "packages starting from", packageIndex)

	t := time.Now()

	var i uint64 = 0
	for ; i < palletsForThisLot; i++ {
		createPallet(PALLET_NTIN_ID, itemsWriter, itemRelationWriter, lotRelationWriter, lot)

		if (*palletIndex)%1000 == 0 {
			t2 := time.Since(t)

			palletPerSecond := float64((*palletIndex)) / t2.Seconds()
			eta := time.Duration(float64(palletsForThisLot)-float64((*palletIndex))*palletPerSecond) * time.Second
			log.Printf("PALLET %012d done (%v pallet/s) - (ETA: %v)", *palletIndex, palletPerSecond, eta)
		}

	}

	log.Println("lot terminated")
}

func exportLots() {
	file, err := os.Create("lots.csv.gz")
	checkError("Cannot create file", err)
	defer file.Close()

	gzw := gzip.NewWriter(file)
	defer gzw.Close()
	defer gzw.Flush()

	csvw := csv.NewWriter(gzw)
	defer csvw.Flush()

	csvw.Write([]string{"WorkOrderId:ID(Lot)", "Lot:string", "Manufactured:int", "Expiry:int"})
	csvw.Write([]string{"BENCHMARK", "BENCHMARK", "20161021", "20191231"})
	csvw.Write([]string{"W_BRA_AGG_ALLBOUND_01_001", "W_BRA_AGG_ALLBOUND_01_001", "20161021", "20191231"})
}

func exportNtins() {
	ntinsFile, err := os.Create("ntins.csv.gz")
	checkError("Cannot create file", err)
	defer ntinsFile.Close()

	ntinsGzip := gzip.NewWriter(ntinsFile)
	defer ntinsGzip.Close()
	defer ntinsGzip.Flush()

	ntinsWriter := csv.NewWriter(ntinsGzip)
	defer ntinsWriter.Flush()

	ntinsWriter.Write([]string{"DbKey:ID(NTIN)", "Id:int", "Ntin:string", "CodingRuleId:string"})

	dbkey := getEncodedTimeStamp()
	ntins[PALLET_NTIN_ID] = dbkey
	ntinsWriter.Write([]string{dbkey, strconv.Itoa(PALLET_NTIN_ID), "00575424", "GS1_SSCC"})

	dbkey = getEncodedTimeStamp()
	ntins[CASE_NTIN_ID] = dbkey
	ntinsWriter.Write([]string{dbkey, strconv.Itoa(CASE_NTIN_ID), "33255587", "GS1_SSCC"})

	dbkey = getEncodedTimeStamp()
	ntins[BUNDLE_NTIN_ID] = dbkey
	ntinsWriter.Write([]string{dbkey, strconv.Itoa(BUNDLE_NTIN_ID), "22847676", "GS1_SSCC"})

	dbkey = getEncodedTimeStamp()
	ntins[PACKAGE_NTIN_ID] = dbkey
	ntinsWriter.Write([]string{dbkey, strconv.Itoa(PACKAGE_NTIN_ID), "11483874937461", "GS1_SGTIN"})
}

func checkError(message string, err error) {
	if err != nil {
		log.Fatal(message, err)
	}
}

func createPallet(ntinId int, itemsWriter *csv.Writer, itemRelationWriter *csv.Writer, lotRelationWriter *csv.Writer, lot string) {

	fullKey := fmt.Sprintf("%s%012d", ntins[ntinId], *palletIndex)

	err := itemsWriter.Write([]string{fullKey, "400", "1", strconv.FormatUint(*palletIndex, 10), "", ""})
	checkError("Cannot create PALLET", err)

	var j uint64 = 0
	for ; j < casesInPallet; j++ {
		createCase(fullKey, CASE_NTIN_ID, itemsWriter, itemRelationWriter)
	}

	itemsWriter.Flush()
	itemRelationWriter.Flush()

	lotRelationWriter.Write([]string{fullKey, lot})
	lotRelationWriter.Flush()

	(*palletIndex)++
}

func createCase(parentFullKey string, ntinId int, itemsWriter *csv.Writer, itemRelationWriter *csv.Writer) {

	fullKey := fmt.Sprintf("%s%012d", ntins[ntinId], caseIndex)

	err := itemsWriter.Write([]string{fullKey, "300", "10", strconv.FormatUint(caseIndex, 10), "", ""})
	checkError("Cannot create CASE", err)

	var k uint64 = 0
	for ; k < bundlesInCase; k++ {
		createBundle(fullKey, BUNDLE_NTIN_ID, itemsWriter, itemRelationWriter)
	}

	err = itemRelationWriter.Write([]string{parentFullKey, fullKey})
	checkError("Cannot create CASE relation", err)

	caseIndex++
}

func createBundle(parentFullKey string, ntinId int, itemsWriter *csv.Writer, itemRelationWriter *csv.Writer) {

	fullKey := fmt.Sprintf("%s%012d", ntins[ntinId], bundleIndex)

	err := itemsWriter.Write([]string{fullKey, "200", "10", strconv.FormatUint(bundleIndex, 10), "", ""})
	checkError("Cannot create BUNDLE", err)

	var l uint64 = 0
	for ; l < packagesInBundle; l++ {
		createPackage(fullKey, PACKAGE_NTIN_ID, itemsWriter, itemRelationWriter)
	}

	err = itemRelationWriter.Write([]string{parentFullKey, fullKey})
	checkError("Cannot create BUNDLE relation", err)

	bundleIndex++
}

func createPackage(parentFullKey string, ntinId int, itemsWriter *csv.Writer, itemRelationWriter *csv.Writer) {

	fullKey := fmt.Sprintf("%s%012d", ntins[ntinId], packageIndex)

	err := itemsWriter.Write([]string{fullKey, "100", "10", strconv.FormatUint(packageIndex, 10), "", ""})
	checkError("Cannot create PACKAGE", err)

	err = itemRelationWriter.Write([]string{parentFullKey, fullKey})
	checkError("Cannot create PACKAGE relation", err)

	packageIndex++
}
<commit_msg>performance counters added<commit_after>package main

import (
	"compress/gzip"
	"encoding/base32"
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"math"
	"math/rand"
	"os"
	"strconv"
	"time"
)

var totalPallets *uint64

const PALLET_NTIN_ID = 1462

var palletIndex *uint64
var palletNumber *uint64

const CASE_NTIN_ID = 1420

var caseIndex uint64 = 0
var casesInPallet uint64 = 4

const BUNDLE_NTIN_ID = 1419

var bundleIndex uint64 = 0
var bundlesInCase uint64 = 8

const PACKAGE_NTIN_ID = 1397

var packageIndex uint64 = 0
var packagesInBundle uint64 = 100

var ntins = make(map[int]string)

var c = make([]byte, 5)
var digits = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"

func getEncodedTimeStamp() string {
	time.Sleep(50 * time.Millisecond)
	l := uint64(time.Now().UnixNano() / 1000 / 1000 / 20)
	c[0] = digits[l&63]
	l = l >> 6
	c[1] = digits[l&63]
	l = l >> 6
	c[2] = digits[l&63]
	l = l >> 6
	c[3] = digits[l&63]
	l = l >> 6
	c[4] = digits[l&63]
	return base32.StdEncoding.EncodeToString(c)
}

func main() {
	palletNumber = flag.Uint64("numberOfPallets", 1, "number of pallets to produce")
	palletIndex = flag.Uint64("palletIndex", 0, "starting pallet index")
	totalPallets = flag.Uint64("totalPallets", 10, "total number of pallets")
	flag.Parse()

	rand.Seed(time.Now().UnixNano())

	itemsFile, err := os.Create("items_header.csv.gz")
	checkError("Cannot create file", err)
	defer itemsFile.Close()
	itemRelationFile, err := os.Create("itemrelations_header.csv.gz")
	checkError("Cannot create file", err)
	defer itemRelationFile.Close()
	lotRelationFile, err := os.Create("lotrelations_header.csv.gz")
	checkError("Cannot create file", err)
	defer lotRelationFile.Close()

	itemsGzip := gzip.NewWriter(itemsFile)
	defer itemsGzip.Close()
	defer itemsGzip.Flush()
	itemRelationGzip := gzip.NewWriter(itemRelationFile)
	defer itemRelationGzip.Close()
	defer itemRelationGzip.Flush()
	lotRelationGzip := gzip.NewWriter(lotRelationFile)
	defer lotRelationGzip.Close()
	defer lotRelationGzip.Flush()

	itemsWriter := csv.NewWriter(itemsGzip)
	defer itemsWriter.Flush()
	itemRelationWriter := csv.NewWriter(itemRelationGzip)
	defer itemRelationWriter.Flush()
	lotRelationWriter := csv.NewWriter(lotRelationGzip)
	defer lotRelationWriter.Flush()

	itemsWriter.Write([]string{"DbKey:ID(Item)", "Type:int", "Status:int", "Sequence:long", "Flags:string", "HelperCode:string"})
	itemRelationWriter.Write([]string{":START_ID(Item)", ":END_ID(Item)"})
	lotRelationWriter.Write([]string{":START_ID(Item)", ":END_ID(Lot)"})

	log.Println("Exporting NTINs")
	exportNtins()
	log.Println("done")

	log.Println("Exporting LOTs")
	exportLots()
	log.Println("done")

	createLot("BENCHMARK", (*palletNumber))
	createLot("W_BRA_AGG_ALLBOUND_01_001", (*totalPallets)-(*palletNumber))
}

func createLot(lot string, palletsForThisLot uint64) {

	itemsFile, err := os.Create("items" + lot + ".csv.gz")
	checkError("Cannot create file", err)
	defer itemsFile.Close()
	itemRelationFile, err := os.Create("itemrelations" + lot + ".csv.gz")
	checkError("Cannot create file", err)
	defer itemRelationFile.Close()
	lotRelationFile, err := os.Create("lotrelations" + lot + ".csv.gz")
	checkError("Cannot create file", err)
	defer lotRelationFile.Close()

	itemsGzip := gzip.NewWriter(itemsFile)
	defer itemsGzip.Close()
	defer itemsGzip.Flush()
	itemRelationGzip := gzip.NewWriter(itemRelationFile)
	defer itemRelationGzip.Close()
	defer itemRelationGzip.Flush()
	lotRelationGzip := gzip.NewWriter(lotRelationFile)
	defer lotRelationGzip.Close()
	defer lotRelationGzip.Flush()

	itemsWriter := csv.NewWriter(itemsGzip)
	defer itemsWriter.Flush()
	itemRelationWriter := csv.NewWriter(itemRelationGzip)
	defer itemRelationWriter.Flush()
	lotRelationWriter := csv.NewWriter(lotRelationGzip)
	defer lotRelationWriter.Flush()

	log.Println("Starting lot", lot)

	caseIndex = casesInPallet * (*palletIndex)
	casesForThisLot := palletsForThisLot * casesInPallet

	bundleIndex = bundlesInCase * caseIndex
	bundlesForThisLot := casesForThisLot * bundlesInCase

	packageIndex = packagesInBundle * bundleIndex
	packagesForThisLot := bundlesForThisLot * packagesInBundle

	log.Println("Aggregating", palletsForThisLot, "pallets starting from", (*palletIndex))
	log.Println("Aggregating", casesForThisLot, "cases starting from", caseIndex)
	log.Println("Aggregating", bundlesForThisLot, "bundles starting from", bundleIndex)
	log.Println("Creating", packagesForThisLot, "packages starting from", packageIndex)

	t := time.Now()

	var i uint64 = 1
	for ; i <= palletsForThisLot; i++ {
		createPallet(PALLET_NTIN_ID, itemsWriter, itemRelationWriter, lotRelationWriter, lot)

		if i%1000 == 0 {
			t2 := time.Since(t)

			secondsElapsed := t2.Seconds()

			palletPerSecond := uint64(math.Floor(float64(*palletIndex) / secondsElapsed))
			remainingPalletsForThisLot := palletsForThisLot - i
			secondsRemaining := remainingPalletsForThisLot / palletPerSecond
			eta := time.Duration(secondsRemaining) * time.Second

			log.Printf("PALLET %012d aggregated (%v/%v) (%v pallet/s) - (ETA: %v)", *palletIndex, i, palletsForThisLot, palletPerSecond, eta)
		}

	}

	log.Println("lot terminated")
}

func exportLots() {
	file, err := os.Create("lots.csv.gz")
	checkError("Cannot create file", err)
	defer file.Close()

	gzw := gzip.NewWriter(file)
	defer gzw.Close()
	defer gzw.Flush()

	csvw := csv.NewWriter(gzw)
	defer csvw.Flush()

	csvw.Write([]string{"WorkOrderId:ID(Lot)", "Lot:string", "Manufactured:int", "Expiry:int"})
	csvw.Write([]string{"BENCHMARK", "BENCHMARK", "20161021", "20191231"})
	csvw.Write([]string{"W_BRA_AGG_ALLBOUND_01_001", "W_BRA_AGG_ALLBOUND_01_001", "20161021", "20191231"})
}

func exportNtins() {
	ntinsFile, err := os.Create("ntins.csv.gz")
	checkError("Cannot create file", err)
	defer ntinsFile.Close()

	ntinsGzip := gzip.NewWriter(ntinsFile)
	defer ntinsGzip.Close()
	defer ntinsGzip.Flush()

	ntinsWriter := csv.NewWriter(ntinsGzip)
	defer ntinsWriter.Flush()

	ntinsWriter.Write([]string{"DbKey:ID(NTIN)", "Id:int", "Ntin:string", "CodingRuleId:string"})

	dbkey := getEncodedTimeStamp()
	ntins[PALLET_NTIN_ID] = dbkey
	ntinsWriter.Write([]string{dbkey, strconv.Itoa(PALLET_NTIN_ID), "00575424", "GS1_SSCC"})

	dbkey = getEncodedTimeStamp()
	ntins[CASE_NTIN_ID] = dbkey
	ntinsWriter.Write([]string{dbkey, strconv.Itoa(CASE_NTIN_ID), "33255587", "GS1_SSCC"})

	dbkey = getEncodedTimeStamp()
	ntins[BUNDLE_NTIN_ID] = dbkey
	ntinsWriter.Write([]string{dbkey, strconv.Itoa(BUNDLE_NTIN_ID), "22847676", "GS1_SSCC"})

	dbkey = getEncodedTimeStamp()
	ntins[PACKAGE_NTIN_ID] = dbkey
	ntinsWriter.Write([]string{dbkey, strconv.Itoa(PACKAGE_NTIN_ID), "11483874937461", "GS1_SGTIN"})
}

func checkError(message string, err error) {
	if err != nil {
		log.Fatal(message, err)
	}
}

func createPallet(ntinId int, itemsWriter *csv.Writer, itemRelationWriter *csv.Writer, lotRelationWriter *csv.Writer, lot string) {

	fullKey := fmt.Sprintf("%s%012d", ntins[ntinId], *palletIndex)

	err := itemsWriter.Write([]string{fullKey, "400", "1", strconv.FormatUint(*palletIndex, 10), "", ""})
	checkError("Cannot create PALLET", err)

	var j uint64 = 0
	for ; j < casesInPallet; j++ {
		createCase(fullKey, CASE_NTIN_ID, itemsWriter, itemRelationWriter)
	}

	itemsWriter.Flush()
	itemRelationWriter.Flush()

	lotRelationWriter.Write([]string{fullKey, lot})
	lotRelationWriter.Flush()

	(*palletIndex)++
}

func createCase(parentFullKey string, ntinId int, itemsWriter *csv.Writer, itemRelationWriter *csv.Writer) {

	fullKey := fmt.Sprintf("%s%012d", ntins[ntinId], caseIndex)

	err := itemsWriter.Write([]string{fullKey, "300", "10", strconv.FormatUint(caseIndex, 10), "", ""})
	checkError("Cannot create CASE", err)

	var k uint64 = 0
	for ; k < bundlesInCase; k++ {
		createBundle(fullKey, BUNDLE_NTIN_ID, itemsWriter, itemRelationWriter)
	}

	err = itemRelationWriter.Write([]string{parentFullKey, fullKey})
	checkError("Cannot create CASE relation", err)

	caseIndex++
}

func createBundle(parentFullKey string, ntinId int, itemsWriter *csv.Writer, itemRelationWriter *csv.Writer) {

	fullKey := fmt.Sprintf("%s%012d", ntins[ntinId], bundleIndex)

	err := itemsWriter.Write([]string{fullKey, "200", "10", strconv.FormatUint(bundleIndex, 10), "", ""})
	checkError("Cannot create BUNDLE", err)

	var l uint64 = 0
	for ; l < packagesInBundle; l++ {
		createPackage(fullKey, PACKAGE_NTIN_ID, itemsWriter, itemRelationWriter)
	}

	err = itemRelationWriter.Write([]string{parentFullKey, fullKey})
	checkError("Cannot create BUNDLE relation", err)

	bundleIndex++
}

func createPackage(parentFullKey string, ntinId int, itemsWriter *csv.Writer, itemRelationWriter *csv.Writer) {

	fullKey := fmt.Sprintf("%s%012d", ntins[ntinId], packageIndex)

	err := itemsWriter.Write([]string{fullKey, "100", "10", strconv.FormatUint(packageIndex, 10), "", ""})
	checkError("Cannot create PACKAGE", err)

	err = itemRelationWriter.Write([]string{parentFullKey, fullKey})
	checkError("Cannot create PACKAGE relation", err)

	packageIndex++
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2015, Ian Chan <icha024@gmail.com>.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *  * Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *  * Neither the name of Redis nor the names of its contributors may be used
 *    to endorse or promote products derived from this software without
 *    specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */
package geoindex

/*
#include "geohash.h"
#include "geohash_helper.h"
#include <math.h>
#cgo LDFLAGS: -lm
*/
import "C"
import (
	"errors"
	"flag"
	"github.com/cznic/sortutil"
	"log"
	"math"
	"sort"
)

type GeoData struct {
	// Must be unique
	Id int
	// Generated automatically
	GeoHash uint64
	// User must specify these
	Latitude, Longitude float64
	Properties          *[]string
}

// GeoSlice sorted by GeoHash
type GeoSlice []*GeoData

func (s GeoSlice) Len() int           { return len(s) }
func (s GeoSlice) Less(i, j int) bool { return s[i].GeoHash < s[j].GeoHash }
func (s GeoSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

// GeoIdSlice sorted by ID
type GeoIdSlice []*GeoData

func (s GeoIdSlice) Len() int           { return len(s) }
func (s GeoIdSlice) Less(i, j int) bool { return s[i].Id < s[j].Id }
func (s GeoIdSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

var searchReady bool = false
var geoStore = make(map[string]GeoSlice)
var geoIdStore GeoIdSlice

const MAX_STEPS C.uint8_t = 26

// Debug logger. Remember to init flag.Parse() in main!!!
var debug *bool = flag.Bool("debugLib", false, "enable debug logging")

func Debugf(format string, args ...interface{}) {
	if *debug {
		log.Printf("DEBUG "+format, args...)
	}
}

func getNeighbours(hashBits uint64, steps uint8) []uint64 {
	var neighbours C.GeoHashNeighbors
	var hash C.GeoHashBits
	hash.bits = C.uint64_t(hashBits)
	hash.step = C.uint8_t(steps)
	C.geohashNeighbors(&hash, &neighbours)

	neighbourArr := sortutil.Uint64Slice{
		uint64(hashBits),
		uint64(neighbours.north.bits),
		uint64(neighbours.east.bits),
		uint64(neighbours.west.bits),
		uint64(neighbours.south.bits),
		uint64(neighbours.north_east.bits),
		uint64(neighbours.south_east.bits),
		uint64(neighbours.north_west.bits),
		uint64(neighbours.south_west.bits),
	}
	//	sort.Sort(neighbourArr)
	if steps <= 6 {
		sortutil.Dedupe(neighbourArr) // Can have duplicates if search range is large (>~5000km)
	}
	return neighbourArr
}

func GetLocation(id int) (geodata *GeoData, err error) {
	if id == 0 || id >= len(geoIdStore) {
		return nil, errors.New("index out of range")
	}
	return geoIdStore[id-1], nil
}

// Add geo location data to search index.
func AddCoord(provider string, geoData *GeoData) {
	hash := geohashEncodeMax(geoData.Latitude, geoData.Longitude)
	geoData.GeoHash = hash
	geoStore[provider] = append(geoStore[provider], geoData)
	geoStore["default"] = geoStore[provider]
	geoIdStore = append(geoIdStore, geoData)
	searchReady = false
}

// Encode a geo hash to MAX(26) steps
func geohashEncodeMax(latitude, longitude float64) uint64 {
	var hash C.GeoHashBits
	C.geohashEncodeWGS84(C.double(latitude), C.double(longitude), MAX_STEPS, &hash)
	return uint64(hash.bits)
}

// Sort the list of geo so search can happen. Normally automatically trigger by search.
func initSearch() {
	for provider := range geoStore {
		sort.Sort(geoStore[provider])
	}
	sort.Sort(geoIdStore)
	searchReady = true // might cause race cond, but assume add doesn't happen often.
}

// Search a latitude/longitude in an area bounded (km) for known location data.
func SearchBound(provider string, latitude, longitude, bound float64) []*GeoData {
	if !searchReady {
		initSearch()
	}
	hashSteps := C.geohashEstimateStepsByRadius(C.double(bound * 1000))
	Debugf("Hash step: %v, for radius: %v km", hashSteps, bound)

	var hash C.GeoHashBits
	C.geohashEncodeWGS84(C.double(latitude), C.double(longitude), C.uint8_t(hashSteps), &hash)
	neighbours := getNeighbours(uint64(hash.bits), uint8(hashSteps))
	box := boundingBox(latitude, longitude, bound)

	locationsFound := make([]*GeoData, 0)
	geoStoreKeysLen := len(geoStore[provider])
	for nIdx := range neighbours {
		neighboursUpperLimit := (neighbours[nIdx] + 1) << uint((MAX_STEPS-hashSteps)*2)
		neighbours[nIdx] = neighbours[nIdx] << uint((MAX_STEPS-hashSteps)*2)
		Debugf("Normalized Neighbours Hash: %v to %v", neighbours[nIdx], neighboursUpperLimit)
		searchIdx := sort.Search(geoStoreKeysLen, func(i int) bool { return geoStore[provider][i].GeoHash >= neighbours[nIdx] })
		if searchIdx < geoStoreKeysLen { // Not found would turn index=N
			Debugf("found location?")
			// found location
			for i := searchIdx; i < geoStoreKeysLen; i++ {
				if geoStore[provider][i].GeoHash < neighboursUpperLimit {
					data := geoStore[provider][i]
					Debugf("filtering by lat/long: %v %v", data.Latitude, data.Longitude)
					Debugf("filtering by bounding box: %v %v %v %v", box[0], box[1], box[2], box[3])
					// filter by strict bounding box
					if ((data.Latitude >= box[0] && data.Latitude <= box[1]) || (data.Latitude <= box[0] && data.Latitude >= box[1])) &&
						((data.Longitude >= box[2] && data.Longitude <= box[3]) || (data.Longitude <= box[2] && data.Longitude >= box[3])) {
						locationsFound = append(locationsFound, data)
						Debugf("Search found location in GeoStore: %v", geoStore[provider][searchIdx])
					}
				} else {
					break
				}
			}
		}
	}
	return locationsFound
}

// The approximate conversions are (doesn't fully correct for the Earth's polar flattening):
// Latitude: 1 deg = 110.574 km
// Longitude: 1 deg = 111.320*cos(latitude) km
// See: http://stackoverflow.com/questions/1253499/simple-calculations-for-working-with-lat-lon-km-distance
// Returns: min/max latitude, min/max longitude.
func boundingBox(latitude, longitude, boundKm float64) []float64 {
	latDiff := boundKm / 110.574
	longDiff := boundKm / (111.320 * math.Cos(latitude))
	minLatitude := latitude - latDiff
	maxLatitude := latitude + latDiff
	minLongitude := longitude - longDiff
	maxLongitude := longitude + longDiff

	Debugf("min lat: %v", minLatitude)
	Debugf("max lat: %v", maxLatitude)
	Debugf("min long: %v", minLongitude)
	Debugf("max long: %v", maxLongitude)

	return []float64{
		minLatitude,
		maxLatitude,
		minLongitude,
		maxLongitude,
	}
}
<commit_msg>Renamed methods and cleanup<commit_after>/*
 * Copyright (c) 2015, Ian Chan <icha024@gmail.com>.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *  * Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *  * Neither the name of Redis nor the names of its contributors may be used
 *    to endorse or promote products derived from this software without
 *    specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */
package geoindex

/*
#include "geohash.h"
#include "geohash_helper.h"
#include <math.h>
#cgo LDFLAGS: -lm
*/
import "C"
import (
	"errors"
	"flag"
	"github.com/cznic/sortutil"
	"log"
	"math"
	"sort"
)

type GeoData struct {
	// Must be unique
	Id int
	// Generated automatically
	GeoHash uint64
	// User must specify these
	Latitude, Longitude float64
	Properties          *[]string
}

// GeoSlice sorted by GeoHash
type GeoSlice []*GeoData

func (s GeoSlice) Len() int           { return len(s) }
func (s GeoSlice) Less(i, j int) bool { return s[i].GeoHash < s[j].GeoHash }
func (s GeoSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

// GeoIdSlice sorted by ID
type GeoIdSlice []*GeoData

func (s GeoIdSlice) Len() int           { return len(s) }
func (s GeoIdSlice) Less(i, j int) bool { return s[i].Id < s[j].Id }
func (s GeoIdSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

var searchReady bool = false
var geoStore = make(map[string]GeoSlice)
var geoIdStore GeoIdSlice

const MAX_STEPS C.uint8_t = 26

// Debug logger. Remember to init flag.Parse() in main!!!
var debug *bool = flag.Bool("debugLib", false, "enable debug logging")

func Debugf(format string, args ...interface{}) {
	if *debug {
		log.Printf("DEBUG "+format, args...)
	}
}

// AddLocation data to search index.
func AddLocation(provider string, geoData *GeoData) {
	hash := geohashEncodeMax(geoData.Latitude, geoData.Longitude)
	geoData.GeoHash = hash
	geoStore[provider] = append(geoStore[provider], geoData)
	geoStore["default"] = geoStore[provider]
	geoIdStore = append(geoIdStore, geoData)
	searchReady = false
}

// GetLocation data for a location ID.
func GetLocation(id int) (geodata *GeoData, err error) {
	if id == 0 || id >= len(geoIdStore) {
		return nil, errors.New("index out of range")
	}
	return geoIdStore[id-1], nil
}

// SearchLocations around latitude/longitude in bounded area (km) for known location points.
func SearchLocations(provider string, latitude, longitude, bound float64) []*GeoData {
	if !searchReady {
		initSearch()
	}
	hashSteps := C.geohashEstimateStepsByRadius(C.double(bound * 1000))
	Debugf("Hash step: %v, for radius: %v km", hashSteps, bound)

	var hash C.GeoHashBits
	C.geohashEncodeWGS84(C.double(latitude), C.double(longitude), C.uint8_t(hashSteps), &hash)
	neighbours := getNeighbours(uint64(hash.bits), uint8(hashSteps))
	box := boundingBox(latitude, longitude, bound)

	locationsFound := make([]*GeoData, 0)
	geoStoreKeysLen := len(geoStore[provider])
	for nIdx := range neighbours {
		neighboursUpperLimit := (neighbours[nIdx] + 1) << uint((MAX_STEPS-hashSteps)*2)
		neighbours[nIdx] = neighbours[nIdx] << uint((MAX_STEPS-hashSteps)*2)
		Debugf("Normalized Neighbours Hash: %v to %v", neighbours[nIdx], neighboursUpperLimit)
		searchIdx := sort.Search(geoStoreKeysLen, func(i int) bool { return geoStore[provider][i].GeoHash >= neighbours[nIdx] })
		if searchIdx < geoStoreKeysLen { // Not found would turn index=N
			Debugf("found location?")
			// found location
			for i := searchIdx; i < geoStoreKeysLen; i++ {
				if geoStore[provider][i].GeoHash < neighboursUpperLimit {
					data := geoStore[provider][i]
					Debugf("filtering by lat/long: %v %v", data.Latitude, data.Longitude)
					Debugf("filtering by bounding box: %v %v %v %v", box[0], box[1], box[2], box[3])
					// filter by strict bounding box
					if ((data.Latitude >= box[0] && data.Latitude <= box[1]) || (data.Latitude <= box[0] && data.Latitude >= box[1])) &&
						((data.Longitude >= box[2] && data.Longitude <= box[3]) || (data.Longitude <= box[2] && data.Longitude >= box[3])) {
						locationsFound = append(locationsFound, data)
						Debugf("Search found location in GeoStore: %v", geoStore[provider][searchIdx])
					}
				} else {
					break
				}
			}
		}
	}
	return locationsFound
}

// Sort the list of geo so binary search can be used. Normally triggered by the first search.
func initSearch() {
	for provider := range geoStore {
		sort.Sort(geoStore[provider])
	}
	sort.Sort(geoIdStore)
	searchReady = true // might cause race cond, but assume add doesn't happen often.
}

func getNeighbours(hashBits uint64, steps uint8) []uint64 {
	var neighbours C.GeoHashNeighbors
	var hash C.GeoHashBits
	hash.bits = C.uint64_t(hashBits)
	hash.step = C.uint8_t(steps)
	C.geohashNeighbors(&hash, &neighbours)

	neighbourArr := sortutil.Uint64Slice{
		uint64(hashBits),
		uint64(neighbours.north.bits),
		uint64(neighbours.east.bits),
		uint64(neighbours.west.bits),
		uint64(neighbours.south.bits),
		uint64(neighbours.north_east.bits),
		uint64(neighbours.south_east.bits),
		uint64(neighbours.north_west.bits),
		uint64(neighbours.south_west.bits),
	}
	//	sort.Sort(neighbourArr)
	if steps <= 6 {
		sortutil.Dedupe(neighbourArr) // Can have duplicates if search range is large (>~5000km)
	}
	return neighbourArr
}

// Encode a geo hash to MAX(26) steps
func geohashEncodeMax(latitude, longitude float64) uint64 {
	var hash C.GeoHashBits
	C.geohashEncodeWGS84(C.double(latitude), C.double(longitude), MAX_STEPS, &hash)
	return uint64(hash.bits)
}

// The approximate conversions are (doesn't fully correct for the Earth's polar flattening):
// Latitude: 1 deg = 110.574 km
// Longitude: 1 deg = 111.320*cos(latitude) km
// See: http://stackoverflow.com/questions/1253499/simple-calculations-for-working-with-lat-lon-km-distance
// Returns: min/max latitude, min/max longitude.
func boundingBox(latitude, longitude, boundKm float64) []float64 {
	latDiff := boundKm / 110.574
	longDiff := boundKm / (111.320 * math.Cos(latitude))
	minLatitude := latitude - latDiff
	maxLatitude := latitude + latDiff
	minLongitude := longitude - longDiff
	maxLongitude := longitude + longDiff

	Debugf("min lat: %v", minLatitude)
	Debugf("max lat: %v", maxLatitude)
	Debugf("min long: %v", minLongitude)
	Debugf("max long: %v", maxLongitude)

	return []float64{
		minLatitude,
		maxLatitude,
		minLongitude,
		maxLongitude,
	}
}
<|endoftext|>


<commit_before>package onedrive

import (
	"testing"
	"time"
)

func TestCalculateThrottle(t *testing.T) {

	time1, _ := time.Parse("02 Jan 06 15:04 MST", "01 Jan 15 15:04 GMT")
	time1e, _ := time.Parse("02 Jan 06 15:04 MST", "01 Jan 15 16:04 GMT")
	time2, _ := time.Parse("02 Jan 06 15:04 MST", "01 Jan 15 15:04 GMT")
	time2e, _ := time.Parse("02 Jan 06 15:04 MST", "01 Jan 15 15:05 GMT")

	tt := []struct {
		currentTime  time.Time
		expectedTime time.Time
		retryAfter   string
	}{
		{time1, time1e, "3600"},
		{time2, time2e, "60"},
	}

	for i, tst := range tt {
		tm, err := calculateThrottle(tst.currentTime, tst.retryAfter)
		if err != nil {
			t.Fatalf("%[%d] Couldn't calculate retry after: %s", i, err.Error())
		}
		if got, want := tm, tst.expectedTime; !got.Equal(want) {
			t.Fatalf("%[%d] Got %s Expected %s", i, got, want)
		}
	}
}

func TestNewRequest(t *testing.T) {
	setup()
	defer teardown()

	tt := []struct {
		method         string
		uri            string
		requestHeaders map[string]string
		debug          bool
		body           interface{}
	}{
		{"GET", "/foo", map[string]string{"Content-Type": "text/plain"}, false, nil},
		{"POST", "/foo/two", nil, true, ""},
		{"DELETE", "/foo/two", nil, true, testStruct1{"a", "b", "c"}},
	}

	for i, tst := range tt {
		oneDrive.Debug = tst.debug

		req, err := oneDrive.newRequest(tst.method, tst.uri, tst.requestHeaders, tst.body)
		if err != nil {
			t.Fatal(err)
		}

		if tst.debug {
			if got, want := req.Header.Get("Accept"), "application/json;format=pretty"; got != want {
				t.Fatalf("[%d] Got %q Expected %q", i, got, want)
			}
		} else {
			if got, want := req.Header.Get("Accept"), "application/json"; got != want {
				t.Fatalf("[%d] Got %q Expected %q", i, got, want)
			}
		}

		if err != nil {
			t.Fatalf("[%d] %q", i, err.Error())
		}
		if got, want := req.Method, tst.method; got != want {
			t.Fatalf("[%d] Got %q Expected %q", i, got, want)
		}
		if got, want := req.URL.String(), oneDrive.BaseURL+tst.uri; got != want {
			t.Fatalf("[%d] Got %q Expected %q", i, got, want)
		}

		for k, v := range tst.requestHeaders {
			if got, want := req.Header.Get(k), v; got != want {
				t.Fatalf("[%d] Got %q Expected %q", i, got, want)
			}
		}
	}
}
<commit_msg>Remove accidental % in tests<commit_after>package onedrive

import (
	"testing"
	"time"
)

func TestCalculateThrottle(t *testing.T) {

	time1, _ := time.Parse("02 Jan 06 15:04 MST", "01 Jan 15 15:04 GMT")
	time1e, _ := time.Parse("02 Jan 06 15:04 MST", "01 Jan 15 16:04 GMT")
	time2, _ := time.Parse("02 Jan 06 15:04 MST", "01 Jan 15 15:04 GMT")
	time2e, _ := time.Parse("02 Jan 06 15:04 MST", "01 Jan 15 15:05 GMT")

	tt := []struct {
		currentTime  time.Time
		expectedTime time.Time
		retryAfter   string
	}{
		{time1, time1e, "3600"},
		{time2, time2e, "60"},
	}

	for i, tst := range tt {
		tm, err := calculateThrottle(tst.currentTime, tst.retryAfter)
		if err != nil {
			t.Fatalf("[%d] Couldn't calculate retry after: %s", i, err.Error())
		}
		if got, want := tm, tst.expectedTime; !got.Equal(want) {
			t.Fatalf("[%d] Got %s Expected %s", i, got, want)
		}
	}
}

func TestNewRequest(t *testing.T) {
	setup()
	defer teardown()

	tt := []struct {
		method         string
		uri            string
		requestHeaders map[string]string
		debug          bool
		body           interface{}
	}{
		{"GET", "/foo", map[string]string{"Content-Type": "text/plain"}, false, nil},
		{"POST", "/foo/two", nil, true, ""},
		{"DELETE", "/foo/two", nil, true, testStruct1{"a", "b", "c"}},
	}

	for i, tst := range tt {
		oneDrive.Debug = tst.debug

		req, err := oneDrive.newRequest(tst.method, tst.uri, tst.requestHeaders, tst.body)
		if err != nil {
			t.Fatal(err)
		}

		if tst.debug {
			if got, want := req.Header.Get("Accept"), "application/json;format=pretty"; got != want {
				t.Fatalf("[%d] Got %q Expected %q", i, got, want)
			}
		} else {
			if got, want := req.Header.Get("Accept"), "application/json"; got != want {
				t.Fatalf("[%d] Got %q Expected %q", i, got, want)
			}
		}

		if err != nil {
			t.Fatalf("[%d] %q", i, err.Error())
		}
		if got, want := req.Method, tst.method; got != want {
			t.Fatalf("[%d] Got %q Expected %q", i, got, want)
		}
		if got, want := req.URL.String(), oneDrive.BaseURL+tst.uri; got != want {
			t.Fatalf("[%d] Got %q Expected %q", i, got, want)
		}

		for k, v := range tst.requestHeaders {
			if got, want := req.Header.Get(k), v; got != want {
				t.Fatalf("[%d] Got %q Expected %q", i, got, want)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"

	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/appc/spec/schema/types"
	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/spf13/cobra"
	"github.com/coreos/rkt/store"
)

var (
	cmdImageRm = &cobra.Command{
		Use:   "rm IMAGEID...",
		Short: "Remove image(s) with the given ID(s) from the local store",
		Run:   runWrapper(runRmImage),
	}
)

func init() {
	cmdImage.AddCommand(cmdImageRm)
}

func rmImages(s *store.Store, images []string) error {
	done := 0
	errors := 0
	staleErrors := 0
	for _, pkey := range images {
		errors++
		h, err := types.NewHash(pkey)
		if err != nil {
			stderr("rkt: wrong image ID %q: %v", pkey, err)
			continue
		}
		key, err := s.ResolveKey(h.String())
		if err != nil {
			stderr("rkt: image ID %q not valid: %v", pkey, err)
			continue
		}
		if key == "" {
			stderr("rkt: image ID %q doesn't exist", pkey)
			continue
		}

		if err = s.RemoveACI(key); err != nil {
			if serr, ok := err.(*store.StoreRemovalError); ok {
				staleErrors++
				stderr("rkt: some files cannot be removed for image ID %q: %v", pkey, serr)
			} else {
				stderr("rkt: error removing aci for image ID %q: %v", pkey, err)
				continue
			}
		}
		stdout("rkt: successfully removed aci for image ID: %q", pkey)
		errors--
		done++
	}

	if done > 0 {
		stderr("rkt: %d image(s) successfully removed", done)
	}

	// If anything didn't complete, return exit status of 1
	if (errors + staleErrors) > 0 {
		if staleErrors > 0 {
			stderr("rkt: %d image(s) removed but left some stale files", staleErrors)
		}
		if errors > 0 {
			stderr("rkt: %d image(s) cannot be removed", errors)
		}
		return fmt.Errorf("error(s) found while removing images")
	}

	return nil
}

func runRmImage(cmd *cobra.Command, args []string) (exit int) {
	if len(args) < 1 {
		stderr("rkt: Must provide at least one image ID")
		return 1
	}

	s, err := store.NewStore(globalFlags.Dir)
	if err != nil {
		stderr("rkt: cannot open store: %v", err)
		return 1
	}

	if err := rmImages(s, args); err != nil {
		stderr("rKt: %v", err)
		return 1
	}

	return 0
}
<commit_msg>rkt: rKt -> rkt<commit_after>// Copyright 2015 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"

	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/appc/spec/schema/types"
	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/spf13/cobra"
	"github.com/coreos/rkt/store"
)

var (
	cmdImageRm = &cobra.Command{
		Use:   "rm IMAGEID...",
		Short: "Remove image(s) with the given ID(s) from the local store",
		Run:   runWrapper(runRmImage),
	}
)

func init() {
	cmdImage.AddCommand(cmdImageRm)
}

func rmImages(s *store.Store, images []string) error {
	done := 0
	errors := 0
	staleErrors := 0
	for _, pkey := range images {
		errors++
		h, err := types.NewHash(pkey)
		if err != nil {
			stderr("rkt: wrong image ID %q: %v", pkey, err)
			continue
		}
		key, err := s.ResolveKey(h.String())
		if err != nil {
			stderr("rkt: image ID %q not valid: %v", pkey, err)
			continue
		}
		if key == "" {
			stderr("rkt: image ID %q doesn't exist", pkey)
			continue
		}

		if err = s.RemoveACI(key); err != nil {
			if serr, ok := err.(*store.StoreRemovalError); ok {
				staleErrors++
				stderr("rkt: some files cannot be removed for image ID %q: %v", pkey, serr)
			} else {
				stderr("rkt: error removing aci for image ID %q: %v", pkey, err)
				continue
			}
		}
		stdout("rkt: successfully removed aci for image ID: %q", pkey)
		errors--
		done++
	}

	if done > 0 {
		stderr("rkt: %d image(s) successfully removed", done)
	}

	// If anything didn't complete, return exit status of 1
	if (errors + staleErrors) > 0 {
		if staleErrors > 0 {
			stderr("rkt: %d image(s) removed but left some stale files", staleErrors)
		}
		if errors > 0 {
			stderr("rkt: %d image(s) cannot be removed", errors)
		}
		return fmt.Errorf("error(s) found while removing images")
	}

	return nil
}

func runRmImage(cmd *cobra.Command, args []string) (exit int) {
	if len(args) < 1 {
		stderr("rkt: Must provide at least one image ID")
		return 1
	}

	s, err := store.NewStore(globalFlags.Dir)
	if err != nil {
		stderr("rkt: cannot open store: %v", err)
		return 1
	}

	if err := rmImages(s, args); err != nil {
		stderr("rkt: %v", err)
		return 1
	}

	return 0
}
<|endoftext|>


<commit_before>// syncwatcher.go
package main

import (
  "code.google.com/p/go.exp/fsnotify"
  "os"
  "bufio"
  "io/ioutil"
  "net/http"
  "encoding/json"
  "log"
  "flag"
  "net/url"
  "strings"
)

type Configuration struct {
  Version      int
  Repositories []RepositoryConfiguration
}

type RepositoryConfiguration struct {
  ID              string
  Directory       string
  ReadOnly        bool
  RescanIntervalS int
}

// HTTP Parameters
var (
  target    string
  authUser  string
  authPass  string
  csrfToken string
  csrfFile  string
  apiKey    string
)

// Main
var (
	stop = make(chan int)
)

func main() {
  flag.StringVar(&target, "target", "localhost:8080", "Target")
  flag.StringVar(&authUser, "user", "", "Username")
  flag.StringVar(&authPass, "pass", "", "Password")
  flag.StringVar(&csrfFile, "csrf", "", "CSRF token file")
  flag.StringVar(&apiKey, "api", "", "API key")
  flag.Parse()

  if len(csrfFile) > 0 {
    fd, err := os.Open(csrfFile)
    if err != nil {
      log.Fatal(err)
    }
    s := bufio.NewScanner(fd)
    for s.Scan() {
      csrfToken = s.Text()
    }
    fd.Close()
  }

  repos := getRepos()
  for i := range repos {
    repo := repos[i]
    go watchRepo(repo.ID, repo.Directory)
  }

  code := <-stop
  println("Exiting")
  os.Exit(code)

}

func getRepos() []RepositoryConfiguration {
  r, err := http.NewRequest("GET", "http://"+target+"/rest/config", nil)
  if err != nil {
    log.Fatal(err)
  }
  if len(csrfToken) > 0 {
    r.Header.Set("X-CSRF-Token", csrfToken)
  }
  if len(authUser) > 0 {
    r.SetBasicAuth(authUser, authPass)
  }
  if len(apiKey) > 0 {
    r.Header.Set("X-API-Key", apiKey)
  }
  res, err := http.DefaultClient.Do(r)
  if err != nil {
    log.Fatal(err)
  }
  defer res.Body.Close()
  if res.StatusCode != 200 {
    log.Fatalf("Status %d != 200 for GET", res.StatusCode)
  }
  bs, err := ioutil.ReadAll(res.Body)
  if err != nil {
    log.Fatal(err)
  }
  var cfg Configuration
  err = json.Unmarshal(bs, &cfg)
  if err != nil {
    log.Fatal(err)
  }
  return cfg.Repositories
}

func watchRepo(repo string, directory string) {
  sw, err := NewSyncWatcher()
  if sw == nil || err != nil {
    log.Fatal(err)
  }
  defer sw.Close()
  err = sw.Watch(directory)
  if err != nil {
    log.Fatal(err)
  }
  log.Println("Watching "+repo+": "+directory)
  for {
    ev, ok := waitForEvent(sw)
    if ok && ev != nil {
      sub := strings.TrimPrefix(ev.Name, directory)
      sub = strings.TrimPrefix(sub, string(os.PathSeparator))
      informChange(repo, sub)
    }
  }
}

func waitForEvent(sw *SyncWatcher) (ev *fsnotify.FileEvent, ok bool) {
  select {
  case ev, ok = <-sw.Event:
    if !ok {
      log.Fatal("Event: channel closed")
    }
  case err, eok := <-sw.Error:
    log.Fatal(err, eok)
  }
  return
}

func informChange(repo string, sub string) {
  data := url.Values {}
  data.Set("repo", repo)
  data.Set("sub", sub)
  r, err := http.NewRequest("POST", "http://"+target+"/rest/scan?"+data.Encode(), nil)
  if err != nil {
    log.Fatal(err)
  }
  if len(csrfToken) > 0 {
    r.Header.Set("X-CSRF-Token", csrfToken)
  }
  if len(authUser) > 0 {
    r.SetBasicAuth(authUser, authPass)
  }
  if len(apiKey) > 0 {
    r.Header.Set("X-API-Key", apiKey)
  }
  res, err := http.DefaultClient.Do(r)
  if err != nil {
    log.Fatal(err)
  }
  defer res.Body.Close()
  if res.StatusCode != 200 {
    log.Fatalf("Status %d != 200 for POST", res.StatusCode)
  } else {
    log.Println("Syncthing will index change in "+repo+": "+sub)
  }
}
<commit_msg>Log empty fsnotify event<commit_after>// syncwatcher.go
package main

import (
  "code.google.com/p/go.exp/fsnotify"
  "os"
  "bufio"
  "io/ioutil"
  "net/http"
  "encoding/json"
  "log"
  "flag"
  "net/url"
  "strings"
)

type Configuration struct {
  Version      int
  Repositories []RepositoryConfiguration
}

type RepositoryConfiguration struct {
  ID              string
  Directory       string
  ReadOnly        bool
  RescanIntervalS int
}

// HTTP Parameters
var (
  target    string
  authUser  string
  authPass  string
  csrfToken string
  csrfFile  string
  apiKey    string
)

// Main
var (
	stop = make(chan int)
)

func main() {
  flag.StringVar(&target, "target", "localhost:8080", "Target")
  flag.StringVar(&authUser, "user", "", "Username")
  flag.StringVar(&authPass, "pass", "", "Password")
  flag.StringVar(&csrfFile, "csrf", "", "CSRF token file")
  flag.StringVar(&apiKey, "api", "", "API key")
  flag.Parse()

  if len(csrfFile) > 0 {
    fd, err := os.Open(csrfFile)
    if err != nil {
      log.Fatal(err)
    }
    s := bufio.NewScanner(fd)
    for s.Scan() {
      csrfToken = s.Text()
    }
    fd.Close()
  }

  repos := getRepos()
  for i := range repos {
    repo := repos[i]
    go watchRepo(repo.ID, repo.Directory)
  }

  code := <-stop
  println("Exiting")
  os.Exit(code)

}

func getRepos() []RepositoryConfiguration {
  r, err := http.NewRequest("GET", "http://"+target+"/rest/config", nil)
  if err != nil {
    log.Fatal(err)
  }
  if len(csrfToken) > 0 {
    r.Header.Set("X-CSRF-Token", csrfToken)
  }
  if len(authUser) > 0 {
    r.SetBasicAuth(authUser, authPass)
  }
  if len(apiKey) > 0 {
    r.Header.Set("X-API-Key", apiKey)
  }
  res, err := http.DefaultClient.Do(r)
  if err != nil {
    log.Fatal(err)
  }
  defer res.Body.Close()
  if res.StatusCode != 200 {
    log.Fatalf("Status %d != 200 for GET", res.StatusCode)
  }
  bs, err := ioutil.ReadAll(res.Body)
  if err != nil {
    log.Fatal(err)
  }
  var cfg Configuration
  err = json.Unmarshal(bs, &cfg)
  if err != nil {
    log.Fatal(err)
  }
  return cfg.Repositories
}

func watchRepo(repo string, directory string) {
  sw, err := NewSyncWatcher()
  if sw == nil || err != nil {
    log.Fatal(err)
  }
  defer sw.Close()
  err = sw.Watch(directory)
  if err != nil {
    log.Fatal(err)
  }
  log.Println("Watching "+repo+": "+directory)
  for {
    ev := waitForEvent(sw)
    if ev == nil {
      log.Fatal("fsnotify event is nil")
    }
    sub := strings.TrimPrefix(ev.Name, directory)
    sub = strings.TrimPrefix(sub, string(os.PathSeparator))
    informChange(repo, sub)
  }
}

func waitForEvent(sw *SyncWatcher) (ev *fsnotify.FileEvent) {
  var ok bool
  select {
  case ev, ok = <-sw.Event:
    if !ok {
      log.Fatal("Event: channel closed")
    }
  case err, eok := <-sw.Error:
    log.Fatal(err, eok)
  }
  return
}

func informChange(repo string, sub string) {
  data := url.Values {}
  data.Set("repo", repo)
  data.Set("sub", sub)
  r, err := http.NewRequest("POST", "http://"+target+"/rest/scan?"+data.Encode(), nil)
  if err != nil {
    log.Fatal(err)
  }
  if len(csrfToken) > 0 {
    r.Header.Set("X-CSRF-Token", csrfToken)
  }
  if len(authUser) > 0 {
    r.SetBasicAuth(authUser, authPass)
  }
  if len(apiKey) > 0 {
    r.Header.Set("X-API-Key", apiKey)
  }
  res, err := http.DefaultClient.Do(r)
  if err != nil {
    log.Fatal(err)
  }
  defer res.Body.Close()
  if res.StatusCode != 200 {
    log.Fatalf("Status %d != 200 for POST", res.StatusCode)
  } else {
    log.Println("Syncthing will index change in "+repo+": "+sub)
  }
}
<|endoftext|>


<commit_before>// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//     http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package engine

import (
	"errors"
	"fmt"
	"io"
	"math"
	"time"

	"github.com/aws/amazon-ecs-init/ecs-init/backoff"
	"github.com/aws/amazon-ecs-init/ecs-init/cache"
	"github.com/aws/amazon-ecs-init/ecs-init/config"
	"github.com/aws/amazon-ecs-init/ecs-init/docker"
	"github.com/aws/amazon-ecs-init/ecs-init/exec"
	"github.com/aws/amazon-ecs-init/ecs-init/exec/iptables"
	"github.com/aws/amazon-ecs-init/ecs-init/exec/sysctl"
	"github.com/aws/amazon-ecs-init/ecs-init/gpu"

	log "github.com/cihub/seelog"
)

const (
	terminalSuccessAgentExitCode  = 0
	containerFailureAgentExitCode = 2
	TerminalFailureAgentExitCode  = 5
	DefaultInitErrorExitCode      = -1
	upgradeAgentExitCode          = 42
	serviceStartMinRetryTime      = time.Millisecond * 500
	serviceStartMaxRetryTime      = time.Second * 15
	serviceStartRetryJitter       = 0.10
	serviceStartRetryMultiplier   = 2.0
	serviceStartMaxRetries        = math.MaxInt64 // essentially retry forever
	failedContainerLogWindowSize  = "200"         // as string for log config
)

// Injection point for testing purposes
var getDockerClient = func() (dockerClient, error) {
	return docker.Client()
}

func dockerError(err error) error {
	return engineError("could not create docker client", err)
}

// Engine contains methods invoked when ecs-init is run
type Engine struct {
	downloader               downloader
	loopbackRouting          loopbackRouting
	credentialsProxyRoute    credentialsProxyRoute
	ipv6RouterAdvertisements ipv6RouterAdvertisements
	nvidiaGPUManager         gpu.GPUManager
}

type TerminalError struct {
	err      string
	exitCode int
}

func (e *TerminalError) Error() string {
	return fmt.Sprintf("%s: %d", e.err, e.exitCode)
}

// New creates an instance of Engine
func New() (*Engine, error) {
	downloader, err := cache.NewDownloader()
	if err != nil {
		return nil, err
	}

	cmdExec := exec.NewExec()
	loopbackRouting, err := sysctl.NewIpv4RouteLocalNet(cmdExec)
	if err != nil {
		return nil, err
	}
	ipv6RouterAdvertisements, err := sysctl.NewIpv6RouterAdvertisements(cmdExec)
	if err != nil {
		return nil, err
	}
	credentialsProxyRoute, err := iptables.NewNetfilterRoute(cmdExec)
	if err != nil {
		return nil, err
	}
	return &Engine{
		downloader:               downloader,
		loopbackRouting:          loopbackRouting,
		credentialsProxyRoute:    credentialsProxyRoute,
		ipv6RouterAdvertisements: ipv6RouterAdvertisements,
		nvidiaGPUManager:         gpu.NewNvidiaGPUManager(),
	}, nil
}

// PreStart prepares the ECS Agent for starting. It also configures the instance
// to handle credentials requests from containers by rerouting these requests to
// to the ECS Agent's credentials endpoint
func (e *Engine) PreStart() error {
	// setup gpu if necessary
	err := e.PreStartGPU()
	if err != nil {
		return err
	}
	// Enable use of loopback addresses for local routing purposes
	err = e.loopbackRouting.Enable()
	if err != nil {
		return engineError("could not enable loopback routing", err)
	}
	// Disable ipv6 router advertisements
	err = e.ipv6RouterAdvertisements.Disable()
	if err != nil {
		return engineError("could not disable ipv6 router advertisements", err)
	}
	// Add the rerouting netfilter rule for credentials endpoint
	err = e.credentialsProxyRoute.Create()
	if err != nil {
		return engineError("could not create route to the credentials proxy", err)
	}

	docker, err := getDockerClient()
	if err != nil {
		return dockerError(err)
	}
	imageLoaded, err := docker.IsAgentImageLoaded()
	if err != nil {
		return engineError("could not check Docker for Agent image presence", err)
	}

	switch e.downloader.AgentCacheStatus() {
	// Uncached, go get the Agent.
	case cache.StatusUncached:
		return e.downloadAndLoadCache(docker)

	// The Agent is cached, and mandates a reload regardless of the
	// already loaded image.
	case cache.StatusReloadNeeded:
		return e.load(docker, e.downloader.LoadCachedAgent)

	// Agent is cached, respect the already loaded Agent.
	case cache.StatusCached:
		if imageLoaded {
			return nil
		}
		return e.load(docker, e.downloader.LoadCachedAgent)

	// There shouldn't be unhandled cache states.
	default:
		return errors.New("could not handle cache state")
	}
}

// PreStartGPU sets up the nvidia gpu manager if it's enabled.
func (e *Engine) PreStartGPU() error {
	docker, err := getDockerClient()
	if err != nil {
		return dockerError(err)
	}
	envVariables := docker.LoadEnvVars()
	if val, ok := envVariables[config.GPUSupportEnvVar]; ok {
		if val == "true" {
			err := e.nvidiaGPUManager.Setup()
			if err != nil {
				log.Errorf("Nvidia GPU Manager: %v", err)
				return engineError("Nvidia GPU Manager", err)
			}
		}
	}
	return nil
}

// ReloadCache reloads the cached image of the ECS Agent into Docker
func (e *Engine) ReloadCache() error {
	docker, err := getDockerClient()
	if err != nil {
		return dockerError(err)
	}
	cached := e.downloader.IsAgentCached()
	if !cached {
		return e.downloadAndLoadCache(docker)
	}
	return e.load(docker, e.downloader.LoadCachedAgent)
}

func (e *Engine) downloadAndLoadCache(docker dockerClient) error {
	err := e.downloadAgent()
	if err != nil {
		return err
	}

	log.Info("Loading Amazon Elastic Container Service Agent into Docker")
	return e.load(docker, e.downloader.LoadCachedAgent)
}

func (e *Engine) downloadAgent() error {
	log.Info("Downloading Amazon Elastic Container Service Agent")
	err := e.downloader.DownloadAgent()
	if err != nil {
		return engineError("could not download Amazon Elastic Container Service Agent", err)
	}
	return nil
}

func (e *Engine) load(docker dockerClient, agentLoader func() (io.ReadCloser, error)) error {
	image, err := agentLoader()
	if err != nil {
		return engineError("could not load Amazon Elastic Container Service Agent from cache", err)
	}
	defer image.Close()
	err = docker.LoadImage(image)
	if err != nil {
		return engineError("could not load Amazon Elastic Container Service Agent into Docker", err)
	}
	return e.downloader.RecordCachedAgent()
}

// StartSupervised starts the ECS Agent and ensures it stays running, except for terminal errors (indicated by an agent exit code of 5)
func (e *Engine) StartSupervised() error {
	docker, err := getDockerClient()
	if err != nil {
		return dockerError(err)
	}
	agentExitCode := -1
	retryBackoff := backoff.NewBackoff(serviceStartMinRetryTime, serviceStartMaxRetryTime,
		serviceStartRetryJitter, serviceStartRetryMultiplier, serviceStartMaxRetries)
	for {
		err := docker.RemoveExistingAgentContainer()
		if err != nil {
			return engineError("could not remove existing Agent container", err)
		}

		log.Info("Starting Amazon Elastic Container Service Agent")
		agentExitCode, err = docker.StartAgent()
		if err != nil {
			return engineError("could not start Agent", err)
		}
		log.Infof("Agent exited with code %d", agentExitCode)

		switch agentExitCode {
		case upgradeAgentExitCode:
			err = e.upgradeAgent(docker)
			if err != nil {
				log.Error("could not upgrade agent", err)
			} else {
				// continuing here because a successful upgrade doesn't need to backoff retries
				continue
			}
		case containerFailureAgentExitCode:
			// capture the tail of the failed agent container
			log.Infof("Captured the last %s lines of the agent container logs====>\n", failedContainerLogWindowSize)
			log.Info(docker.GetContainerLogTail(failedContainerLogWindowSize))
			log.Infof("<====end %s lines of the failed agent container logs\n", failedContainerLogWindowSize)
		case TerminalFailureAgentExitCode:
			return &TerminalError{
				err:      "agent exited with terminal exit code",
				exitCode: TerminalFailureAgentExitCode,
			}
		case terminalSuccessAgentExitCode:
			return nil
		}
		d := retryBackoff.Duration()
		log.Warnf("ECS Agent failed to start, retrying in %s", d)
		time.Sleep(d)
	}
}

func (e *Engine) upgradeAgent(docker dockerClient) error {
	log.Info("Loading new desired Amazon Elastic Container Service Agent into Docker")
	return e.load(docker, e.downloader.LoadDesiredAgent)
}

// PreStop sends commands to Docker to stop the ECS Agent
func (e *Engine) PreStop() error {
	docker, err := getDockerClient()
	if err != nil {
		return dockerError(err)
	}
	log.Info("Stopping Amazon Elastic Container Service Agent")
	err = docker.StopAgent()
	if err != nil {
		return engineError("could not stop Amazon Elastic Container Service Agent", err)
	}
	return nil
}

// PostStop cleans up the credentials endpoint setup by disabling loopback
// routing and removing the rerouting rule from the netfilter table
func (e *Engine) PostStop() error {
	log.Info("Cleaning up the credentials endpoint setup for Amazon Elastic Container Service Agent")
	err := e.loopbackRouting.RestoreDefault()

	// Ignore error from Remove() as the netfilter might never have been
	// added in the first place
	e.credentialsProxyRoute.Remove()
	return err
}

type _engineError struct {
	err     error
	message string
}

func (e _engineError) Error() string {
	return fmt.Sprintf("%s: %s", e.message, e.err.Error())
}

func engineError(message string, err error) _engineError {
	return _engineError{
		message: message,
		err:     err,
	}
}
<commit_msg>Log what pre-start is doing<commit_after>// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//     http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package engine

import (
	"errors"
	"fmt"
	"io"
	"math"
	"time"

	"github.com/aws/amazon-ecs-init/ecs-init/backoff"
	"github.com/aws/amazon-ecs-init/ecs-init/cache"
	"github.com/aws/amazon-ecs-init/ecs-init/config"
	"github.com/aws/amazon-ecs-init/ecs-init/docker"
	"github.com/aws/amazon-ecs-init/ecs-init/exec"
	"github.com/aws/amazon-ecs-init/ecs-init/exec/iptables"
	"github.com/aws/amazon-ecs-init/ecs-init/exec/sysctl"
	"github.com/aws/amazon-ecs-init/ecs-init/gpu"

	log "github.com/cihub/seelog"
)

const (
	terminalSuccessAgentExitCode  = 0
	containerFailureAgentExitCode = 2
	TerminalFailureAgentExitCode  = 5
	DefaultInitErrorExitCode      = -1
	upgradeAgentExitCode          = 42
	serviceStartMinRetryTime      = time.Millisecond * 500
	serviceStartMaxRetryTime      = time.Second * 15
	serviceStartRetryJitter       = 0.10
	serviceStartRetryMultiplier   = 2.0
	serviceStartMaxRetries        = math.MaxInt64 // essentially retry forever
	failedContainerLogWindowSize  = "200"         // as string for log config
)

// Injection point for testing purposes
var getDockerClient = func() (dockerClient, error) {
	return docker.Client()
}

func dockerError(err error) error {
	return engineError("could not create docker client", err)
}

// Engine contains methods invoked when ecs-init is run
type Engine struct {
	downloader               downloader
	loopbackRouting          loopbackRouting
	credentialsProxyRoute    credentialsProxyRoute
	ipv6RouterAdvertisements ipv6RouterAdvertisements
	nvidiaGPUManager         gpu.GPUManager
}

type TerminalError struct {
	err      string
	exitCode int
}

func (e *TerminalError) Error() string {
	return fmt.Sprintf("%s: %d", e.err, e.exitCode)
}

// New creates an instance of Engine
func New() (*Engine, error) {
	downloader, err := cache.NewDownloader()
	if err != nil {
		return nil, err
	}

	cmdExec := exec.NewExec()
	loopbackRouting, err := sysctl.NewIpv4RouteLocalNet(cmdExec)
	if err != nil {
		return nil, err
	}
	ipv6RouterAdvertisements, err := sysctl.NewIpv6RouterAdvertisements(cmdExec)
	if err != nil {
		return nil, err
	}
	credentialsProxyRoute, err := iptables.NewNetfilterRoute(cmdExec)
	if err != nil {
		return nil, err
	}
	return &Engine{
		downloader:               downloader,
		loopbackRouting:          loopbackRouting,
		credentialsProxyRoute:    credentialsProxyRoute,
		ipv6RouterAdvertisements: ipv6RouterAdvertisements,
		nvidiaGPUManager:         gpu.NewNvidiaGPUManager(),
	}, nil
}

// PreStart prepares the ECS Agent for starting. It also configures the instance
// to handle credentials requests from containers by rerouting these requests to
// to the ECS Agent's credentials endpoint
func (e *Engine) PreStart() error {
	// setup gpu if necessary
	err := e.PreStartGPU()
	if err != nil {
		return err
	}
	// Enable use of loopback addresses for local routing purposes
	log.Info("pre-start: enabling loopback routing")
	err = e.loopbackRouting.Enable()
	if err != nil {
		return engineError("could not enable loopback routing", err)
	}
	// Disable ipv6 router advertisements
	log.Info("pre-start: disabling ipv6 router advertisements")
	err = e.ipv6RouterAdvertisements.Disable()
	if err != nil {
		return engineError("could not disable ipv6 router advertisements", err)
	}
	// Add the rerouting netfilter rule for credentials endpoint
	log.Info("pre-start: creating credentials proxy route")
	err = e.credentialsProxyRoute.Create()
	if err != nil {
		return engineError("could not create route to the credentials proxy", err)
	}

	docker, err := getDockerClient()
	if err != nil {
		return dockerError(err)
	}
	log.Info("pre-start: checking ecs agent container image loaded presence")
	imageLoaded, err := docker.IsAgentImageLoaded()
	if err != nil {
		return engineError("could not check Docker for Agent image presence", err)
	}
	log.Infof("pre-start: ecs agent container image loaded presence: %s", imageLoaded)

	switch e.downloader.AgentCacheStatus() {
	// Uncached, go get the Agent.
	case cache.StatusUncached:
		log.Info("pre-start: downloading agent")
		return e.downloadAndLoadCache(docker)

	// The Agent is cached, and mandates a reload regardless of the
	// already loaded image.
	case cache.StatusReloadNeeded:
		log.Info("pre-start: reloading agent")
		return e.load(docker, e.downloader.LoadCachedAgent)

	// Agent is cached, respect the already loaded Agent.
	case cache.StatusCached:
		if imageLoaded {
			return nil
		}
		log.Info("pre-start: loading cached agent")
		return e.load(docker, e.downloader.LoadCachedAgent)

	// There shouldn't be unhandled cache states.
	default:
		return errors.New("could not handle cache state")
	}
}

// PreStartGPU sets up the nvidia gpu manager if it's enabled.
func (e *Engine) PreStartGPU() error {
	docker, err := getDockerClient()
	if err != nil {
		return dockerError(err)
	}
	envVariables := docker.LoadEnvVars()
	if val, ok := envVariables[config.GPUSupportEnvVar]; ok {
		if val == "true" {
			log.Info("pre-start: setting up GPUs")
			defer log.Info("pre-start: done setting up GPUs")
			err := e.nvidiaGPUManager.Setup()
			if err != nil {
				log.Errorf("Nvidia GPU Manager: %v", err)
				return engineError("Nvidia GPU Manager", err)
			}
		}
	}
	return nil
}

// ReloadCache reloads the cached image of the ECS Agent into Docker
func (e *Engine) ReloadCache() error {
	docker, err := getDockerClient()
	if err != nil {
		return dockerError(err)
	}
	cached := e.downloader.IsAgentCached()
	if !cached {
		return e.downloadAndLoadCache(docker)
	}
	return e.load(docker, e.downloader.LoadCachedAgent)
}

func (e *Engine) downloadAndLoadCache(docker dockerClient) error {
	err := e.downloadAgent()
	if err != nil {
		return err
	}

	log.Info("Loading Amazon Elastic Container Service Agent into Docker")
	return e.load(docker, e.downloader.LoadCachedAgent)
}

func (e *Engine) downloadAgent() error {
	log.Info("Downloading Amazon Elastic Container Service Agent")
	err := e.downloader.DownloadAgent()
	if err != nil {
		return engineError("could not download Amazon Elastic Container Service Agent", err)
	}
	return nil
}

func (e *Engine) load(docker dockerClient, agentLoader func() (io.ReadCloser, error)) error {
	image, err := agentLoader()
	if err != nil {
		return engineError("could not load Amazon Elastic Container Service Agent from cache", err)
	}
	defer image.Close()
	err = docker.LoadImage(image)
	if err != nil {
		return engineError("could not load Amazon Elastic Container Service Agent into Docker", err)
	}
	return e.downloader.RecordCachedAgent()
}

// StartSupervised starts the ECS Agent and ensures it stays running, except for terminal errors (indicated by an agent exit code of 5)
func (e *Engine) StartSupervised() error {
	docker, err := getDockerClient()
	if err != nil {
		return dockerError(err)
	}
	agentExitCode := -1
	retryBackoff := backoff.NewBackoff(serviceStartMinRetryTime, serviceStartMaxRetryTime,
		serviceStartRetryJitter, serviceStartRetryMultiplier, serviceStartMaxRetries)
	for {
		err := docker.RemoveExistingAgentContainer()
		if err != nil {
			return engineError("could not remove existing Agent container", err)
		}

		log.Info("Starting Amazon Elastic Container Service Agent")
		agentExitCode, err = docker.StartAgent()
		if err != nil {
			return engineError("could not start Agent", err)
		}
		log.Infof("Agent exited with code %d", agentExitCode)

		switch agentExitCode {
		case upgradeAgentExitCode:
			err = e.upgradeAgent(docker)
			if err != nil {
				log.Error("could not upgrade agent", err)
			} else {
				// continuing here because a successful upgrade doesn't need to backoff retries
				continue
			}
		case containerFailureAgentExitCode:
			// capture the tail of the failed agent container
			log.Infof("Captured the last %s lines of the agent container logs====>\n", failedContainerLogWindowSize)
			log.Info(docker.GetContainerLogTail(failedContainerLogWindowSize))
			log.Infof("<====end %s lines of the failed agent container logs\n", failedContainerLogWindowSize)
		case TerminalFailureAgentExitCode:
			return &TerminalError{
				err:      "agent exited with terminal exit code",
				exitCode: TerminalFailureAgentExitCode,
			}
		case terminalSuccessAgentExitCode:
			return nil
		}
		d := retryBackoff.Duration()
		log.Warnf("ECS Agent failed to start, retrying in %s", d)
		time.Sleep(d)
	}
}

func (e *Engine) upgradeAgent(docker dockerClient) error {
	log.Info("Loading new desired Amazon Elastic Container Service Agent into Docker")
	return e.load(docker, e.downloader.LoadDesiredAgent)
}

// PreStop sends commands to Docker to stop the ECS Agent
func (e *Engine) PreStop() error {
	docker, err := getDockerClient()
	if err != nil {
		return dockerError(err)
	}
	log.Info("Stopping Amazon Elastic Container Service Agent")
	err = docker.StopAgent()
	if err != nil {
		return engineError("could not stop Amazon Elastic Container Service Agent", err)
	}
	return nil
}

// PostStop cleans up the credentials endpoint setup by disabling loopback
// routing and removing the rerouting rule from the netfilter table
func (e *Engine) PostStop() error {
	log.Info("Cleaning up the credentials endpoint setup for Amazon Elastic Container Service Agent")
	err := e.loopbackRouting.RestoreDefault()

	// Ignore error from Remove() as the netfilter might never have been
	// added in the first place
	e.credentialsProxyRoute.Remove()
	return err
}

type _engineError struct {
	err     error
	message string
}

func (e _engineError) Error() string {
	return fmt.Sprintf("%s: %s", e.message, e.err.Error())
}

func engineError(message string, err error) _engineError {
	return _engineError{
		message: message,
		err:     err,
	}
}
<|endoftext|>