<commit_before>package boltrepository

import (
	"encoding/json"
	"fmt"
	"github.com/boltdb/bolt"
)

type Repository struct {
	FilePath  string
	GetObject func([]byte) interface{}
}

func NewRepository(filePath string) *Repository {
	repository := &Repository{
		FilePath: filePath,
	}
	return repository
}

func (r *Repository) Deserialize(jsonBytes []byte, target interface{}) error {

	err := json.Unmarshal(jsonBytes, &target)

	return err
}

func (r *Repository) ForEach(bucketName string, action func(string, interface{})) error {

	db, err := bolt.Open(r.FilePath, 0644, &bolt.Options{ReadOnly: true})
	if err != nil {
		return err
	}
	defer db.Close()

	// retrieve the data
	err = db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte(bucketName))
		if bucket == nil {
			return fmt.Errorf("Bucket %q not found!", []byte(bucketName))
		}

		bucket.ForEach(func(k, v []byte) error {

			key := string(k)
			value := r.GetObject(v)

			action(key, value)

			return nil
		})
		return nil

	})

	return err
}

func (r *Repository) HasItem(bucketName string, keyName string) (hasItem bool, err error) {

	db, err := bolt.Open(r.FilePath, 0644, &bolt.Options{ReadOnly: true})
	if err != nil {
		return false, err
	}
	defer db.Close()

	// retrieve the data
	err = db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte(bucketName))
		if bucket == nil {
			return fmt.Errorf("Bucket %q not found!", []byte(bucketName))
		}

		val := bucket.Get([]byte(keyName))

		hasItem = val != nil

		return nil
	})

	return hasItem, err
}

func (r *Repository) Read(bucketName string, keyName string) (obj interface{}, err error) {

	db, err := bolt.Open(r.FilePath, 0644, &bolt.Options{ReadOnly: true})
	if err != nil {
		return nil, err
	}
	defer db.Close()

	// retrieve the data
	err = db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte(bucketName))
		if bucket == nil {
			return fmt.Errorf("Bucket %q not found!", []byte(bucketName))
		}

		val := bucket.Get([]byte(keyName))

		obj = r.GetObject(val)

		return nil
	})

	return obj, err
}

func (r *Repository) ReadInto(bucketName string, keyName string, target interface{}) error {

	db, err := bolt.Open(r.FilePath, 0644, &bolt.Options{ReadOnly: true})
	if err != nil {
		return err
	}
	defer db.Close()

	// retrieve the data
	err = db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte(bucketName))
		if bucket == nil {
			return fmt.Errorf("Bucket %q not found!", []byte(bucketName))
		}

		val := bucket.Get([]byte(keyName))

		return r.Deserialize(val, &target)
	})

	return err
}

func (r *Repository) Save(bucketName string, keyName string, value interface{}) error {

	db, err := bolt.Open(r.FilePath, 0644, nil)
	if err != nil {
		return err
	}
	defer db.Close()

	serialized, _ := r.Serialize(value)

	// store some data
	err = db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte(bucketName))
		if err != nil {
			return err
		}

		err = bucket.Put([]byte(keyName), serialized)
		if err != nil {
			return err
		}
		return nil
	})

	return err
}

func (r *Repository) Serialize(value interface{}) ([]byte, error) {

	jsonBytes, err := json.Marshal(value)

	return jsonBytes, err
}
<commit_msg>Initialize new file<commit_after>package boltrepository

import (
	"encoding/json"
	"fmt"
	"github.com/boltdb/bolt"
)

type Repository struct {
	FilePath  string
	GetObject func([]byte) interface{}
}

func NewRepository(filePath string) *Repository {
	repository := &Repository{
		FilePath: filePath,
	}

	db, _ := bolt.Open(repository.FilePath, 0644, nil)
	db.Close()

	return repository
}

func (r *Repository) Deserialize(jsonBytes []byte, target interface{}) error {

	err := json.Unmarshal(jsonBytes, &target)

	return err
}

func (r *Repository) ForEach(bucketName string, action func(string, interface{})) error {

	db, err := bolt.Open(r.FilePath, 0644, &bolt.Options{ReadOnly: true})
	if err != nil {
		return err
	}
	defer db.Close()

	// retrieve the data
	err = db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte(bucketName))
		if bucket == nil {
			return fmt.Errorf("Bucket %q not found!", []byte(bucketName))
		}

		bucket.ForEach(func(k, v []byte) error {

			key := string(k)
			value := r.GetObject(v)

			action(key, value)

			return nil
		})
		return nil

	})

	return err
}

func (r *Repository) HasItem(bucketName string, keyName string) (hasItem bool, err error) {

	db, err := bolt.Open(r.FilePath, 0644, &bolt.Options{ReadOnly: true})
	if err != nil {
		return false, err
	}
	defer db.Close()

	// retrieve the data
	err = db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte(bucketName))
		if bucket == nil {
			return fmt.Errorf("Bucket %q not found!", []byte(bucketName))
		}

		val := bucket.Get([]byte(keyName))

		hasItem = val != nil

		return nil
	})

	return hasItem, err
}

func (r *Repository) Read(bucketName string, keyName string) (obj interface{}, err error) {

	db, err := bolt.Open(r.FilePath, 0644, &bolt.Options{ReadOnly: true})
	if err != nil {
		return nil, err
	}
	defer db.Close()

	// retrieve the data
	err = db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte(bucketName))
		if bucket == nil {
			return fmt.Errorf("Bucket %q not found!", []byte(bucketName))
		}

		val := bucket.Get([]byte(keyName))

		obj = r.GetObject(val)

		return nil
	})

	return obj, err
}

func (r *Repository) ReadInto(bucketName string, keyName string, target interface{}) error {

	db, err := bolt.Open(r.FilePath, 0644, &bolt.Options{ReadOnly: true})
	if err != nil {
		return err
	}
	defer db.Close()

	// retrieve the data
	err = db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte(bucketName))
		if bucket == nil {
			return fmt.Errorf("Bucket %q not found!", []byte(bucketName))
		}

		val := bucket.Get([]byte(keyName))

		return r.Deserialize(val, &target)
	})

	return err
}

func (r *Repository) Save(bucketName string, keyName string, value interface{}) error {

	db, err := bolt.Open(r.FilePath, 0644, nil)
	if err != nil {
		return err
	}
	defer db.Close()

	serialized, _ := r.Serialize(value)

	// store some data
	err = db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte(bucketName))
		if err != nil {
			return err
		}

		err = bucket.Put([]byte(keyName), serialized)
		if err != nil {
			return err
		}
		return nil
	})

	return err
}

func (r *Repository) Serialize(value interface{}) ([]byte, error) {

	jsonBytes, err := json.Marshal(value)

	return jsonBytes, err
}
<|endoftext|>


<commit_before>package gojenkins

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

type Auth struct {
	Username string
	ApiToken string
}

type Jenkins struct {
	auth    *Auth
	baseUrl string
}

func NewJenkins(auth *Auth, baseUrl string) *Jenkins {
	return &Jenkins{
		auth:    auth,
		baseUrl: baseUrl,
	}
}

func (jenkins *Jenkins) buildUrl(path string, params url.Values) (requestUrl string) {
	if params == nil {
		requestUrl = jenkins.baseUrl + "job/" + path
		if !strings.Contains(path, "/api/json") {
			requestUrl = jenkins.baseUrl + path + "/api/json"
		}
	}
	if params != nil {
		queryString := params.Encode()
		if queryString != "" {
			requestUrl = requestUrl + "?" + queryString
		}
	}
	return
}

func (jenkins *Jenkins) sendRequest(req *http.Request) (*http.Response, error) {
	//req.SetBasicAuth(jenkins.auth.Username, jenkins.auth.ApiToken)
	return http.DefaultClient.Do(req)
}

func (jenkins *Jenkins) parseXmlResponse(resp *http.Response, body interface{}) (err error) {
	defer resp.Body.Close()
	if body == nil {
		return
	}
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	return xml.Unmarshal(data, body)
}

func (jenkins *Jenkins) parseResponse(resp *http.Response, body interface{}) (err error) {
	defer resp.Body.Close()

	if body == nil {
		return
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	return json.Unmarshal(data, body)
}

func (jenkins *Jenkins) get(path string, params url.Values, body interface{}) (err error) {
	requestUrl := jenkins.buildUrl(path, params)
	req, err := http.NewRequest("GET", requestUrl, nil)
	if err != nil {
		return
	}

	resp, err := jenkins.sendRequest(req)
	if err != nil {
		return
	}
	return jenkins.parseResponse(resp, body)
}

func (jenkins *Jenkins) post(path string, params url.Values, body interface{}) (err error) {
	requestUrl := jenkins.buildUrl(path, params)
	req, err := http.NewRequest("POST", requestUrl, nil)
	if err != nil {
		return
	}

	resp, err := jenkins.sendRequest(req)
	if err != nil {
		return
	}

	return jenkins.parseResponse(resp, body)
}
func (jenkins *Jenkins) postXml(path string, params url.Values, xmlBody io.Reader, body interface{}) (err error) {
	requestUrl := jenkins.baseUrl + path
	if params != nil {
		queryString := params.Encode()
		if queryString != "" {
			requestUrl = requestUrl + "?" + queryString
		}
	}

	req, err := http.NewRequest("POST", requestUrl, xmlBody)
	if err != nil {
		return
	}

	req.Header.Add("Content-Type", "application/xml")
	resp, err := jenkins.sendRequest(req)
	if err != nil {
		return
	}
	if resp.StatusCode != 200 {
		return errors.New(fmt.Sprintf("error: HTTP POST returned status code returned: %d", resp.StatusCode))
	}

	return jenkins.parseXmlResponse(resp, body)
}

// GetJobs returns all jobs you can read.
func (jenkins *Jenkins) GetJobs() ([]Job, error) {
	var payload = struct {
		Jobs []Job `json:"jobs"`
	}{}
	err := jenkins.get("", nil, &payload)
	return payload.Jobs, err
}

// GetJob returns a job which has specified name.
func (jenkins *Jenkins) GetJob(name string) (job Job, err error) {
	err = jenkins.get(fmt.Sprintf("/job/%s", name), nil, &job)
	return
}

// GetBuild returns a number-th build result of specified job.
func (jenkins *Jenkins) GetBuild(job Job, number int) (build Build, err error) {
	err = jenkins.get(fmt.Sprintf("/job/%s/%d", job.Name, number), nil, &build)
	return
}

// Create a new job
func (jenkins *Jenkins) CreateJob(mavenJobItem MavenJobItem, jobName string) error {
	mavenJobItemXml, _ := xml.Marshal(mavenJobItem)
	reader := bytes.NewReader(mavenJobItemXml)
	params := url.Values{"name": []string{jobName}}

	return jenkins.postXml("/createItem", params, reader, nil)
}

// Add job to view
func (jenkins *Jenkins) AddJobToView(viewName string, job Job) error {
	params := url.Values{"name": []string{job.Name}}
	return jenkins.post(fmt.Sprintf("/view/%s/addJobToView", viewName), params, nil)
}

// Create a new view
func (jenkins *Jenkins) CreateView(listView ListView) error {
	xmlListView, _ := xml.Marshal(listView)
	reader := bytes.NewReader(xmlListView)
	params := url.Values{"name": []string{listView.Name}}

	return jenkins.postXml("/createView", params, reader, nil)
}

// Create a new build for this job.
// Params can be nil.
func (jenkins *Jenkins) Build(job Job, params url.Values) error {
	if params == nil {
		return jenkins.post(fmt.Sprintf("/job/%s/build", job.Name), params, nil)
	} else {
		return jenkins.post(fmt.Sprintf("/job/%s/buildWithParameters", job.Name), params, nil)
	}
}

// Get the console output from a build.
func (jenkins *Jenkins) GetBuildConsoleOutput(build Build) ([]byte, error) {
	requestUrl := fmt.Sprintf("%s/consoleText", build.Url)
	req, err := http.NewRequest("GET", requestUrl, nil)
	if err != nil {
		return nil, err
	}

	res, err := jenkins.sendRequest(req)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

// GetQueue returns the current build queue from Jenkins
func (jenkins *Jenkins) GetQueue() (queue Queue, err error) {
	err = jenkins.get(fmt.Sprintf("/queue"), nil, &queue)
	return
}
<commit_msg>jenkins auth using basic authentication<commit_after>package gojenkins

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

type Auth struct {
	Username string
	ApiToken string
}

type Jenkins struct {
	auth    *Auth
	baseUrl string
}

func NewJenkins(auth *Auth, baseUrl string) *Jenkins {
	return &Jenkins{
		auth:    auth,
		baseUrl: baseUrl,
	}
}

func (jenkins *Jenkins) buildUrl(path string, params url.Values) (requestUrl string) {
	if params == nil {
		requestUrl = jenkins.baseUrl + "job/" + path
		if !strings.Contains(path, "/api/json") {
			requestUrl = jenkins.baseUrl + path + "/api/json"
		}
	}
	if params != nil {
		queryString := params.Encode()
		if queryString != "" {
			requestUrl = requestUrl + "?" + queryString
		}
	}
	return
}

func (jenkins *Jenkins) sendRequest(req *http.Request) (*http.Response, error) {
	req.SetBasicAuth(jenkins.auth.Username, jenkins.auth.ApiToken)
	return http.DefaultClient.Do(req)
}

func (jenkins *Jenkins) parseXmlResponse(resp *http.Response, body interface{}) (err error) {
	defer resp.Body.Close()
	if body == nil {
		return
	}
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	return xml.Unmarshal(data, body)
}

func (jenkins *Jenkins) parseResponse(resp *http.Response, body interface{}) (err error) {
	defer resp.Body.Close()

	if body == nil {
		return
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	return json.Unmarshal(data, body)
}

func (jenkins *Jenkins) get(path string, params url.Values, body interface{}) (err error) {
	requestUrl := jenkins.buildUrl(path, params)
	req, err := http.NewRequest("GET", requestUrl, nil)
	if err != nil {
		return
	}

	resp, err := jenkins.sendRequest(req)
	if err != nil {
		return
	}
	return jenkins.parseResponse(resp, body)
}

func (jenkins *Jenkins) post(path string, params url.Values, body interface{}) (err error) {
	requestUrl := jenkins.buildUrl(path, params)
	req, err := http.NewRequest("POST", requestUrl, nil)
	if err != nil {
		return
	}

	resp, err := jenkins.sendRequest(req)
	if err != nil {
		return
	}

	return jenkins.parseResponse(resp, body)
}
func (jenkins *Jenkins) postXml(path string, params url.Values, xmlBody io.Reader, body interface{}) (err error) {
	requestUrl := jenkins.baseUrl + path
	if params != nil {
		queryString := params.Encode()
		if queryString != "" {
			requestUrl = requestUrl + "?" + queryString
		}
	}

	req, err := http.NewRequest("POST", requestUrl, xmlBody)
	if err != nil {
		return
	}

	req.Header.Add("Content-Type", "application/xml")
	resp, err := jenkins.sendRequest(req)
	if err != nil {
		return
	}
	if resp.StatusCode != 200 {
		return errors.New(fmt.Sprintf("error: HTTP POST returned status code returned: %d", resp.StatusCode))
	}

	return jenkins.parseXmlResponse(resp, body)
}

// GetJobs returns all jobs you can read.
func (jenkins *Jenkins) GetJobs() ([]Job, error) {
	var payload = struct {
		Jobs []Job `json:"jobs"`
	}{}
	err := jenkins.get("", nil, &payload)
	return payload.Jobs, err
}

// GetJob returns a job which has specified name.
func (jenkins *Jenkins) GetJob(name string) (job Job, err error) {
	err = jenkins.get(fmt.Sprintf("/job/%s", name), nil, &job)
	return
}

// GetBuild returns a number-th build result of specified job.
func (jenkins *Jenkins) GetBuild(job Job, number int) (build Build, err error) {
	err = jenkins.get(fmt.Sprintf("/job/%s/%d", job.Name, number), nil, &build)
	return
}

// Create a new job
func (jenkins *Jenkins) CreateJob(mavenJobItem MavenJobItem, jobName string) error {
	mavenJobItemXml, _ := xml.Marshal(mavenJobItem)
	reader := bytes.NewReader(mavenJobItemXml)
	params := url.Values{"name": []string{jobName}}

	return jenkins.postXml("/createItem", params, reader, nil)
}

// Add job to view
func (jenkins *Jenkins) AddJobToView(viewName string, job Job) error {
	params := url.Values{"name": []string{job.Name}}
	return jenkins.post(fmt.Sprintf("/view/%s/addJobToView", viewName), params, nil)
}

// Create a new view
func (jenkins *Jenkins) CreateView(listView ListView) error {
	xmlListView, _ := xml.Marshal(listView)
	reader := bytes.NewReader(xmlListView)
	params := url.Values{"name": []string{listView.Name}}

	return jenkins.postXml("/createView", params, reader, nil)
}

// Create a new build for this job.
// Params can be nil.
func (jenkins *Jenkins) Build(job Job, params url.Values) error {
	if params == nil {
		return jenkins.post(fmt.Sprintf("/job/%s/build", job.Name), params, nil)
	} else {
		return jenkins.post(fmt.Sprintf("/job/%s/buildWithParameters", job.Name), params, nil)
	}
}

// Get the console output from a build.
func (jenkins *Jenkins) GetBuildConsoleOutput(build Build) ([]byte, error) {
	requestUrl := fmt.Sprintf("%s/consoleText", build.Url)
	req, err := http.NewRequest("GET", requestUrl, nil)
	if err != nil {
		return nil, err
	}

	res, err := jenkins.sendRequest(req)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

// GetQueue returns the current build queue from Jenkins
func (jenkins *Jenkins) GetQueue() (queue Queue, err error) {
	err = jenkins.get(fmt.Sprintf("/queue"), nil, &queue)
	return
}
<|endoftext|>


<commit_before>package encoders

import (
	"fmt"
	//"github.com/cznic/mathutil"
	//"github.com/zacg/floats"
	//"github.com/nupic-community/htm"
	"github.com/nupic-community/htm/utils"
	//"github.com/zacg/ints"
	//"math"
	"time"
)

/*
	Params for the date encoder
*/
type DateEncoderParams struct {
	HolidayWidth    int
	HolidayRadius   float64
	SeasonWidth     int
	SeasonRadius    float64
	DayOfWeekWidth  int
	DayOfWeekRadius float64
	WeekendWidth    int
	WeekendRadius   float64
	TimeOfDayWidth  int
	TimeOfDayRadius float64
	//CustomDays     int
	Name string
}

func NewDateEncoderParams() *DateEncoderParams {
	p := new(DateEncoderParams)

	//set defaults
	p.SeasonRadius = 91.5 //days
	p.DayOfWeekRadius = 1
	p.TimeOfDayRadius = 4
	p.WeekendRadius = 1
	p.HolidayRadius = 1

	return p
}

/*
	Date encoder encodes a datetime to a SDR. Params allow for tuning
	for specific date attributes
*/
type DateEncoder struct {
	DateEncoderParams
	seasonEncoder    *ScalerEncoder
	holidayEncoder   *ScalerEncoder
	dayOfWeekEncoder *ScalerEncoder
	weekendEncoder   *ScalerEncoder
	timeOfDayEncoder *ScalerEncoder

	width           int
	seasonOffset    int
	weekendOffset   int
	dayOfWeekOffset int
	holidayOffset   int
	timeOfDayOffset int
	Description     string
}

/*
	Intializes a new date encoder
*/
func NewDateEncoder(params *DateEncoderParams) *DateEncoder {
	de := new(DateEncoder)

	de.DateEncoderParams = *params

	de.width = 0

	if params.SeasonWidth != 0 {
		// Ignore leapyear differences -- assume 366 days in a year
		// Radius = 91.5 days = length of season
		// Value is number of days since beginning of year (0 - 355)

		sep := NewScalerEncoderParams(params.SeasonWidth, 0, 366)
		sep.Name = "Season"
		sep.Periodic = true
		sep.Radius = de.SeasonRadius
		de.seasonEncoder = NewScalerEncoder(sep)
		de.seasonOffset = de.width
		de.width += de.seasonEncoder.N
		de.Description += fmt.Sprintf("season %v", de.seasonOffset)
	}

	if params.DayOfWeekWidth != 0 {
		// Value is day of week (floating point)
		// Radius is 1 day

		sep := NewScalerEncoderParams(params.DayOfWeekWidth, 0, 7)
		sep.Name = "day of week"
		sep.Radius = de.DayOfWeekRadius
		sep.Periodic = true
		de.dayOfWeekEncoder = NewScalerEncoder(sep)
		de.dayOfWeekOffset = de.width
		de.Description += fmt.Sprintf(" day of week: %v", de.dayOfWeekOffset)
		de.width += de.dayOfWeekEncoder.N
	}

	if params.WeekendWidth != 0 {
		// Binary value. Not sure if this makes sense. Also is somewhat redundant
		// with dayOfWeek
		//Append radius if it was not provided

		sep := NewScalerEncoderParams(params.WeekendWidth, 0, 1)
		sep.Name = "weekend"
		sep.Radius = params.WeekendRadius
		de.weekendEncoder = NewScalerEncoder(sep)
		de.weekendOffset = de.width
		de.width += de.weekendEncoder.N
		de.Description += fmt.Sprintf("weekend: %v", de.weekendOffset)

	}

	if params.HolidayWidth > 0 {
		// A "continuous" binary value. = 1 on the holiday itself and smooth ramp
		// 0->1 on the day before the holiday and 1->0 on the day after the holiday.

		sep := NewScalerEncoderParams(params.HolidayWidth, 0, 1)
		sep.Name = "holiday"
		sep.Radius = params.HolidayRadius
		de.holidayEncoder = NewScalerEncoder(sep)
		de.holidayOffset = de.width
		de.width += de.holidayEncoder.N
		de.Description += fmt.Sprintf(" holiday %v", de.holidayOffset)
	}

	if params.TimeOfDayWidth > 0 {
		// Value is time of day in hours
		// Radius = 4 hours, e.g. morning, afternoon, evening, early night,
		// late night, etc.

		sep := NewScalerEncoderParams(params.TimeOfDayWidth, 0, 24)
		sep.Name = "time of day"
		sep.Radius = params.TimeOfDayRadius
		sep.Periodic = true
		de.timeOfDayEncoder = NewScalerEncoder(sep)
		de.timeOfDayOffset = de.width
		de.width += de.timeOfDayEncoder.N
		de.Description += fmt.Sprintf(" time of day: %v ", de.timeOfDayOffset)

	}

	return de
}

/*
	get season scaler from time
*/
func (de *DateEncoder) getSeasonScaler(date time.Time) float64 {
	if de.seasonEncoder == nil {
		return 0.0
	}

	//make year 0 based
	dayOfYear := float64(date.YearDay() - 1)
	return dayOfYear

}

/*
	get day of week scaler from time
*/
func (de *DateEncoder) getDayOfWeekScaler(date time.Time) float64 {
	if de.dayOfWeekEncoder == nil {
		return 0.0
	}
	return float64(date.Weekday())
}

/*
	get weekend scaler from time
*/
func (de *DateEncoder) getWeekendScaler(date time.Time) float64 {
	if de.weekendEncoder == nil {
		return 0.0
	}
	dayOfWeek := date.Weekday()
	timeOfDay := date.Hour() + date.Minute()/60.0

	// saturday, sunday or friday evening
	weekend := 0.0
	if dayOfWeek == time.Saturday ||
		dayOfWeek == time.Sunday ||
		(dayOfWeek == time.Friday && timeOfDay > 18) {
		weekend = 1.0
	}
	return weekend
}

/*
	get holiday scaler from time
*/
func (de *DateEncoder) getHolidayScaler(date time.Time) float64 {
	if de.holidayEncoder == nil {
		return 0.0
	}
	// A "continuous" binary value. = 1 on the holiday itself and smooth ramp
	// 0->1 on the day before the holiday and 1->0 on the day after the holiday.
	// Currently the only holiday we know about is December 25
	// holidays is a list of holidays that occur on a fixed date every year
	val := 0.0
	holidays := []utils.TupleInt{{12, 25}}

	for _, h := range holidays {
		// hdate is midnight on the holiday
		hDate := time.Date(date.Year(), time.Month(h.A), h.B, 0, 0, 0, 0, time.UTC)
		if date.After(hDate) {
			diff := date.Sub(hDate)
			if (diff/time.Hour)/24 == 0 {
				val = 1
				break
			} else if (diff/time.Hour)/24 == 1 {
				// ramp smoothly from 1 -> 0 on the next day
				val = 1.0 - (float64(diff/time.Second) / (86400))
				break
			}
		} else {
			diff := hDate.Sub(date)
			if (diff/time.Hour)/24 == 1 {
				// ramp smoothly from 0 -> 1 on the previous day
				val = 1.0 - (float64(diff/time.Second) / 86400)
			}

		}
	}

	return val

}

/*

*/
func (de *DateEncoder) getTimeOfDayScaler(date time.Time) float64 {
	if de.timeOfDayEncoder == nil {
		return 0.0
	}
	return float64(date.Hour()) + (float64(date.Minute()) / 60.0)

}

/*
	Encodes input to specifed slice
*/
func (de *DateEncoder) EncodeToSlice(date time.Time, output []bool) {

	learn := false

	// Get a scaler value for each subfield and encode it with the
	// appropriate encoder
	if de.seasonEncoder != nil {
		val := de.getSeasonScaler(date)
		de.seasonEncoder.EncodeToSlice(val, learn, output[de.seasonOffset:])
	}

	if de.holidayEncoder != nil {
		val := de.getHolidayScaler(date)
		de.holidayEncoder.EncodeToSlice(val, learn, output[de.holidayOffset:])
	}

	if de.dayOfWeekEncoder != nil {
		val := de.getDayOfWeekScaler(date)
		de.dayOfWeekEncoder.EncodeToSlice(val, learn, output[de.dayOfWeekOffset:])
	}

	if de.weekendEncoder != nil {
		val := de.getWeekendScaler(date)
		de.weekendEncoder.EncodeToSlice(val, learn, output[de.weekendOffset:])
	}

	if de.timeOfDayEncoder != nil {
		val := de.getTimeOfDayScaler(date)
		de.timeOfDayEncoder.EncodeToSlice(val, learn, output[de.timeOfDayOffset:])
	}

}

/*
	Returns encoded date/time
*/
func (de *DateEncoder) Encode(date time.Time) []bool {
	output := make([]bool, de.width)
	de.EncodeToSlice(date, output)
	return output
}
<commit_msg>add description function<commit_after>package encoders

import (
	"fmt"
	//"github.com/cznic/mathutil"
	//"github.com/zacg/floats"
	//"github.com/nupic-community/htm"
	"github.com/nupic-community/htm/utils"
	//"github.com/zacg/ints"
	//"math"
	"time"
)

/*
	Params for the date encoder
*/
type DateEncoderParams struct {
	HolidayWidth    int
	HolidayRadius   float64
	SeasonWidth     int
	SeasonRadius    float64
	DayOfWeekWidth  int
	DayOfWeekRadius float64
	WeekendWidth    int
	WeekendRadius   float64
	TimeOfDayWidth  int
	TimeOfDayRadius float64
	//CustomDays     int
	Name string
}

func NewDateEncoderParams() *DateEncoderParams {
	p := new(DateEncoderParams)

	//set defaults
	p.SeasonRadius = 91.5 //days
	p.DayOfWeekRadius = 1
	p.TimeOfDayRadius = 4
	p.WeekendRadius = 1
	p.HolidayRadius = 1

	return p
}

/*
	Date encoder encodes a datetime to a SDR. Params allow for tuning
	for specific date attributes
*/
type DateEncoder struct {
	DateEncoderParams
	seasonEncoder    *ScalerEncoder
	holidayEncoder   *ScalerEncoder
	dayOfWeekEncoder *ScalerEncoder
	weekendEncoder   *ScalerEncoder
	timeOfDayEncoder *ScalerEncoder

	width           int
	seasonOffset    int
	weekendOffset   int
	dayOfWeekOffset int
	holidayOffset   int
	timeOfDayOffset int
}

/*
	Intializes a new date encoder
*/
func NewDateEncoder(params *DateEncoderParams) *DateEncoder {
	de := new(DateEncoder)

	de.DateEncoderParams = *params

	de.width = 0

	if params.SeasonWidth != 0 {
		// Ignore leapyear differences -- assume 366 days in a year
		// Radius = 91.5 days = length of season
		// Value is number of days since beginning of year (0 - 355)

		sep := NewScalerEncoderParams(params.SeasonWidth, 0, 366)
		sep.Name = "Season"
		sep.Periodic = true
		sep.Radius = de.SeasonRadius
		de.seasonEncoder = NewScalerEncoder(sep)
		de.seasonOffset = de.width
		de.width += de.seasonEncoder.N
	}

	if params.DayOfWeekWidth != 0 {
		// Value is day of week (floating point)
		// Radius is 1 day

		sep := NewScalerEncoderParams(params.DayOfWeekWidth, 0, 7)
		sep.Name = "day of week"
		sep.Radius = de.DayOfWeekRadius
		sep.Periodic = true
		de.dayOfWeekEncoder = NewScalerEncoder(sep)
		de.dayOfWeekOffset = de.width
		de.width += de.dayOfWeekEncoder.N
	}

	if params.WeekendWidth != 0 {
		// Binary value. Not sure if this makes sense. Also is somewhat redundant
		// with dayOfWeek
		//Append radius if it was not provided

		sep := NewScalerEncoderParams(params.WeekendWidth, 0, 1)
		sep.Name = "weekend"
		sep.Radius = params.WeekendRadius
		de.weekendEncoder = NewScalerEncoder(sep)
		de.weekendOffset = de.width
		de.width += de.weekendEncoder.N
	}

	if params.HolidayWidth > 0 {
		// A "continuous" binary value. = 1 on the holiday itself and smooth ramp
		// 0->1 on the day before the holiday and 1->0 on the day after the holiday.

		sep := NewScalerEncoderParams(params.HolidayWidth, 0, 1)
		sep.Name = "holiday"
		sep.Radius = params.HolidayRadius
		de.holidayEncoder = NewScalerEncoder(sep)
		de.holidayOffset = de.width
		de.width += de.holidayEncoder.N
	}

	if params.TimeOfDayWidth > 0 {
		// Value is time of day in hours
		// Radius = 4 hours, e.g. morning, afternoon, evening, early night,
		// late night, etc.

		sep := NewScalerEncoderParams(params.TimeOfDayWidth, 0, 24)
		sep.Name = "time of day"
		sep.Radius = params.TimeOfDayRadius
		sep.Periodic = true
		de.timeOfDayEncoder = NewScalerEncoder(sep)
		de.timeOfDayOffset = de.width
		de.width += de.timeOfDayEncoder.N

	}

	return de
}

/*
	get season scaler from time
*/
func (de *DateEncoder) getSeasonScaler(date time.Time) float64 {
	if de.seasonEncoder == nil {
		return 0.0
	}

	//make year 0 based
	dayOfYear := float64(date.YearDay() - 1)
	return dayOfYear

}

/*
	get day of week scaler from time
*/
func (de *DateEncoder) getDayOfWeekScaler(date time.Time) float64 {
	if de.dayOfWeekEncoder == nil {
		return 0.0
	}
	return float64(date.Weekday())
}

/*
	get weekend scaler from time
*/
func (de *DateEncoder) getWeekendScaler(date time.Time) float64 {
	if de.weekendEncoder == nil {
		return 0.0
	}
	dayOfWeek := date.Weekday()
	timeOfDay := date.Hour() + date.Minute()/60.0

	// saturday, sunday or friday evening
	weekend := 0.0
	if dayOfWeek == time.Saturday ||
		dayOfWeek == time.Sunday ||
		(dayOfWeek == time.Friday && timeOfDay > 18) {
		weekend = 1.0
	}
	return weekend
}

/*
	get holiday scaler from time
*/
func (de *DateEncoder) getHolidayScaler(date time.Time) float64 {
	if de.holidayEncoder == nil {
		return 0.0
	}
	// A "continuous" binary value. = 1 on the holiday itself and smooth ramp
	// 0->1 on the day before the holiday and 1->0 on the day after the holiday.
	// Currently the only holiday we know about is December 25
	// holidays is a list of holidays that occur on a fixed date every year
	val := 0.0
	holidays := []utils.TupleInt{{12, 25}}

	for _, h := range holidays {
		// hdate is midnight on the holiday
		hDate := time.Date(date.Year(), time.Month(h.A), h.B, 0, 0, 0, 0, time.UTC)
		if date.After(hDate) {
			diff := date.Sub(hDate)
			if (diff/time.Hour)/24 == 0 {
				val = 1
				break
			} else if (diff/time.Hour)/24 == 1 {
				// ramp smoothly from 1 -> 0 on the next day
				val = 1.0 - (float64(diff/time.Second) / (86400))
				break
			}
		} else {
			diff := hDate.Sub(date)
			if (diff/time.Hour)/24 == 1 {
				// ramp smoothly from 0 -> 1 on the previous day
				val = 1.0 - (float64(diff/time.Second) / 86400)
			}

		}
	}

	return val

}

/*

*/
func (de *DateEncoder) getTimeOfDayScaler(date time.Time) float64 {
	if de.timeOfDayEncoder == nil {
		return 0.0
	}
	return float64(date.Hour()) + (float64(date.Minute()) / 60.0)

}

/*
	Encodes input to specifed slice
*/
func (de *DateEncoder) EncodeToSlice(date time.Time, output []bool) {

	learn := false

	// Get a scaler value for each subfield and encode it with the
	// appropriate encoder
	if de.seasonEncoder != nil {
		val := de.getSeasonScaler(date)
		de.seasonEncoder.EncodeToSlice(val, learn, output[de.seasonOffset:])
	}

	if de.holidayEncoder != nil {
		val := de.getHolidayScaler(date)
		de.holidayEncoder.EncodeToSlice(val, learn, output[de.holidayOffset:])
	}

	if de.dayOfWeekEncoder != nil {
		val := de.getDayOfWeekScaler(date)
		de.dayOfWeekEncoder.EncodeToSlice(val, learn, output[de.dayOfWeekOffset:])
	}

	if de.weekendEncoder != nil {
		val := de.getWeekendScaler(date)
		de.weekendEncoder.EncodeToSlice(val, learn, output[de.weekendOffset:])
	}

	if de.timeOfDayEncoder != nil {
		val := de.getTimeOfDayScaler(date)
		de.timeOfDayEncoder.EncodeToSlice(val, learn, output[de.timeOfDayOffset:])
	}

}

/*
	Returns encoded date/time
*/
func (de *DateEncoder) Encode(date time.Time) []bool {
	output := make([]bool, de.width)
	de.EncodeToSlice(date, output)
	return output
}

/*
 Encoder description
*/
func (de *DateEncoder) Description(date time.Time) string {
	return fmt.Sprintf("season %v ", de.seasonOffset) +
		fmt.Sprintf(" day of week: %v", de.dayOfWeekOffset) +
		fmt.Sprintf(" weekend: %v", de.weekendOffset) +
		fmt.Sprintf(" holiday %v", de.holidayOffset) +
		fmt.Sprintf(" time of day: %v ", de.timeOfDayOffset)
}
<|endoftext|>


<commit_before>/*--------------------------------------------------------*\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: https://hprose.com                     |
|                                                          |
| encoding/big_decoder.go                                  |
|                                                          |
| LastModified: Apr 25, 2020                               |
| Author: Ma Bingyao <andot@hprose.com>                    |
|                                                          |
\*________________________________________________________*/

package encoding

import (
	"math"
	"math/big"
)

var (
	bigIntZero   = big.NewInt(0)
	bigIntOne    = big.NewInt(1)
	bigFloatZero = big.NewFloat(0)
	bigFloatOne  = big.NewFloat(1)
)

func (dec *Decoder) strToBigInt(s string) *big.Int {
	if bi, ok := new(big.Int).SetString(s, 10); ok {
		return bi
	}
	dec.decodeStringError(s, "big.Int")
	return nil
}

// ReadBigInt read *big.Int
func (dec *Decoder) ReadBigInt() *big.Int {
	return dec.strToBigInt(unsafeString(dec.Until(TagSemicolon)))
}

func (dec *Decoder) strToBigFloat(s string) *big.Float {
	if bf, ok := new(big.Float).SetString(s); ok {
		return bf
	}
	dec.decodeStringError(s, "big.Float")
	return nil
}

// ReadBigFloat read *big.Float
func (dec *Decoder) ReadBigFloat() *big.Float {
	return dec.strToBigFloat(unsafeString(dec.Until(TagSemicolon)))
}

// bigIntDecoder is the implementation of ValueDecoder for big.Int/*big.Int.
type bigIntDecoder struct{}

func (valdec bigIntDecoder) decode(dec *Decoder, tag byte) *big.Int {
	if i := intDigits[tag]; i != invalidDigit {
		return big.NewInt(int64(i))
	}
	switch tag {
	case TagEmpty, TagFalse:
		return bigIntZero
	case TagTrue:
		return bigIntOne
	case TagInteger:
		return big.NewInt(dec.ReadInt64())
	case TagLong:
		return dec.ReadBigInt()
	case TagDouble:
		if bf := dec.strToBigFloat(unsafeString(dec.Until(TagSemicolon))); bf != nil {
			bi, _ := bf.Int(nil)
			return bi
		}
	case TagUTF8Char:
		return dec.strToBigInt(dec.readUnsafeString(1))
	case TagString:
		return dec.strToBigInt(dec.ReadString())
	}
	return nil
}

func (valdec bigIntDecoder) Decode(dec *Decoder, p interface{}, tag byte) {
	if tag == TagNull {
		switch pv := p.(type) {
		case **big.Int:
			*pv = nil
		case *big.Int:
			*pv = *bigIntZero
		}
		return
	}
	bi := valdec.decode(dec, tag)
	if bi == nil {
		dec.decodeError(p, tag)
		return
	}
	switch pv := p.(type) {
	case **big.Int:
		*pv = bi
	case *big.Int:
		*pv = *bi
	}
}

// bigFloatDecoder is the implementation of ValueDecoder for big.Float/*big.Float.
type bigFloatDecoder struct{}

func (valdec bigFloatDecoder) decode(dec *Decoder, tag byte) *big.Float {
	if i := intDigits[tag]; i != invalidDigit {
		return big.NewFloat(float64(i))
	}
	switch tag {
	case TagEmpty, TagFalse:
		return bigFloatZero
	case TagTrue:
		return bigFloatOne
	case TagInteger:
		return big.NewFloat(float64(dec.ReadInt64()))
	case TagLong, TagDouble:
		return dec.ReadBigFloat()
	case TagInfinity:
		if dec.NextByte() == TagNeg {
			return big.NewFloat(math.Inf(-1))
		}
		return big.NewFloat(math.Inf(1))
	case TagUTF8Char:
		return dec.strToBigFloat(dec.readUnsafeString(1))
	case TagString:
		return dec.strToBigFloat(dec.ReadString())
	}
	return nil
}

func (valdec bigFloatDecoder) Decode(dec *Decoder, p interface{}, tag byte) {
	if tag == TagNull {
		switch pv := p.(type) {
		case **big.Float:
			*pv = nil
		case *big.Float:
			*pv = *bigFloatZero
		}
		return
	}
	bf := valdec.decode(dec, tag)
	if bf == nil {
		dec.decodeError(p, tag)
		return
	}
	switch pv := p.(type) {
	case **big.Float:
		*pv = bf
	case *big.Float:
		*pv = *bf
	}
}

func init() {
	RegisterValueDecoder((*big.Int)(nil), bigIntDecoder{})
	RegisterValueDecoder((*big.Float)(nil), bigFloatDecoder{})
	// RegisterValueDecoder((*big.Rat)(nil), bigRatDecoder{})
}
<commit_msg>Update big_decoder.go<commit_after>/*--------------------------------------------------------*\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: https://hprose.com                     |
|                                                          |
| encoding/big_decoder.go                                  |
|                                                          |
| LastModified: May 1, 2020                                |
| Author: Ma Bingyao <andot@hprose.com>                    |
|                                                          |
\*________________________________________________________*/

package encoding

import (
	"math"
	"math/big"
)

var (
	bigIntZero   = big.NewInt(0)
	bigIntOne    = big.NewInt(1)
	bigFloatZero = big.NewFloat(0)
	bigFloatOne  = big.NewFloat(1)
)

func (dec *Decoder) strToBigInt(s string) *big.Int {
	if bi, ok := new(big.Int).SetString(s, 10); ok {
		return bi
	}
	dec.decodeStringError(s, "big.Int")
	return nil
}

// ReadBigInt read *big.Int
func (dec *Decoder) ReadBigInt() *big.Int {
	return dec.strToBigInt(unsafeString(dec.Until(TagSemicolon)))
}

func (dec *Decoder) strToBigFloat(s string) *big.Float {
	if bf, ok := new(big.Float).SetString(s); ok {
		return bf
	}
	dec.decodeStringError(s, "big.Float")
	return nil
}

// ReadBigFloat read *big.Float
func (dec *Decoder) ReadBigFloat() *big.Float {
	return dec.strToBigFloat(unsafeString(dec.Until(TagSemicolon)))
}

// bigIntDecoder is the implementation of ValueDecoder for big.Int/*big.Int.
type bigIntDecoder struct{}

func (valdec bigIntDecoder) decode(dec *Decoder, p interface{}, tag byte) *big.Int {
	if i := intDigits[tag]; i != invalidDigit {
		return big.NewInt(int64(i))
	}
	switch tag {
	case TagEmpty, TagFalse:
		return bigIntZero
	case TagTrue:
		return bigIntOne
	case TagInteger:
		return big.NewInt(dec.ReadInt64())
	case TagLong:
		return dec.ReadBigInt()
	case TagDouble:
		if bf := dec.strToBigFloat(unsafeString(dec.Until(TagSemicolon))); bf != nil {
			bi, _ := bf.Int(nil)
			return bi
		}
	case TagUTF8Char:
		return dec.strToBigInt(dec.readUnsafeString(1))
	case TagString:
		return dec.strToBigInt(dec.ReadString())
	default:
		dec.decodeError(p, tag)
	}
	return nil
}

func (valdec bigIntDecoder) Decode(dec *Decoder, p interface{}, tag byte) {
	if tag == TagNull {
		switch pv := p.(type) {
		case **big.Int:
			*pv = nil
		case *big.Int:
			*pv = *bigIntZero
		}
		return
	}
	bi := valdec.decode(dec, p, tag)
	if dec.Error != nil {
		return
	}
	switch pv := p.(type) {
	case **big.Int:
		*pv = bi
	case *big.Int:
		*pv = *bi
	}
}

// bigFloatDecoder is the implementation of ValueDecoder for big.Float/*big.Float.
type bigFloatDecoder struct{}

func (valdec bigFloatDecoder) decode(dec *Decoder, p interface{}, tag byte) *big.Float {
	if i := intDigits[tag]; i != invalidDigit {
		return big.NewFloat(float64(i))
	}
	switch tag {
	case TagEmpty, TagFalse:
		return bigFloatZero
	case TagTrue:
		return bigFloatOne
	case TagInteger:
		return big.NewFloat(float64(dec.ReadInt64()))
	case TagLong, TagDouble:
		return dec.ReadBigFloat()
	case TagInfinity:
		if dec.NextByte() == TagNeg {
			return big.NewFloat(math.Inf(-1))
		}
		return big.NewFloat(math.Inf(1))
	case TagUTF8Char:
		return dec.strToBigFloat(dec.readUnsafeString(1))
	case TagString:
		return dec.strToBigFloat(dec.ReadString())
	default:
		dec.decodeError(p, tag)
	}
	return nil
}

func (valdec bigFloatDecoder) Decode(dec *Decoder, p interface{}, tag byte) {
	if tag == TagNull {
		switch pv := p.(type) {
		case **big.Float:
			*pv = nil
		case *big.Float:
			*pv = *bigFloatZero
		}
		return
	}
	bf := valdec.decode(dec, p, tag)
	if dec.Error != nil {
		return
	}
	switch pv := p.(type) {
	case **big.Float:
		*pv = bf
	case *big.Float:
		*pv = *bf
	}
}

func init() {
	RegisterValueDecoder((*big.Int)(nil), bigIntDecoder{})
	RegisterValueDecoder((*big.Float)(nil), bigFloatDecoder{})
	// RegisterValueDecoder((*big.Rat)(nil), bigRatDecoder{})
}
<|endoftext|>


<commit_before>package job

import (
	"net/url"

	"github.com/Sirupsen/logrus"
	"github.com/robfig/cron"
)

// Scheduler interface implements the Schedule method wich will schedule a job
// to be executed in the future
type Scheduler interface {
	Schedule(c cron.Cron) error
}

// Job is the most basic unit of job without execution
type Job struct {
	ID          int
	Name        string
	Description string
	When        string
	Active      bool
}

// HTTPJob is the unit of job to be executed periodically making an HTTP call
type HTTPJob struct {
	Job
	URL *url.URL
	// TODO: method
	// TODO: body
	// TODO: headers
}

// Schedule schedules an HTTP call
func (h *HTTPJob) Schedule(c cron.Cron) error {
	logrus.Debug("Schedule %s job on %s with the url $v", h.Name, h.When, *h.URL)
	return nil
}
<commit_msg>Add custom json marshaller to httpjob<commit_after>package job

import (
	"encoding/json"
	"net/url"

	"github.com/Sirupsen/logrus"
	"github.com/robfig/cron"
)

// Scheduler interface implements the Schedule method wich will schedule a job
// to be executed in the future
type Scheduler interface {
	Schedule(c cron.Cron) error
}

// Job is the most basic unit of job without execution
type Job struct {
	ID          int
	Name        string
	Description string
	When        string
	Active      bool
}

// HTTPJob is the unit of job to be executed periodically making an HTTP call
type HTTPJob struct {
	Job
	URL *url.URL
	// TODO: method
	// TODO: body
	// TODO: headers
}

// Schedule schedules an HTTP call
func (h *HTTPJob) Schedule(c cron.Cron) error {
	logrus.Debug("Schedule %s job on %s with the url $v", h.Name, h.When, *h.URL)
	return nil
}

// MarshalJSON is a vustom json marshaller for HTTPJob
func (h *HTTPJob) MarshalJSON() ([]byte, error) {
	// Alias is a custo type to inherint all the properties of HTTPJob but not the methods
	type Alias HTTPJob

	return json.Marshal(struct {
		Alias
		URL string
	}{
		Alias: Alias(*h),
		URL:   h.URL.String(),
	})
}
<|endoftext|>


<commit_before>package walnut

import (
	"errors"
	"reflect"
	"testing"
)

var definitionTests = []struct {
	in   string
	defs []definition
	err  error
}{
	{
		"",
		[]definition{},
		nil,
	},
	{
		"\n\t\t\n\n ",
		[]definition{},
		nil,
	},
	{
		"#abc",
		[]definition{},
		nil,
	},
	{
		"a=1",
		[]definition{
			{"a", int64(1), "1", 1},
		},
		nil,
	},
	{
		"b=2\nc=3",
		[]definition{
			{"b", int64(2), "2", 1},
			{"c", int64(3), "3", 2},
		},
		nil,
	},
	{
		"d\n e=4",
		[]definition{
			{"d.e", int64(4), "4", 2},
		},
		nil,
	},
	{
		"foo\n\tbar=5\n\tbaz=6",
		[]definition{
			{"foo.bar", int64(5), "5", 2},
			{"foo.baz", int64(6), "6", 3},
		},
		nil,
	},
	{
		"#\nabc\n def=7\n #\n ghi=8",
		[]definition{
			{"abc.def", int64(7), "7", 3},
			{"abc.ghi", int64(8), "8", 5},
		},
		nil,
	},
	{
		" foo=3",
		nil,
		errors.New("invalid indentation on line 1"),
	},
	{
		"a=1\n b=2",
		nil,
		errors.New("invalid indentation on line 2"),
	},
	{
		"c\n d=3\n  e=4",
		nil,
		errors.New("invalid indentation on line 3"),
	},
	{
		"f\n  g=5\n h=6",
		nil,
		errors.New("invalid indentation on line 3"),
	},
	{
		"i\n\t j=7\n  k=8",
		nil,
		errors.New("invalid indentation on line 3"),
	},
}

func TestDefinitions(t *testing.T) {
	for _, test := range definitionTests {
		defs, err := parse([]byte(test.in))

		if !reflect.DeepEqual(err, test.err) ||
			!reflect.DeepEqual(defs, test.defs) {
			t.Errorf("parse(%#v) -> %v, %#v (want %v, %#v)",
				test.in, defs, err, test.defs, test.err)
		}
	}
}
<commit_msg>Extend parse_test.go<commit_after>package walnut

import (
	"errors"
	"reflect"
	"testing"
	"time"
)

var definitionTests = []struct {
	in   string
	defs []definition
	err  error
}{
	{
		"",
		[]definition{},
		nil,
	},
	{
		"\n\t\t\n\n ",
		[]definition{},
		nil,
	},
	{
		"#abc",
		[]definition{},
		nil,
	},
	{
		"a=1",
		[]definition{
			{"a", int64(1), "1", 1},
		},
		nil,
	},
	{
		"b=2\nc=3",
		[]definition{
			{"b", int64(2), "2", 1},
			{"c", int64(3), "3", 2},
		},
		nil,
	},
	{
		"d\n e=4",
		[]definition{
			{"d.e", int64(4), "4", 2},
		},
		nil,
	},
	{
		"foo\n\tbar=5\n\tbaz=6",
		[]definition{
			{"foo.bar", int64(5), "5", 2},
			{"foo.baz", int64(6), "6", 3},
		},
		nil,
	},
	{
		"#\nabc\n def=7\n #\n ghi=8",
		[]definition{
			{"abc.def", int64(7), "7", 3},
			{"abc.ghi", int64(8), "8", 5},
		},
		nil,
	},
	{
		"label\n  # empty\n key = \"test\"",
		[]definition{
			{"label.key", "test", "\"test\"", 3},
		},
		nil,
	},
	{
		"bool = true",
		[]definition{
			{"bool", true, "true", 1},
		},
		nil,
	},
	{
		"int64 = 12345",
		[]definition{
			{"int64", int64(12345), "12345", 1},
		},
		nil,
	},
	{
		"float64 = 123.45",
		[]definition{
			{"float64", float64(123.45), "123.45", 1},
		},
		nil,
	},
	{
		"string = \"hello\"",
		[]definition{
			{"string", "hello", "\"hello\"", 1},
		},
		nil,
	},
	{
		"time = 2012-01-02 15:30:28.000000000789 +0000",
		func() []definition {
			raw := "2012-01-02 15:30:28.000000000789 +0000"
			t, _ := time.Parse("2006-01-02 15:04:05 -0700", raw)
			return []definition{{"time", t, raw, 1}}
		}(),
		nil,
	},
	{
		"duration = 10m 20s",
		[]definition{
			{"duration", 10*time.Minute + 20*time.Second, "10m 20s", 1},
		},
		nil,
	},
	{
		" foo=3",
		nil,
		errors.New("invalid indentation on line 1"),
	},
	{
		"a=1\n b=2",
		nil,
		errors.New("invalid indentation on line 2"),
	},
	{
		"c\n d=3\n  e=4",
		nil,
		errors.New("invalid indentation on line 3"),
	},
	{
		"f\n  g=5\n h=6",
		nil,
		errors.New("invalid indentation on line 3"),
	},
	{
		"i\n\t j=7\n  k=8",
		nil,
		errors.New("invalid indentation on line 3"),
	},
	{
		"= \"missing\"",
		nil,
		errors.New("invalid key on line 1"),
	},
	{
		"invalid = 3 3",
		nil,
		errors.New("unrecognized value on line 1: \"3 3\""),
	},
	{
		"invalid == 3",
		nil,
		errors.New("unrecognized value on line 1: \"= 3\""),
	},
}

func TestDefinitions(t *testing.T) {
	for _, test := range definitionTests {
		defs, err := parse([]byte(test.in))

		if !reflect.DeepEqual(err, test.err) ||
			!reflect.DeepEqual(defs, test.defs) {
			t.Errorf("parse(%#v) -> %v, %#v (want %v, %#v)",
				test.in, defs, err, test.defs, test.err)
		}
	}
}
<|endoftext|>


<commit_before>package core

// TODO: nsq logger

import (
	"errors"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"time"

	"github.com/boltdb/bolt"
	_ "github.com/go-sql-driver/mysql"
	"github.com/jinzhu/gorm"
	_ "github.com/lib/pq"
	"github.com/nsqio/go-nsq"

	"github.com/Sirupsen/logrus"
	_ "github.com/toorop/go-sqlite3"
	"github.com/toorop/gopenstack/context"
	"github.com/toorop/gopenstack/identity"
)

const (
	// Time822 formt time for RFC 822
	Time822 = "02 Jan 2006 15:04:05 -0700" // "02 Jan 06 15:04 -0700"
)

var (
	// Version is tamil version
	Version string
	Cfg     *Config
	DB      *gorm.DB
	Bolt    *bolt.DB
	//Log                              *Logger
	Logger                           *logrus.Logger
	NsqQueueProducer                 *nsq.Producer
	SmtpSessionsCount                int
	ChSmtpSessionsCount              chan int
	DeliverdConcurrencyLocalCount    int
	DeliverdConcurrencyRemoteCount   int
	ChDeliverdConcurrencyRemoteCount chan int
	Store                            Storer
)

// Bootstrap DB, config,...
// TODO check validity of each element
func Bootstrap() (err error) {
	// Load config
	Cfg, err = InitConfig("tmail")
	if err != nil {
		return
	}

	// linit logger
	var out io.Writer
	logPath := Cfg.GetLogPath()
	customFormatter := new(logrus.TextFormatter)
	if logPath == "stdout" {
		out = os.Stdout
	} else if logPath == "discard" {
		out = ioutil.Discard
	} else {
		file := path.Join(logPath, "current.log")
		out, err = os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		customFormatter = *logrus.TextFormatter(new(FileFormatter))
		if err != nil {
			return
		}
	}

	customFormatter.TimestampFormat = time.RFC3339Nano
	customFormatter.FullTimestamp = true

	Logger = logrus.New()
	Logger.Formatter = customFormatter
	if Cfg.GetDebugEnabled() {
		Logger.Level = logrus.DebugLevel
	} else {
		Logger.Level = logrus.InfoLevel
	}
	Logger.Out = out
	Logger.Debug("Logger initialized")

	// Init DB
	DB, err = gorm.Open(Cfg.GetDbDriver(), Cfg.GetDbSource())
	if err != nil {
		return
	}
	DB.SetLogger(Logger)
	DB.LogMode(Cfg.GetDebugEnabled())

	// ping DB
	if DB.DB().Ping() != nil {
		return errors.New("I could not access to database " + Cfg.GetDbDriver() + " " + Cfg.GetDbSource())
	}

	// TODO remove from bootstrap
	// init NSQ MailQueueProducer (Nmqp)
	if Cfg.GetLaunchSmtpd() {
		err = initMailQueueProducer()
		if err != nil {
			return err
		}
	}

	// SMTP in sessions counter
	SmtpSessionsCount = 0
	ChSmtpSessionsCount = make(chan int)
	go func() {
		for {
			SmtpSessionsCount += <-ChSmtpSessionsCount
		}
	}()

	// Deliverd remote process
	DeliverdConcurrencyRemoteCount = 0
	ChDeliverdConcurrencyRemoteCount = make(chan int)
	go func() {
		for {
			DeliverdConcurrencyRemoteCount += <-ChDeliverdConcurrencyRemoteCount
		}
	}()

	// openstack
	if Cfg.GetOpenstackEnable() {
		if !context.Keyring.IsPopulate() {
			return errors.New("No credentials found from ENV. See http://docs.openstack.org/cli-reference/content/cli_openrc.html")
		}
		// Do auth
		err = identity.DoAuth()
		if err != nil {
			return err
		}
		// auto update Token
		identity.AutoUpdate(30, new(log.Logger))
	}

	// init store
	Store, err = NewStore(Cfg.GetStoreDriver(), Cfg.GetStoreSource())
	if err != nil {
		return err
	}
	// TODO gestion erreur
	execTmailPlugins("postinit")

	return
}

// InitBolt init bolt
func InitBolt() error {
	var err error
	// init Bolt DB
	Bolt, err = bolt.Open(Cfg.GetBoltFile(), 0600, nil)
	if err != nil {
		return err
	}
	// create buckets if not exists
	return Bolt.Update(func(tx *bolt.Tx) error {
		if _, err = tx.CreateBucketIfNotExists([]byte("koip")); err != nil {
			return err
		}
		return nil
	})
}

// initMailQueueProducer init producer for queue
func initMailQueueProducer() (err error) {
	nsqCfg := nsq.NewConfig()
	nsqCfg.UserAgent = "tmail.queue"
	NsqQueueProducer, err = nsq.NewProducer("127.0.0.1:4150", nsqCfg)
	if Cfg.GetDebugEnabled() {
		NsqQueueProducer.SetLogger(NewNSQLogger(), nsq.LogLevelDebug)
	} else {
		NsqQueueProducer.SetLogger(NewNSQLogger(), nsq.LogLevelError)
	}
	return err
}
<commit_msg>Toorop saved me !<commit_after>package core

// TODO: nsq logger

import (
	"errors"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"time"

	"github.com/boltdb/bolt"
	_ "github.com/go-sql-driver/mysql"
	"github.com/jinzhu/gorm"
	_ "github.com/lib/pq"
	"github.com/nsqio/go-nsq"

	"github.com/Sirupsen/logrus"
	_ "github.com/toorop/go-sqlite3"
	"github.com/toorop/gopenstack/context"
	"github.com/toorop/gopenstack/identity"
)

const (
	// Time822 formt time for RFC 822
	Time822 = "02 Jan 2006 15:04:05 -0700" // "02 Jan 06 15:04 -0700"
)

var (
	// Version is tamil version
	Version string
	Cfg     *Config
	DB      *gorm.DB
	Bolt    *bolt.DB
	//Log                              *Logger
	Logger                           *logrus.Logger
	NsqQueueProducer                 *nsq.Producer
	SmtpSessionsCount                int
	ChSmtpSessionsCount              chan int
	DeliverdConcurrencyLocalCount    int
	DeliverdConcurrencyRemoteCount   int
	ChDeliverdConcurrencyRemoteCount chan int
	Store                            Storer
)

// Bootstrap DB, config,...
// TODO check validity of each element
func Bootstrap() (err error) {
	// Load config
	Cfg, err = InitConfig("tmail")
	if err != nil {
		return
	}

	// linit logger
	var out io.Writer

	logPath := Cfg.GetLogPath()
	Logger = logrus.New()
	//customFormatter := new(logrus.TextFormatter)
	//customFormatter := new(FileFormatter)
	if logPath == "stdout" {
		out = os.Stdout
		f := new(logrus.TextFormatter)
		f.TimestampFormat = time.RFC3339Nano
		f.FullTimestamp = true
		Logger.Formatter = f

	} else if logPath == "discard" {
		out = ioutil.Discard
		f := new(logrus.TextFormatter)
		f.TimestampFormat = time.RFC3339Nano
		f.FullTimestamp = true
		Logger.Formatter = f
	} else {
		file := path.Join(logPath, "current.log")
		out, err = os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			return
		}
		f := new(FileFormatter)
		f.TimestampFormat = time.RFC3339Nano
		f.FullTimestamp = true
		Logger.Formatter = f
	}

	if Cfg.GetDebugEnabled() {
		Logger.Level = logrus.DebugLevel
	} else {
		Logger.Level = logrus.InfoLevel
	}
	Logger.Out = out
	Logger.Debug("Logger initialized")

	// Init DB
	DB, err = gorm.Open(Cfg.GetDbDriver(), Cfg.GetDbSource())
	if err != nil {
		return
	}
	DB.SetLogger(Logger)
	DB.LogMode(Cfg.GetDebugEnabled())

	// ping DB
	if DB.DB().Ping() != nil {
		return errors.New("I could not access to database " + Cfg.GetDbDriver() + " " + Cfg.GetDbSource())
	}

	// TODO remove from bootstrap
	// init NSQ MailQueueProducer (Nmqp)
	if Cfg.GetLaunchSmtpd() {
		err = initMailQueueProducer()
		if err != nil {
			return err
		}
	}

	// SMTP in sessions counter
	SmtpSessionsCount = 0
	ChSmtpSessionsCount = make(chan int)
	go func() {
		for {
			SmtpSessionsCount += <-ChSmtpSessionsCount
		}
	}()

	// Deliverd remote process
	DeliverdConcurrencyRemoteCount = 0
	ChDeliverdConcurrencyRemoteCount = make(chan int)
	go func() {
		for {
			DeliverdConcurrencyRemoteCount += <-ChDeliverdConcurrencyRemoteCount
		}
	}()

	// openstack
	if Cfg.GetOpenstackEnable() {
		if !context.Keyring.IsPopulate() {
			return errors.New("No credentials found from ENV. See http://docs.openstack.org/cli-reference/content/cli_openrc.html")
		}
		// Do auth
		err = identity.DoAuth()
		if err != nil {
			return err
		}
		// auto update Token
		identity.AutoUpdate(30, new(log.Logger))
	}

	// init store
	Store, err = NewStore(Cfg.GetStoreDriver(), Cfg.GetStoreSource())
	if err != nil {
		return err
	}
	// TODO gestion erreur
	execTmailPlugins("postinit")

	return
}

// InitBolt init bolt
func InitBolt() error {
	var err error
	// init Bolt DB
	Bolt, err = bolt.Open(Cfg.GetBoltFile(), 0600, nil)
	if err != nil {
		return err
	}
	// create buckets if not exists
	return Bolt.Update(func(tx *bolt.Tx) error {
		if _, err = tx.CreateBucketIfNotExists([]byte("koip")); err != nil {
			return err
		}
		return nil
	})
}

// initMailQueueProducer init producer for queue
func initMailQueueProducer() (err error) {
	nsqCfg := nsq.NewConfig()
	nsqCfg.UserAgent = "tmail.queue"
	NsqQueueProducer, err = nsq.NewProducer("127.0.0.1:4150", nsqCfg)
	if Cfg.GetDebugEnabled() {
		NsqQueueProducer.SetLogger(NewNSQLogger(), nsq.LogLevelDebug)
	} else {
		NsqQueueProducer.SetLogger(NewNSQLogger(), nsq.LogLevelError)
	}
	return err
}
<|endoftext|>


<commit_before>/*

	golden is a package designed to make it possible to compare a game to a
	golden run for testing purposes. It takes a record saved in
	storage/filesystem format and compares it.

*/
package golden

import (
	"errors"
	"github.com/jkomoros/boardgame"
	"github.com/jkomoros/boardgame/storage/filesystem/record"
	"github.com/jkomoros/boardgame/storage/memory"
	"github.com/yudai/gojsondiff"
	"github.com/yudai/gojsondiff/formatter"
	"io/ioutil"
	"path/filepath"
	"reflect"
	"strconv"
)

//Compare is the primary method in the package. It takes a game delegate and a
//filename denoting a record to compare against. delegate shiould be a fresh
//delegate not yet affiliated with a manager. It compares every version and
//move in the history (ignoring things that shouldn't be the same, like
//timestamps) and reports the first place they divrge. Any time it finds a
//move not proposed by AdminPlayerIndex it will propose that move. As long as
//your game uses state.Rand() for all randomness and is otherwise
//deterministic then everything should work.
func Compare(delegate boardgame.GameDelegate, recFilename string) error {

	manager, err := boardgame.NewGameManager(delegate, memory.NewStorageManager())

	if err != nil {
		return errors.New("Couldn't create new manager: " + err.Error())
	}

	rec, err := record.New(recFilename)

	if err != nil {
		return errors.New("Couldn't create record: " + err.Error())
	}

	return compare(manager, rec)

}

//CompareFolder is like Compare, except it will iterate through any file in
//recFolder that ends in .json. Errors if any of those files cannot be parsed
//into recs.
func CompareFolder(delegate boardgame.GameDelegate, recFolder string) error {
	manager, err := boardgame.NewGameManager(delegate, memory.NewStorageManager())

	if err != nil {
		return errors.New("Couldn't create new manager: " + err.Error())
	}

	infos, err := ioutil.ReadDir(recFolder)

	if err != nil {
		return errors.New("Couldn't read folder: " + err.Error())
	}

	for _, info := range infos {
		if info.IsDir() {
			continue
		}

		if filepath.Ext(info.Name()) != ".json" {
			continue
		}

		rec, err := record.New(filepath.Join(recFolder, info.Name()))

		if err != nil {
			return errors.New("File with name " + info.Name() + " couldn't be loaded into rec: " + err.Error())
		}

		if err := compare(manager, rec); err != nil {
			return errors.New("File named " + info.Name() + " had compare error: " + err.Error())
		}

	}

	return nil
}

func compare(manager *boardgame.GameManager, rec *record.Record) error {
	game, err := manager.Internals().RecreateGame(rec.Game())

	if err != nil {
		return errors.New("Couldn't create game: " + err.Error())
	}

	lastVerifiedVersion := 0

	for !game.Finished() {
		//Verify all new moves that have happened since the last time we
		//checked (often, fix-up moves).
		for lastVerifiedVersion < game.Version() {
			stateToCompare, err := rec.State(lastVerifiedVersion)

			if err != nil {
				return errors.New("Couldn't get " + strconv.Itoa(lastVerifiedVersion) + " state: " + err.Error())
			}

			//We used to just do
			//game.State(lastVerifiedVersion).StorageRecord(), but that
			//doesn't guarantee that it's the same as
			//manager.Storage().State() because it relies on state in
			//timerManager. This is unexpected and needs its own issue.
			storageRec, err := manager.Storage().State(game.Id(), lastVerifiedVersion)

			if err != nil {
				return errors.New("Couldn't get state storage rec from game: " + err.Error())
			}

			if err := compareJsonBlobs(storageRec, stateToCompare); err != nil {
				return errors.New("State " + strconv.Itoa(lastVerifiedVersion) + " compared differently: " + err.Error())
			}

			if lastVerifiedVersion > 0 {

				//Version 0 has no associated move

				recMove, err := rec.Move(lastVerifiedVersion)

				if err != nil {
					return errors.New("Couldn't get move " + strconv.Itoa(lastVerifiedVersion) + " from record")
				}

				moves := game.MoveRecords(lastVerifiedVersion)

				if len(moves) < 1 {
					return errors.New("Didn't fetch historical move records for " + strconv.Itoa(lastVerifiedVersion))
				}

				//Warning: records are modified by this method
				if err := compareMoveStorageRecords(moves[len(moves)-1], recMove); err != nil {
					return errors.New("Move " + strconv.Itoa(lastVerifiedVersion) + " compared differently: " + err.Error())
				}
			}

			lastVerifiedVersion++
		}

		nextMoveRec, err := rec.Move(lastVerifiedVersion + 1)

		if err != nil {
			//We'll assume that menas that's all of the moves there are to make.
			break
		}

		if nextMoveRec.Proposer < 0 {
			//We could be waiting for a timer to fire.
			//If there was a timer, try to force it to fire early.
			if manager.Internals().ForceNextTimer() {
				continue
			}

			//There wasn't a timer pending, so it's just an error
			return errors.New("At version " + strconv.Itoa(lastVerifiedVersion) + " the next player move to apply was not applied by a player")
		}

		nextMove, err := nextMoveRec.Inflate(game)

		if err != nil {
			return errors.New("Couldn't inflate move: " + err.Error())
		}

		if err := <-game.ProposeMove(nextMove, nextMoveRec.Proposer); err != nil {
			return errors.New("Couldn't propose next move in chain: " + err.Error())
		}

	}

	if game.Finished() != rec.Game().Finished {
		return errors.New("Game finished did not match rec")
	}

	if !reflect.DeepEqual(game.Winners(), rec.Game().Winners) {
		return errors.New("Game winners did not match")
	}

	return nil
}

var differ = gojsondiff.New()

var diffformatter = formatter.NewDeltaFormatter()

func compareJsonBlobs(one, two []byte) error {

	diff, err := differ.Compare(one, two)

	if err != nil {
		return errors.New("Couldn't diff: " + err.Error())
	}

	if diff.Modified() {

		str, err := diffformatter.Format(diff)

		if err != nil {
			return errors.New("Couldn't format diff: " + err.Error())
		}

		return errors.New("Diff: " + str)
	}

	return nil

}

//warning: modifies the records
func compareMoveStorageRecords(one, two *boardgame.MoveStorageRecord) error {

	if one == nil {
		return errors.New("One was nil")
	}

	if two == nil {
		return errors.New("Two was nil")
	}

	oneBlob := one.Blob
	twoBlob := two.Blob

	//Set the fields we know might differ to known values
	one.Blob = nil
	two.Blob = nil

	two.Timestamp = one.Timestamp

	if !reflect.DeepEqual(one, two) {
		return errors.New("Move storage records differed in base fields")
	}

	return compareJsonBlobs(oneBlob, twoBlob)

}
<commit_msg>Change it so that in golden when the test doesn't match we output a more easily human-readable output. Part of #648.<commit_after>/*

	golden is a package designed to make it possible to compare a game to a
	golden run for testing purposes. It takes a record saved in
	storage/filesystem format and compares it.

*/
package golden

import (
	"encoding/json"
	"errors"
	"github.com/jkomoros/boardgame"
	"github.com/jkomoros/boardgame/storage/filesystem/record"
	"github.com/jkomoros/boardgame/storage/memory"
	"github.com/yudai/gojsondiff"
	"github.com/yudai/gojsondiff/formatter"
	"io/ioutil"
	"path/filepath"
	"reflect"
	"strconv"
)

//Compare is the primary method in the package. It takes a game delegate and a
//filename denoting a record to compare against. delegate shiould be a fresh
//delegate not yet affiliated with a manager. It compares every version and
//move in the history (ignoring things that shouldn't be the same, like
//timestamps) and reports the first place they divrge. Any time it finds a
//move not proposed by AdminPlayerIndex it will propose that move. As long as
//your game uses state.Rand() for all randomness and is otherwise
//deterministic then everything should work.
func Compare(delegate boardgame.GameDelegate, recFilename string) error {

	manager, err := boardgame.NewGameManager(delegate, memory.NewStorageManager())

	if err != nil {
		return errors.New("Couldn't create new manager: " + err.Error())
	}

	rec, err := record.New(recFilename)

	if err != nil {
		return errors.New("Couldn't create record: " + err.Error())
	}

	return compare(manager, rec)

}

//CompareFolder is like Compare, except it will iterate through any file in
//recFolder that ends in .json. Errors if any of those files cannot be parsed
//into recs.
func CompareFolder(delegate boardgame.GameDelegate, recFolder string) error {
	manager, err := boardgame.NewGameManager(delegate, memory.NewStorageManager())

	if err != nil {
		return errors.New("Couldn't create new manager: " + err.Error())
	}

	infos, err := ioutil.ReadDir(recFolder)

	if err != nil {
		return errors.New("Couldn't read folder: " + err.Error())
	}

	for _, info := range infos {
		if info.IsDir() {
			continue
		}

		if filepath.Ext(info.Name()) != ".json" {
			continue
		}

		rec, err := record.New(filepath.Join(recFolder, info.Name()))

		if err != nil {
			return errors.New("File with name " + info.Name() + " couldn't be loaded into rec: " + err.Error())
		}

		if err := compare(manager, rec); err != nil {
			return errors.New("File named " + info.Name() + " had compare error: " + err.Error())
		}

	}

	return nil
}

func compare(manager *boardgame.GameManager, rec *record.Record) error {
	game, err := manager.Internals().RecreateGame(rec.Game())

	if err != nil {
		return errors.New("Couldn't create game: " + err.Error())
	}

	lastVerifiedVersion := 0

	for !game.Finished() {
		//Verify all new moves that have happened since the last time we
		//checked (often, fix-up moves).
		for lastVerifiedVersion < game.Version() {
			stateToCompare, err := rec.State(lastVerifiedVersion)

			if err != nil {
				return errors.New("Couldn't get " + strconv.Itoa(lastVerifiedVersion) + " state: " + err.Error())
			}

			//We used to just do
			//game.State(lastVerifiedVersion).StorageRecord(), but that
			//doesn't guarantee that it's the same as
			//manager.Storage().State() because it relies on state in
			//timerManager. This is unexpected and needs its own issue.
			storageRec, err := manager.Storage().State(game.Id(), lastVerifiedVersion)

			if err != nil {
				return errors.New("Couldn't get state storage rec from game: " + err.Error())
			}

			if err := compareJsonBlobs(storageRec, stateToCompare); err != nil {
				return errors.New("State " + strconv.Itoa(lastVerifiedVersion) + " compared differently: " + err.Error())
			}

			if lastVerifiedVersion > 0 {

				//Version 0 has no associated move

				recMove, err := rec.Move(lastVerifiedVersion)

				if err != nil {
					return errors.New("Couldn't get move " + strconv.Itoa(lastVerifiedVersion) + " from record")
				}

				moves := game.MoveRecords(lastVerifiedVersion)

				if len(moves) < 1 {
					return errors.New("Didn't fetch historical move records for " + strconv.Itoa(lastVerifiedVersion))
				}

				//Warning: records are modified by this method
				if err := compareMoveStorageRecords(moves[len(moves)-1], recMove); err != nil {
					return errors.New("Move " + strconv.Itoa(lastVerifiedVersion) + " compared differently: " + err.Error())
				}
			}

			lastVerifiedVersion++
		}

		nextMoveRec, err := rec.Move(lastVerifiedVersion + 1)

		if err != nil {
			//We'll assume that menas that's all of the moves there are to make.
			break
		}

		if nextMoveRec.Proposer < 0 {
			//We could be waiting for a timer to fire.
			//If there was a timer, try to force it to fire early.
			if manager.Internals().ForceNextTimer() {
				continue
			}

			//There wasn't a timer pending, so it's just an error
			return errors.New("At version " + strconv.Itoa(lastVerifiedVersion) + " the next player move to apply was not applied by a player")
		}

		nextMove, err := nextMoveRec.Inflate(game)

		if err != nil {
			return errors.New("Couldn't inflate move: " + err.Error())
		}

		if err := <-game.ProposeMove(nextMove, nextMoveRec.Proposer); err != nil {
			return errors.New("Couldn't propose next move in chain: " + err.Error())
		}

	}

	if game.Finished() != rec.Game().Finished {
		return errors.New("Game finished did not match rec")
	}

	if !reflect.DeepEqual(game.Winners(), rec.Game().Winners) {
		return errors.New("Game winners did not match")
	}

	return nil
}

var differ = gojsondiff.New()

func compareJsonBlobs(one, two []byte) error {

	diff, err := differ.Compare(one, two)

	if err != nil {
		return errors.New("Couldn't diff: " + err.Error())
	}

	if diff.Modified() {

		var oneJson map[string]interface{}

		if err := json.Unmarshal(one, &oneJson); err != nil {
			return errors.New("Couldn't unmarshal left")
		}

		diffformatter := formatter.NewAsciiFormatter(oneJson, formatter.AsciiFormatterConfig{
			Coloring: true,
		})

		str, err := diffformatter.Format(diff)

		if err != nil {
			return errors.New("Couldn't format diff: " + err.Error())
		}

		return errors.New("Diff: " + str)
	}

	return nil

}

//warning: modifies the records
func compareMoveStorageRecords(one, two *boardgame.MoveStorageRecord) error {

	if one == nil {
		return errors.New("One was nil")
	}

	if two == nil {
		return errors.New("Two was nil")
	}

	oneBlob := one.Blob
	twoBlob := two.Blob

	//Set the fields we know might differ to known values
	one.Blob = nil
	two.Blob = nil

	two.Timestamp = one.Timestamp

	if !reflect.DeepEqual(one, two) {
		return errors.New("Move storage records differed in base fields")
	}

	return compareJsonBlobs(oneBlob, twoBlob)

}
<|endoftext|>


<commit_before>// +build !confonly

package dispatcher

//go:generate errorgen

import (
	"context"
	"strings"
	"sync"
	"time"

	"v2ray.com/core"
	"v2ray.com/core/common"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/log"
	"v2ray.com/core/common/net"
	"v2ray.com/core/common/log"
	"v2ray.com/core/common/protocol"
	"v2ray.com/core/common/session"
	"v2ray.com/core/features/outbound"
	"v2ray.com/core/features/policy"
	"v2ray.com/core/features/routing"
	"v2ray.com/core/features/stats"
	"v2ray.com/core/transport"
	"v2ray.com/core/transport/pipe"
)

var (
	errSniffingTimeout = newError("timeout on sniffing")
)

type cachedReader struct {
	sync.Mutex
	reader *pipe.Reader
	cache  buf.MultiBuffer
}

func (r *cachedReader) Cache(b *buf.Buffer) {
	mb, _ := r.reader.ReadMultiBufferTimeout(time.Millisecond * 100)
	r.Lock()
	if !mb.IsEmpty() {
		r.cache, _ = buf.MergeMulti(r.cache, mb)
	}
	b.Clear()
	rawBytes := b.Extend(buf.Size)
	n := r.cache.Copy(rawBytes)
	b.Resize(0, int32(n))
	r.Unlock()
}

func (r *cachedReader) readInternal() buf.MultiBuffer {
	r.Lock()
	defer r.Unlock()

	if r.cache != nil && !r.cache.IsEmpty() {
		mb := r.cache
		r.cache = nil
		return mb
	}

	return nil
}

func (r *cachedReader) ReadMultiBuffer() (buf.MultiBuffer, error) {
	mb := r.readInternal()
	if mb != nil {
		return mb, nil
	}

	return r.reader.ReadMultiBuffer()
}

func (r *cachedReader) ReadMultiBufferTimeout(timeout time.Duration) (buf.MultiBuffer, error) {
	mb := r.readInternal()
	if mb != nil {
		return mb, nil
	}

	return r.reader.ReadMultiBufferTimeout(timeout)
}

func (r *cachedReader) Interrupt() {
	r.Lock()
	if r.cache != nil {
		r.cache = buf.ReleaseMulti(r.cache)
	}
	r.Unlock()
	r.reader.Interrupt()
}

// DefaultDispatcher is a default implementation of Dispatcher.
type DefaultDispatcher struct {
	ohm    outbound.Manager
	router routing.Router
	policy policy.Manager
	stats  stats.Manager
}

func init() {
	common.Must(common.RegisterConfig((*Config)(nil), func(ctx context.Context, config interface{}) (interface{}, error) {
		d := new(DefaultDispatcher)
		if err := core.RequireFeatures(ctx, func(om outbound.Manager, router routing.Router, pm policy.Manager, sm stats.Manager) error {
			return d.Init(config.(*Config), om, router, pm, sm)
		}); err != nil {
			return nil, err
		}
		return d, nil
	}))
}

// Init initializes DefaultDispatcher.
func (d *DefaultDispatcher) Init(config *Config, om outbound.Manager, router routing.Router, pm policy.Manager, sm stats.Manager) error {
	d.ohm = om
	d.router = router
	d.policy = pm
	d.stats = sm
	return nil
}

// Type implements common.HasType.
func (*DefaultDispatcher) Type() interface{} {
	return routing.DispatcherType()
}

// Start implements common.Runnable.
func (*DefaultDispatcher) Start() error {
	return nil
}

// Close implements common.Closable.
func (*DefaultDispatcher) Close() error { return nil }

func (d *DefaultDispatcher) getLink(ctx context.Context) (*transport.Link, *transport.Link) {
	opt := pipe.OptionsFromContext(ctx)
	uplinkReader, uplinkWriter := pipe.New(opt...)
	downlinkReader, downlinkWriter := pipe.New(opt...)

	inboundLink := &transport.Link{
		Reader: downlinkReader,
		Writer: uplinkWriter,
	}

	outboundLink := &transport.Link{
		Reader: uplinkReader,
		Writer: downlinkWriter,
	}

	sessionInbound := session.InboundFromContext(ctx)
	var user *protocol.MemoryUser
	if sessionInbound != nil {
		user = sessionInbound.User
	}

	if user != nil && len(user.Email) > 0 {
		p := d.policy.ForLevel(user.Level)
		if p.Stats.UserUplink {
			name := "user>>>" + user.Email + ">>>traffic>>>uplink"
			if c, _ := stats.GetOrRegisterCounter(d.stats, name); c != nil {
				inboundLink.Writer = &SizeStatWriter{
					Counter: c,
					Writer:  inboundLink.Writer,
				}
			}
		}
		if p.Stats.UserDownlink {
			name := "user>>>" + user.Email + ">>>traffic>>>downlink"
			if c, _ := stats.GetOrRegisterCounter(d.stats, name); c != nil {
				outboundLink.Writer = &SizeStatWriter{
					Counter: c,
					Writer:  outboundLink.Writer,
				}
			}
		}
	}

	return inboundLink, outboundLink
}

func shouldOverride(result SniffResult, domainOverride []string) bool {
	for _, p := range domainOverride {
		if strings.HasPrefix(result.Protocol(), p) {
			return true
		}
	}
	return false
}

// Dispatch implements routing.Dispatcher.
func (d *DefaultDispatcher) Dispatch(ctx context.Context, destination net.Destination) (*transport.Link, error) {
	if !destination.IsValid() {
		panic("Dispatcher: Invalid destination.")
	}
	ob := &session.Outbound{
		Target: destination,
	}
	ctx = session.ContextWithOutbound(ctx, ob)

	inbound, outbound := d.getLink(ctx)
	content := session.ContentFromContext(ctx)
	if content == nil {
		content = new(session.Content)
		ctx = session.ContextWithContent(ctx, content)
	}
	sniffingRequest := content.SniffingRequest
	if destination.Network != net.Network_TCP || !sniffingRequest.Enabled {
		go d.routedDispatch(ctx, outbound, destination)
	} else {
		go func() {
			cReader := &cachedReader{
				reader: outbound.Reader.(*pipe.Reader),
			}
			outbound.Reader = cReader
			result, err := sniffer(ctx, cReader)
			if err == nil {
				content.Protocol = result.Protocol()
			}
			if err == nil && shouldOverride(result, sniffingRequest.OverrideDestinationForProtocol) {
				domain := result.Domain()
				newError("sniffed domain: ", domain).WriteToLog(session.ExportIDToError(ctx))
				destination.Address = net.ParseAddress(domain)
				ob.Target = destination
			}
			d.routedDispatch(ctx, outbound, destination)
		}()
	}
	return inbound, nil
}

func sniffer(ctx context.Context, cReader *cachedReader) (SniffResult, error) {
	payload := buf.New()
	defer payload.Release()

	sniffer := NewSniffer()
	totalAttempt := 0
	for {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
			totalAttempt++
			if totalAttempt > 2 {
				return nil, errSniffingTimeout
			}

			cReader.Cache(payload)
			if !payload.IsEmpty() {
				result, err := sniffer.Sniff(payload.Bytes())
				if err != common.ErrNoClue {
					return result, err
				}
			}
			if payload.IsFull() {
				return nil, errUnknownContent
			}
		}
	}
}

func (d *DefaultDispatcher) routedDispatch(ctx context.Context, link *transport.Link, destination net.Destination) {
	var handler outbound.Handler

	skipRoutePick := false
	if content := session.ContentFromContext(ctx); content != nil {
		skipRoutePick = content.SkipRoutePick
	}

	if d.router != nil && !skipRoutePick {
		if tag, err := d.router.PickRoute(ctx); err == nil {
			if h := d.ohm.GetHandler(tag); h != nil {
				newError("taking detour [", tag, "] for [", destination, "]").WriteToLog(session.ExportIDToError(ctx))
				handler = h
			} else {
				newError("non existing tag: ", tag).AtWarning().WriteToLog(session.ExportIDToError(ctx))
			}
		} else {
			newError("default route for ", destination).WriteToLog(session.ExportIDToError(ctx))
		}
	}

	if handler == nil {
		handler = d.ohm.GetDefaultHandler()
	}

	if handler == nil {
		newError("default outbound handler not exist").WriteToLog(session.ExportIDToError(ctx))
		common.Close(link.Writer)
		common.Interrupt(link.Reader)
		return
	}

	accessMessage := log.AccessMessageFromContext(ctx)
	if accessMessage != nil {
		accessMessage.Detour = "[" + handler.Tag() + "]"
		log.Record(accessMessage)
	}

	handler.Dispatch(ctx, link)
}
<commit_msg>Fix merge error<commit_after>// +build !confonly

package dispatcher

//go:generate errorgen

import (
	"context"
	"strings"
	"sync"
	"time"

	"v2ray.com/core"
	"v2ray.com/core/common"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/log"
	"v2ray.com/core/common/net"
	"v2ray.com/core/common/protocol"
	"v2ray.com/core/common/session"
	"v2ray.com/core/features/outbound"
	"v2ray.com/core/features/policy"
	"v2ray.com/core/features/routing"
	"v2ray.com/core/features/stats"
	"v2ray.com/core/transport"
	"v2ray.com/core/transport/pipe"
)

var (
	errSniffingTimeout = newError("timeout on sniffing")
)

type cachedReader struct {
	sync.Mutex
	reader *pipe.Reader
	cache  buf.MultiBuffer
}

func (r *cachedReader) Cache(b *buf.Buffer) {
	mb, _ := r.reader.ReadMultiBufferTimeout(time.Millisecond * 100)
	r.Lock()
	if !mb.IsEmpty() {
		r.cache, _ = buf.MergeMulti(r.cache, mb)
	}
	b.Clear()
	rawBytes := b.Extend(buf.Size)
	n := r.cache.Copy(rawBytes)
	b.Resize(0, int32(n))
	r.Unlock()
}

func (r *cachedReader) readInternal() buf.MultiBuffer {
	r.Lock()
	defer r.Unlock()

	if r.cache != nil && !r.cache.IsEmpty() {
		mb := r.cache
		r.cache = nil
		return mb
	}

	return nil
}

func (r *cachedReader) ReadMultiBuffer() (buf.MultiBuffer, error) {
	mb := r.readInternal()
	if mb != nil {
		return mb, nil
	}

	return r.reader.ReadMultiBuffer()
}

func (r *cachedReader) ReadMultiBufferTimeout(timeout time.Duration) (buf.MultiBuffer, error) {
	mb := r.readInternal()
	if mb != nil {
		return mb, nil
	}

	return r.reader.ReadMultiBufferTimeout(timeout)
}

func (r *cachedReader) Interrupt() {
	r.Lock()
	if r.cache != nil {
		r.cache = buf.ReleaseMulti(r.cache)
	}
	r.Unlock()
	r.reader.Interrupt()
}

// DefaultDispatcher is a default implementation of Dispatcher.
type DefaultDispatcher struct {
	ohm    outbound.Manager
	router routing.Router
	policy policy.Manager
	stats  stats.Manager
}

func init() {
	common.Must(common.RegisterConfig((*Config)(nil), func(ctx context.Context, config interface{}) (interface{}, error) {
		d := new(DefaultDispatcher)
		if err := core.RequireFeatures(ctx, func(om outbound.Manager, router routing.Router, pm policy.Manager, sm stats.Manager) error {
			return d.Init(config.(*Config), om, router, pm, sm)
		}); err != nil {
			return nil, err
		}
		return d, nil
	}))
}

// Init initializes DefaultDispatcher.
func (d *DefaultDispatcher) Init(config *Config, om outbound.Manager, router routing.Router, pm policy.Manager, sm stats.Manager) error {
	d.ohm = om
	d.router = router
	d.policy = pm
	d.stats = sm
	return nil
}

// Type implements common.HasType.
func (*DefaultDispatcher) Type() interface{} {
	return routing.DispatcherType()
}

// Start implements common.Runnable.
func (*DefaultDispatcher) Start() error {
	return nil
}

// Close implements common.Closable.
func (*DefaultDispatcher) Close() error { return nil }

func (d *DefaultDispatcher) getLink(ctx context.Context) (*transport.Link, *transport.Link) {
	opt := pipe.OptionsFromContext(ctx)
	uplinkReader, uplinkWriter := pipe.New(opt...)
	downlinkReader, downlinkWriter := pipe.New(opt...)

	inboundLink := &transport.Link{
		Reader: downlinkReader,
		Writer: uplinkWriter,
	}

	outboundLink := &transport.Link{
		Reader: uplinkReader,
		Writer: downlinkWriter,
	}

	sessionInbound := session.InboundFromContext(ctx)
	var user *protocol.MemoryUser
	if sessionInbound != nil {
		user = sessionInbound.User
	}

	if user != nil && len(user.Email) > 0 {
		p := d.policy.ForLevel(user.Level)
		if p.Stats.UserUplink {
			name := "user>>>" + user.Email + ">>>traffic>>>uplink"
			if c, _ := stats.GetOrRegisterCounter(d.stats, name); c != nil {
				inboundLink.Writer = &SizeStatWriter{
					Counter: c,
					Writer:  inboundLink.Writer,
				}
			}
		}
		if p.Stats.UserDownlink {
			name := "user>>>" + user.Email + ">>>traffic>>>downlink"
			if c, _ := stats.GetOrRegisterCounter(d.stats, name); c != nil {
				outboundLink.Writer = &SizeStatWriter{
					Counter: c,
					Writer:  outboundLink.Writer,
				}
			}
		}
	}

	return inboundLink, outboundLink
}

func shouldOverride(result SniffResult, domainOverride []string) bool {
	for _, p := range domainOverride {
		if strings.HasPrefix(result.Protocol(), p) {
			return true
		}
	}
	return false
}

// Dispatch implements routing.Dispatcher.
func (d *DefaultDispatcher) Dispatch(ctx context.Context, destination net.Destination) (*transport.Link, error) {
	if !destination.IsValid() {
		panic("Dispatcher: Invalid destination.")
	}
	ob := &session.Outbound{
		Target: destination,
	}
	ctx = session.ContextWithOutbound(ctx, ob)

	inbound, outbound := d.getLink(ctx)
	content := session.ContentFromContext(ctx)
	if content == nil {
		content = new(session.Content)
		ctx = session.ContextWithContent(ctx, content)
	}
	sniffingRequest := content.SniffingRequest
	if destination.Network != net.Network_TCP || !sniffingRequest.Enabled {
		go d.routedDispatch(ctx, outbound, destination)
	} else {
		go func() {
			cReader := &cachedReader{
				reader: outbound.Reader.(*pipe.Reader),
			}
			outbound.Reader = cReader
			result, err := sniffer(ctx, cReader)
			if err == nil {
				content.Protocol = result.Protocol()
			}
			if err == nil && shouldOverride(result, sniffingRequest.OverrideDestinationForProtocol) {
				domain := result.Domain()
				newError("sniffed domain: ", domain).WriteToLog(session.ExportIDToError(ctx))
				destination.Address = net.ParseAddress(domain)
				ob.Target = destination
			}
			d.routedDispatch(ctx, outbound, destination)
		}()
	}
	return inbound, nil
}

func sniffer(ctx context.Context, cReader *cachedReader) (SniffResult, error) {
	payload := buf.New()
	defer payload.Release()

	sniffer := NewSniffer()
	totalAttempt := 0
	for {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
			totalAttempt++
			if totalAttempt > 2 {
				return nil, errSniffingTimeout
			}

			cReader.Cache(payload)
			if !payload.IsEmpty() {
				result, err := sniffer.Sniff(payload.Bytes())
				if err != common.ErrNoClue {
					return result, err
				}
			}
			if payload.IsFull() {
				return nil, errUnknownContent
			}
		}
	}
}

func (d *DefaultDispatcher) routedDispatch(ctx context.Context, link *transport.Link, destination net.Destination) {
	var handler outbound.Handler

	skipRoutePick := false
	if content := session.ContentFromContext(ctx); content != nil {
		skipRoutePick = content.SkipRoutePick
	}

	if d.router != nil && !skipRoutePick {
		if tag, err := d.router.PickRoute(ctx); err == nil {
			if h := d.ohm.GetHandler(tag); h != nil {
				newError("taking detour [", tag, "] for [", destination, "]").WriteToLog(session.ExportIDToError(ctx))
				handler = h
			} else {
				newError("non existing tag: ", tag).AtWarning().WriteToLog(session.ExportIDToError(ctx))
			}
		} else {
			newError("default route for ", destination).WriteToLog(session.ExportIDToError(ctx))
		}
	}

	if handler == nil {
		handler = d.ohm.GetDefaultHandler()
	}

	if handler == nil {
		newError("default outbound handler not exist").WriteToLog(session.ExportIDToError(ctx))
		common.Close(link.Writer)
		common.Interrupt(link.Reader)
		return
	}

	accessMessage := log.AccessMessageFromContext(ctx)
	if accessMessage != nil {
		accessMessage.Detour = "[" + handler.Tag() + "]"
		log.Record(accessMessage)
	}

	handler.Dispatch(ctx, link)
}
<|endoftext|>


<commit_before>package docker

import (
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"sort"
	"strings"

	log "github.com/Sirupsen/logrus"
	flag "github.com/docker/docker/pkg/mflag"
	"github.com/docker/docker/runconfig"
	shlex "github.com/flynn/go-shlex"
	dockerClient "github.com/fsouza/go-dockerclient"
	"github.com/rancherio/os/config"
	"github.com/rancherio/os/util"
)

const (
	LABEL = "label"
	HASH  = "io.rancher.os.hash"
	ID    = "io.rancher.os.id"
)

type Container struct {
	Err          error
	Name         string
	remove       bool
	detach       bool
	Config       *runconfig.Config
	HostConfig   *runconfig.HostConfig
	dockerHost   string
	Container    *dockerClient.Container
	ContainerCfg *config.ContainerConfig
}

type ByCreated []dockerClient.APIContainers

func (c ByCreated) Len() int           { return len(c) }
func (c ByCreated) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }
func (c ByCreated) Less(i, j int) bool { return c[j].Created < c[i].Created }

func getHash(containerCfg *config.ContainerConfig) (string, error) {
	hash := sha1.New()
	w := util.NewErrorWriter(hash)

	w.Write([]byte(containerCfg.Id))
	w.Write([]byte(containerCfg.Cmd))

	if w.Err != nil {
		return "", w.Err
	}

	return hex.EncodeToString(hash.Sum([]byte{})), nil
}

func StartAndWait(dockerHost string, containerCfg *config.ContainerConfig) error {
	container := NewContainer(dockerHost, containerCfg).start(true)
	return container.Err
}

func NewContainer(dockerHost string, containerCfg *config.ContainerConfig) *Container {
	c := &Container{
		dockerHost:   dockerHost,
		ContainerCfg: containerCfg,
	}
	return c.Parse()
}

func (c *Container) returnErr(err error) *Container {
	c.Err = err
	return c
}

func getByLabel(client *dockerClient.Client, key, value string) (*dockerClient.APIContainers, error) {
	containers, err := client.ListContainers(dockerClient.ListContainersOptions{
		All: true,
		Filters: map[string][]string{
			LABEL: []string{fmt.Sprintf("%s=%s", key, value)},
		},
	})

	if err != nil {
		return nil, err
	}

	if len(containers) == 0 {
		return nil, nil
	}

	sort.Sort(ByCreated(containers))
	return &containers[0], nil
}

func (c *Container) Lookup() *Container {
	c.Parse()

	if c.Err != nil || (c.Container != nil && c.Container.HostConfig != nil) {
		return c
	}

	hash, err := getHash(c.ContainerCfg)
	if err != nil {
		return c.returnErr(err)
	}

	client, err := NewClient(c.dockerHost)
	if err != nil {
		return c.returnErr(err)
	}

	containers, err := client.ListContainers(dockerClient.ListContainersOptions{
		All: true,
		Filters: map[string][]string{
			LABEL: []string{fmt.Sprintf("%s=%s", HASH, hash)},
		},
	})
	if err != nil {
		return c.returnErr(err)
	}

	if len(containers) == 0 {
		return c
	}

	c.Container, c.Err = inspect(client, containers[0].ID)

	return c
}

func inspect(client *dockerClient.Client, id string) (*dockerClient.Container, error) {
	c, err := client.InspectContainer(id)
	if err != nil {
		return nil, err
	}

	if strings.HasPrefix(c.Name, "/") {
		c.Name = c.Name[1:]
	}

	return c, err
}

func (c *Container) Exists() bool {
	c.Lookup()
	return c.Container != nil
}

func (c *Container) Reset() *Container {
	c.Config = nil
	c.HostConfig = nil
	c.Container = nil
	c.Err = nil

	return c
}

func (c *Container) Parse() *Container {
	if c.Config != nil || c.Err != nil {
		return c
	}

	flags := flag.NewFlagSet("run", flag.ExitOnError)

	flRemove := flags.Bool([]string{"#rm", "-rm"}, false, "")
	flDetach := flags.Bool([]string{"d", "-detach"}, false, "")
	flName := flags.String([]string{"#name", "-name"}, "", "")

	args, err := shlex.Split(c.ContainerCfg.Cmd)
	if err != nil {
		return c.returnErr(err)
	}

	log.Debugf("Parsing [%s]", strings.Join(args, ","))
	c.Config, c.HostConfig, _, c.Err = runconfig.Parse(flags, args)

	c.Name = *flName
	c.detach = *flDetach
	c.remove = *flRemove

	if c.ContainerCfg.Id == "" {
		c.ContainerCfg.Id = c.Name
	}

	return c
}

func (c *Container) Start() *Container {
	return c.start(false)
}

func (c *Container) StartAndWait() *Container {
	return c.start(true)
}

func (c *Container) Stage() *Container {
	c.Parse()

	if c.Err != nil {
		return c
	}

	client, err := NewClient(c.dockerHost)
	if err != nil {
		c.Err = err
		return c
	}

	_, err = client.InspectImage(c.Config.Image)
	if err == dockerClient.ErrNoSuchImage {
		c.Err = client.PullImage(dockerClient.PullImageOptions{
			Repository:   c.Config.Image,
			OutputStream: os.Stdout,
		}, dockerClient.AuthConfiguration{})
	} else if err != nil {
		c.Err = err
	}

	return c
}

func (c *Container) Delete() *Container {
	c.Parse()
	c.Stage()
	c.Lookup()

	if c.Err != nil {
		return c
	}

	if !c.Exists() {
		return c
	}

	client, err := NewClient(c.dockerHost)
	if err != nil {
		return c.returnErr(err)
	}

	err = client.RemoveContainer(dockerClient.RemoveContainerOptions{
		ID:    c.Container.ID,
		Force: true,
	})
	if err != nil {
		return c.returnErr(err)
	}

	return c
}

func (c *Container) renameCurrent(client *dockerClient.Client) error {
	if c.Name == "" {
		return nil
	}

	if c.Name == c.Container.Name {
		return nil
	}

	err := client.RenameContainer(c.Container.ID, c.Name)
	if err != nil {
		return err
	}

	c.Container, err = inspect(client, c.Container.ID)
	return err
}

func (c *Container) renameOld(client *dockerClient.Client, opts *dockerClient.CreateContainerOptions) error {
	if len(opts.Name) == 0 {
		return nil
	}

	existing, err := inspect(client, opts.Name)
	if _, ok := err.(dockerClient.NoSuchContainer); ok {
		return nil
	}

	if err != nil {
		return nil
	}

	if c.Container != nil && existing.ID == c.Container.ID {
		return nil
	}

	if label, ok := existing.Config.Labels[HASH]; ok {
		newName := fmt.Sprintf("%s-%s", existing.Name, label)

		if existing.State.Running {
			err := client.StopContainer(existing.ID, 2)
			if err != nil {
				return err
			}

			_, err = client.WaitContainer(existing.ID)
			if err != nil {
				return err
			}
		}

		log.Debugf("Renaming %s to %s", existing.Name, newName)
		return client.RenameContainer(existing.ID, newName)
	} else {
		//TODO: do something with containers with no hash
		return errors.New("Existing container doesn't have a hash")
	}
}

func (c *Container) getCreateOpts(client *dockerClient.Client) (*dockerClient.CreateContainerOptions, error) {
	bytes, err := json.Marshal(c)
	if err != nil {
		return nil, err
	}

	var opts dockerClient.CreateContainerOptions

	err = json.Unmarshal(bytes, &opts)
	if err != nil {
		return nil, err
	}

	if opts.Config.Labels == nil {
		opts.Config.Labels = make(map[string]string)
	}

	hash, err := getHash(c.ContainerCfg)
	if err != nil {
		return nil, err
	}

	opts.Config.Labels[HASH] = hash
	opts.Config.Labels[ID] = c.ContainerCfg.Id

	return &opts, nil
}

func appendVolumesFrom(client *dockerClient.Client, containerCfg *config.ContainerConfig, opts *dockerClient.CreateContainerOptions) error {
	if !containerCfg.MigrateVolumes {
		return nil
	}

	container, err := getByLabel(client, ID, containerCfg.Id)
	if err != nil || container == nil {
		return err
	}

	if opts.HostConfig.VolumesFrom == nil {
		opts.HostConfig.VolumesFrom = []string{container.ID}
	} else {
		opts.HostConfig.VolumesFrom = append(opts.HostConfig.VolumesFrom, container.ID)
	}

	return nil
}

func (c *Container) start(wait bool) *Container {
	c.Lookup()
	c.Stage()

	if c.Err != nil {
		return c
	}

	client, err := NewClient(c.dockerHost)
	if err != nil {
		return c.returnErr(err)
	}

	container := c.Container
	created := false

	opts, err := c.getCreateOpts(client)
	if err != nil {
		return c.returnErr(err)
	}

	if c.Exists() && c.remove {
		log.Debugf("Deleting container %s", c.Container.ID)
		c.Delete()

		if c.Err != nil {
			return c
		}

		c.Reset().Lookup()
		if c.Err != nil {
			return c
		}
	}

	if !c.Exists() {
		err = c.renameOld(client, opts)
		if err != nil {
			return c.returnErr(err)
		}

		err := appendVolumesFrom(client, c.ContainerCfg, opts)
		if err != nil {
			return c.returnErr(err)
		}

		container, err = client.CreateContainer(*opts)
		created = true
		if err != nil {
			return c.returnErr(err)
		}
	}

	c.Container = container

	hostConfig := c.Container.HostConfig
	if created {
		hostConfig = opts.HostConfig
	}

	if !c.Container.State.Running {
		if !created {
			err = c.renameOld(client, opts)
			if err != nil {
				return c.returnErr(err)
			}
		}

		err = c.renameCurrent(client)
		if err != nil {
			return c.returnErr(err)
		}

		err = client.StartContainer(c.Container.ID, hostConfig)
		if err != nil {
			return c.returnErr(err)
		}
	}

	if !c.detach && wait {
		_, c.Err = client.WaitContainer(c.Container.ID)
		return c
	}

	return c
}
<commit_msg>Handle system containers with no hash<commit_after>package docker

import (
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"os"
	"sort"
	"strings"

	log "github.com/Sirupsen/logrus"
	flag "github.com/docker/docker/pkg/mflag"
	"github.com/docker/docker/runconfig"
	shlex "github.com/flynn/go-shlex"
	dockerClient "github.com/fsouza/go-dockerclient"
	"github.com/rancherio/os/config"
	"github.com/rancherio/os/util"
)

const (
	LABEL = "label"
	HASH  = "io.rancher.os.hash"
	ID    = "io.rancher.os.id"
)

type Container struct {
	Err          error
	Name         string
	remove       bool
	detach       bool
	Config       *runconfig.Config
	HostConfig   *runconfig.HostConfig
	dockerHost   string
	Container    *dockerClient.Container
	ContainerCfg *config.ContainerConfig
}

type ByCreated []dockerClient.APIContainers

func (c ByCreated) Len() int           { return len(c) }
func (c ByCreated) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }
func (c ByCreated) Less(i, j int) bool { return c[j].Created < c[i].Created }

func getHash(containerCfg *config.ContainerConfig) (string, error) {
	hash := sha1.New()
	w := util.NewErrorWriter(hash)

	w.Write([]byte(containerCfg.Id))
	w.Write([]byte(containerCfg.Cmd))

	if w.Err != nil {
		return "", w.Err
	}

	return hex.EncodeToString(hash.Sum([]byte{})), nil
}

func StartAndWait(dockerHost string, containerCfg *config.ContainerConfig) error {
	container := NewContainer(dockerHost, containerCfg).start(true)
	return container.Err
}

func NewContainer(dockerHost string, containerCfg *config.ContainerConfig) *Container {
	c := &Container{
		dockerHost:   dockerHost,
		ContainerCfg: containerCfg,
	}
	return c.Parse()
}

func (c *Container) returnErr(err error) *Container {
	c.Err = err
	return c
}

func getByLabel(client *dockerClient.Client, key, value string) (*dockerClient.APIContainers, error) {
	containers, err := client.ListContainers(dockerClient.ListContainersOptions{
		All: true,
		Filters: map[string][]string{
			LABEL: []string{fmt.Sprintf("%s=%s", key, value)},
		},
	})

	if err != nil {
		return nil, err
	}

	if len(containers) == 0 {
		return nil, nil
	}

	sort.Sort(ByCreated(containers))
	return &containers[0], nil
}

func (c *Container) Lookup() *Container {
	c.Parse()

	if c.Err != nil || (c.Container != nil && c.Container.HostConfig != nil) {
		return c
	}

	hash, err := getHash(c.ContainerCfg)
	if err != nil {
		return c.returnErr(err)
	}

	client, err := NewClient(c.dockerHost)
	if err != nil {
		return c.returnErr(err)
	}

	containers, err := client.ListContainers(dockerClient.ListContainersOptions{
		All: true,
		Filters: map[string][]string{
			LABEL: []string{fmt.Sprintf("%s=%s", HASH, hash)},
		},
	})
	if err != nil {
		return c.returnErr(err)
	}

	if len(containers) == 0 {
		return c
	}

	c.Container, c.Err = inspect(client, containers[0].ID)

	return c
}

func inspect(client *dockerClient.Client, id string) (*dockerClient.Container, error) {
	c, err := client.InspectContainer(id)
	if err != nil {
		return nil, err
	}

	if strings.HasPrefix(c.Name, "/") {
		c.Name = c.Name[1:]
	}

	return c, err
}

func (c *Container) Exists() bool {
	c.Lookup()
	return c.Container != nil
}

func (c *Container) Reset() *Container {
	c.Config = nil
	c.HostConfig = nil
	c.Container = nil
	c.Err = nil

	return c
}

func (c *Container) Parse() *Container {
	if c.Config != nil || c.Err != nil {
		return c
	}

	flags := flag.NewFlagSet("run", flag.ExitOnError)

	flRemove := flags.Bool([]string{"#rm", "-rm"}, false, "")
	flDetach := flags.Bool([]string{"d", "-detach"}, false, "")
	flName := flags.String([]string{"#name", "-name"}, "", "")

	args, err := shlex.Split(c.ContainerCfg.Cmd)
	if err != nil {
		return c.returnErr(err)
	}

	log.Debugf("Parsing [%s]", strings.Join(args, ","))
	c.Config, c.HostConfig, _, c.Err = runconfig.Parse(flags, args)

	c.Name = *flName
	c.detach = *flDetach
	c.remove = *flRemove

	if c.ContainerCfg.Id == "" {
		c.ContainerCfg.Id = c.Name
	}

	return c
}

func (c *Container) Start() *Container {
	return c.start(false)
}

func (c *Container) StartAndWait() *Container {
	return c.start(true)
}

func (c *Container) Stage() *Container {
	c.Parse()

	if c.Err != nil {
		return c
	}

	client, err := NewClient(c.dockerHost)
	if err != nil {
		c.Err = err
		return c
	}

	_, err = client.InspectImage(c.Config.Image)
	if err == dockerClient.ErrNoSuchImage {
		c.Err = client.PullImage(dockerClient.PullImageOptions{
			Repository:   c.Config.Image,
			OutputStream: os.Stdout,
		}, dockerClient.AuthConfiguration{})
	} else if err != nil {
		c.Err = err
	}

	return c
}

func (c *Container) Delete() *Container {
	c.Parse()
	c.Stage()
	c.Lookup()

	if c.Err != nil {
		return c
	}

	if !c.Exists() {
		return c
	}

	client, err := NewClient(c.dockerHost)
	if err != nil {
		return c.returnErr(err)
	}

	err = client.RemoveContainer(dockerClient.RemoveContainerOptions{
		ID:    c.Container.ID,
		Force: true,
	})
	if err != nil {
		return c.returnErr(err)
	}

	return c
}

func (c *Container) renameCurrent(client *dockerClient.Client) error {
	if c.Name == "" {
		return nil
	}

	if c.Name == c.Container.Name {
		return nil
	}

	err := client.RenameContainer(c.Container.ID, c.Name)
	if err != nil {
		return err
	}

	c.Container, err = inspect(client, c.Container.ID)
	return err
}

func (c *Container) renameOld(client *dockerClient.Client, opts *dockerClient.CreateContainerOptions) error {
	if len(opts.Name) == 0 {
		return nil
	}

	existing, err := inspect(client, opts.Name)
	if _, ok := err.(dockerClient.NoSuchContainer); ok {
		return nil
	}

	if err != nil {
		return nil
	}

	if c.Container != nil && existing.ID == c.Container.ID {
		return nil
	}

	var newName string
	if label, ok := existing.Config.Labels[HASH]; ok {
		newName = fmt.Sprintf("%s-%s", existing.Name, label)
	} else {
		newName = fmt.Sprintf("%s-unknown-%s", existing.Name, util.RandSeq(12))
	}

	if existing.State.Running {
		err := client.StopContainer(existing.ID, 2)
		if err != nil {
			return err
		}

		_, err = client.WaitContainer(existing.ID)
		if err != nil {
			return err
		}
	}

	log.Debugf("Renaming %s to %s", existing.Name, newName)
	return client.RenameContainer(existing.ID, newName)
}

func (c *Container) getCreateOpts(client *dockerClient.Client) (*dockerClient.CreateContainerOptions, error) {
	bytes, err := json.Marshal(c)
	if err != nil {
		return nil, err
	}

	var opts dockerClient.CreateContainerOptions

	err = json.Unmarshal(bytes, &opts)
	if err != nil {
		return nil, err
	}

	if opts.Config.Labels == nil {
		opts.Config.Labels = make(map[string]string)
	}

	hash, err := getHash(c.ContainerCfg)
	if err != nil {
		return nil, err
	}

	opts.Config.Labels[HASH] = hash
	opts.Config.Labels[ID] = c.ContainerCfg.Id

	return &opts, nil
}

func appendVolumesFrom(client *dockerClient.Client, containerCfg *config.ContainerConfig, opts *dockerClient.CreateContainerOptions) error {
	if !containerCfg.MigrateVolumes {
		return nil
	}

	container, err := getByLabel(client, ID, containerCfg.Id)
	if err != nil || container == nil {
		return err
	}

	if opts.HostConfig.VolumesFrom == nil {
		opts.HostConfig.VolumesFrom = []string{container.ID}
	} else {
		opts.HostConfig.VolumesFrom = append(opts.HostConfig.VolumesFrom, container.ID)
	}

	return nil
}

func (c *Container) start(wait bool) *Container {
	c.Lookup()
	c.Stage()

	if c.Err != nil {
		return c
	}

	client, err := NewClient(c.dockerHost)
	if err != nil {
		return c.returnErr(err)
	}

	container := c.Container
	created := false

	opts, err := c.getCreateOpts(client)
	if err != nil {
		return c.returnErr(err)
	}

	if c.Exists() && c.remove {
		log.Debugf("Deleting container %s", c.Container.ID)
		c.Delete()

		if c.Err != nil {
			return c
		}

		c.Reset().Lookup()
		if c.Err != nil {
			return c
		}
	}

	if !c.Exists() {
		err = c.renameOld(client, opts)
		if err != nil {
			return c.returnErr(err)
		}

		err := appendVolumesFrom(client, c.ContainerCfg, opts)
		if err != nil {
			return c.returnErr(err)
		}

		container, err = client.CreateContainer(*opts)
		created = true
		if err != nil {
			return c.returnErr(err)
		}
	}

	c.Container = container

	hostConfig := c.Container.HostConfig
	if created {
		hostConfig = opts.HostConfig
	}

	if !c.Container.State.Running {
		if !created {
			err = c.renameOld(client, opts)
			if err != nil {
				return c.returnErr(err)
			}
		}

		err = c.renameCurrent(client)
		if err != nil {
			return c.returnErr(err)
		}

		err = client.StartContainer(c.Container.ID, hostConfig)
		if err != nil {
			return c.returnErr(err)
		}
	}

	if !c.detach && wait {
		_, c.Err = client.WaitContainer(c.Container.ID)
		return c
	}

	return c
}
<|endoftext|>