<commit_before>package destiny

import (
	"encoding/json"
	"fmt"
	"net/http"
	"sync"
	"time"
)

// Platform Definitions
const (
	PlatformXBL = 1
	PlatformPSN = 2
)

// ErrorCode Values (response envelope)
const (
	errorSuccess = 1
)

// The response envelope wrapping all api repsonses
type responseEnvelope struct {
	ErrorCode       int
	ErrorStatus     string
	Message         string
	ThrottleSeconds int64
	Response        *json.RawMessage
	// MessageData  interface{}
}

// Client provides an interface to BungieNet.Platform.DestinyServices
// see: https://www.bungie.net/platform/destiny/help/
type Client struct {
	apiKey    string
	client    *http.Client
	baseURL   string
	userAgent string
	wait      *time.Time
	lock      sync.Mutex
}

// AccountSummary returns data from the /{membershipType}/Account/{destinyMembershipId}/Summary/
// endpoint
func (c *Client) AccountSummary(platform int, id string, into interface{}) error {
	if err := c.get(fmt.Sprintf("/%d/Account/%s/Summary/", platform, id), into); err != nil {
		return err
	}
	return nil
}

func (c *Client) get(uri string, into interface{}) error {
	c.lock.Lock()
	if c.wait != nil {
		time.Sleep(time.Now().Sub(*c.wait))
		c.wait = nil
	}
	// Prepare our request
	req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s", c.baseURL, uri), nil)
	c.lock.Unlock()
	if err != nil {
		return err
	}
	req.Header.Add("X-API-KEY", c.apiKey)
	req.Header.Add("User-Agent", c.userAgent)

	// Make our request
	rsp, err := c.client.Do(req)
	if err != nil {
		return err
	}

	// Prepare to parse our response
	dec := json.NewDecoder(rsp.Body)
	defer rsp.Body.Close()
	var envelope responseEnvelope

	// Unmarshal the JSON
	if err := dec.Decode(&envelope); err != nil {
		return err
	}

	// Check for failure
	if envelope.ErrorCode != errorSuccess {
		return fmt.Errorf("API returned error code %d: %s %s", envelope.ErrorCode, envelope.ErrorStatus, envelope.Message)
	}

	if envelope.ThrottleSeconds > 0 {
		t := time.Now().Add(time.Duration(envelope.ThrottleSeconds) * time.Second)
		c.wait = &t
	}

	return json.Unmarshal(*envelope.Response, &into)
}

// New returns a new client with which you can make API calls to
// Bungies Destiny API
func New(apiKey, purpose string) *Client {
	return &Client{
		apiKey:    apiKey,
		client:    &http.Client{},
		baseURL:   "http://www.bungie.net/Platform/Destiny/",
		userAgent: fmt.Sprintf("Go (golang; net/http; github.com/apokalyptik/fof/lib/destiny; +%s)", purpose),
	}
}
<commit_msg>lib updates<commit_after>package destiny

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"sync"
	"time"
)

// Platform Definitions
const (
	PlatformXBL = 1
	PlatformPSN = 2
)

// ErrorCode Values (response envelope)
const (
	errorSuccess = 1
)

// The response envelope wrapping all api repsonses
type responseEnvelope struct {
	ErrorCode       int
	ErrorStatus     string
	Message         string
	ThrottleSeconds int64
	Response        *json.RawMessage
	// MessageData  interface{}
}

// Client provides an interface to BungieNet.Platform.DestinyServices
// see: https://www.bungie.net/platform/destiny/help/
type Client struct {
	apiKey    string
	client    *http.Client
	baseURL   string
	userAgent string
	wait      *time.Time
	lock      sync.Mutex
}

type Request struct {
	*url.URL
	url.Values
	c *Client
}

func (r *Request) ToURL() string {
	return fmt.Sprintf("%s%s?%s", r.c.baseURL, r.URL.String()[1:], r.Values.Encode())
}

func (r *Request) Into(into interface{}) error {
	return r.c.get(r.URL.String()[1:]+"?"+r.Values.Encode(), into)
}

func (c *Client) AccountSummary(platform int, id string) (*Request, error) {
	u, e := url.ParseRequestURI(fmt.Sprintf("/%d/Account/%s/Summary/", platform, id))
	return &Request{URL: u, c: c, Values: url.Values{}}, e
}

func (c *Client) ActivityHistory(platform int, id string, cid string) (*Request, error) {
	u, e := url.ParseRequestURI(fmt.Sprintf("/Stats/ActivityHistory/%d/%s/%s/", platform, id, cid))
	return &Request{URL: u, c: c, Values: url.Values{}}, e
}

func (c *Client) get(uri string, into interface{}) error {
	c.lock.Lock()
	if c.wait != nil {
		time.Sleep(time.Now().Sub(*c.wait))
		c.wait = nil
	}
	// Prepare our request
	//log.Println(fmt.Sprintf("%s%s", c.baseURL, uri))
	req, err := http.NewRequest("GET", fmt.Sprintf("%s%s", c.baseURL, uri), nil)
	c.lock.Unlock()
	if err != nil {
		return err
	}
	req.Header.Add("X-API-KEY", c.apiKey)
	req.Header.Add("User-Agent", c.userAgent)

	// Make our request
	rsp, err := c.client.Do(req)
	if err != nil {
		return err
	}

	// Prepare to parse our response
	dec := json.NewDecoder(rsp.Body)
	defer rsp.Body.Close()
	var envelope responseEnvelope

	// Unmarshal the JSON
	if err := dec.Decode(&envelope); err != nil {
		return err
	}

	// Check for failure
	if envelope.ErrorCode != errorSuccess {
		return fmt.Errorf("API returned error code %d: %s %s", envelope.ErrorCode, envelope.ErrorStatus, envelope.Message)
	}

	if envelope.ThrottleSeconds > 0 {
		t := time.Now().Add(time.Duration(envelope.ThrottleSeconds) * time.Second)
		c.wait = &t
	}

	return json.Unmarshal(*envelope.Response, &into)
}

// New returns a new client with which you can make API calls to
// Bungies Destiny API
func New(apiKey, purpose string) *Client {
	return &Client{
		apiKey:    apiKey,
		client:    &http.Client{},
		baseURL:   "http://www.bungie.net/Platform/Destiny/",
		userAgent: fmt.Sprintf("Go (golang; net/http; github.com/apokalyptik/fof/lib/destiny; +%s)", purpose),
	}
}
<|endoftext|>


<commit_before>package elasticsearch

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/packetzoom/logslammer/buffer"
	"github.com/packetzoom/logslammer/output"
	"github.com/paulbellamy/ratecounter"
	"gopkg.in/olivere/elastic.v2"
)

const (
	defaultHost        = "127.0.0.1"
	defaultIndexPrefix = "logstash"
	esFlushInterval    = 5
	esMaxConns         = 20
	esRecvBuffer       = 100
	esSendBuffer       = 100
)

type Indexer struct {
	bulkService       *elastic.BulkService
	indexPrefix       string
	indexType         string
	RateCounter       *ratecounter.RateCounter
	lastDisplayUpdate time.Time
}

type Config struct {
	Hosts       []string `json:"hosts"`
	IndexPrefix string   `json:"index"`
	IndexType   string   `json:"indexType"`
	Timeout     int      `json:"timeout"`
}

type ESServer struct {
	config Config
	host   string
	hosts  []string
	b      buffer.Sender
	term   chan bool
}

func init() {
	output.Register("elasticsearch", &ESServer{
		host: fmt.Sprintf("%s:%d", defaultHost, time.Now().Unix()),
		term: make(chan bool, 1),
	})
}

func indexName(idx string) string {
	if len(idx) == 0 {
		idx = defaultIndexPrefix
	}

	return fmt.Sprintf("%s-%s", idx, time.Now().Format("2006.01.02"))
}

func (i *Indexer) flush() error {
	numEvents := i.bulkService.NumberOfActions()

	if numEvents > 0 {
		if time.Now().Sub(i.lastDisplayUpdate) >= time.Duration(1*time.Second) {
			log.Printf("Flushing %d event(s) to Elasticsearch, current rate: %d/s", numEvents, i.RateCounter.Rate())
			i.lastDisplayUpdate = time.Now()
		}

		_, err := i.bulkService.Do()

		if err != nil {
			log.Printf("Unable to flush events: %s", err)
		}

		return err
	}

	return nil
}

func (i *Indexer) index(ev *buffer.Event) error {
	doc := *ev.Text
	idx := indexName(i.indexPrefix)
	typ := i.indexType

	request := elastic.NewBulkIndexRequest().Index(idx).Type(typ).Doc(doc)
	i.bulkService.Add(request)
	i.RateCounter.Incr(1)

	numEvents := i.bulkService.NumberOfActions()

	if numEvents < esSendBuffer {
		return nil
	}

	return i.flush()
}

func (e *ESServer) Init(config json.RawMessage, b buffer.Sender) error {
	var esConfig *Config
	if err := json.Unmarshal(config, &esConfig); err != nil {
		return fmt.Errorf("Error parsing elasticsearch config: %v", err)
	}

	e.config = *esConfig
	e.hosts = esConfig.Hosts
	e.b = b

	return nil
}

func readInputChannel(idx *Indexer, receiveChan chan *buffer.Event) {
	// Drain the channel only if we have room
	if idx.bulkService.NumberOfActions() < esSendBuffer {
		select {
		case ev := <-receiveChan:
			idx.index(ev)
		}
	} else {
		log.Printf("Internal Elasticsearch buffer is full, waiting")
		time.Sleep(1 * time.Second)
	}
}

func (es *ESServer) Start() error {
	var client *elastic.Client
	var err error

	for {
		httpClient := http.DefaultClient
		timeout := 60 * time.Second

		if es.config.Timeout > 0 {
			timeout = time.Duration(es.config.Timeout) * time.Second
		}

		log.Println("Setting HTTP timeout to", timeout)
		httpClient.Timeout = timeout
		client, err = elastic.NewClient(elastic.SetURL(es.hosts...),
			elastic.SetHttpClient(httpClient))

		if err != nil {
			log.Printf("Error starting Elasticsearch: %s, will retry", err)
			time.Sleep(2 * time.Second)
			continue
		}

		break
	}

	log.Printf("Connected to Elasticsearch")

	service := elastic.NewBulkService(client)

	// Add the client as a subscriber
	receiveChan := make(chan *buffer.Event, esRecvBuffer)
	es.b.AddSubscriber(es.host, receiveChan)
	defer es.b.DelSubscriber(es.host)

	rateCounter := ratecounter.NewRateCounter(1 * time.Second)

	// Create indexer
	idx := &Indexer{service, es.config.IndexPrefix, es.config.IndexType, rateCounter, time.Now()}

	// Loop events and publish to elasticsearch
	tick := time.NewTicker(time.Duration(esFlushInterval) * time.Second)

	for {
		readInputChannel(idx, receiveChan)

		if len(tick.C) > 0 || len(es.term) > 0 {
			select {
			case <-tick.C:
				idx.flush()
			case <-es.term:
				tick.Stop()
				log.Println("Elasticsearch received term signal")
				break
			}
		}
	}

	return nil
}

func (es *ESServer) Stop() error {
	es.term <- true
	return nil
}
<commit_msg>Enable GZIP compression and error/info logging<commit_after>package elasticsearch

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/packetzoom/logslammer/buffer"
	"github.com/packetzoom/logslammer/output"
	"github.com/paulbellamy/ratecounter"
	"gopkg.in/olivere/elastic.v2"
)

const (
	defaultHost        = "127.0.0.1"
	defaultIndexPrefix = "logstash"
	esFlushInterval    = 5
	esMaxConns         = 20
	esRecvBuffer       = 100
	esSendBuffer       = 100
)

type Indexer struct {
	bulkService       *elastic.BulkService
	indexPrefix       string
	indexType         string
	RateCounter       *ratecounter.RateCounter
	lastDisplayUpdate time.Time
}

type Config struct {
	Hosts       []string `json:"hosts"`
	IndexPrefix string   `json:"index"`
	IndexType   string   `json:"indexType"`
	Timeout     int      `json:"timeout"`
}

type ESServer struct {
	config Config
	host   string
	hosts  []string
	b      buffer.Sender
	term   chan bool
}

func init() {
	output.Register("elasticsearch", &ESServer{
		host: fmt.Sprintf("%s:%d", defaultHost, time.Now().Unix()),
		term: make(chan bool, 1),
	})
}

func indexName(idx string) string {
	if len(idx) == 0 {
		idx = defaultIndexPrefix
	}

	return fmt.Sprintf("%s-%s", idx, time.Now().Format("2006.01.02"))
}

func (i *Indexer) flush() error {
	numEvents := i.bulkService.NumberOfActions()

	if numEvents > 0 {
		if time.Now().Sub(i.lastDisplayUpdate) >= time.Duration(1*time.Second) {
			log.Printf("Flushing %d event(s) to Elasticsearch, current rate: %d/s", numEvents, i.RateCounter.Rate())
			i.lastDisplayUpdate = time.Now()
		}

		_, err := i.bulkService.Do()

		if err != nil {
			log.Printf("Unable to flush events: %s", err)
		}

		return err
	}

	return nil
}

func (i *Indexer) index(ev *buffer.Event) error {
	doc := *ev.Text
	idx := indexName(i.indexPrefix)
	typ := i.indexType

	request := elastic.NewBulkIndexRequest().Index(idx).Type(typ).Doc(doc)
	i.bulkService.Add(request)
	i.RateCounter.Incr(1)

	numEvents := i.bulkService.NumberOfActions()

	if numEvents < esSendBuffer {
		return nil
	}

	return i.flush()
}

func (e *ESServer) Init(config json.RawMessage, b buffer.Sender) error {
	var esConfig *Config
	if err := json.Unmarshal(config, &esConfig); err != nil {
		return fmt.Errorf("Error parsing elasticsearch config: %v", err)
	}

	e.config = *esConfig
	e.hosts = esConfig.Hosts
	e.b = b

	return nil
}

func readInputChannel(idx *Indexer, receiveChan chan *buffer.Event) {
	// Drain the channel only if we have room
	if idx.bulkService.NumberOfActions() < esSendBuffer {
		select {
		case ev := <-receiveChan:
			idx.index(ev)
		}
	} else {
		log.Printf("Internal Elasticsearch buffer is full, waiting")
		time.Sleep(1 * time.Second)
	}
}

func (es *ESServer) Start() error {
	var client *elastic.Client
	var err error

	for {
		httpClient := http.DefaultClient
		timeout := 60 * time.Second

		if es.config.Timeout > 0 {
			timeout = time.Duration(es.config.Timeout) * time.Second
		}

		log.Println("Setting HTTP timeout to", timeout)
		httpClient.Timeout = timeout
		client, err = elastic.NewClient(elastic.SetURL(es.hosts...),
			elastic.SetHttpClient(httpClient),
			elastic.SetGzip(true),
			elastic.SetInfoLog(log.New(os.Stdout, "", log.LstdFlags)),
			elastic.SetErrorLog(log.New(os.Stderr, "", log.LstdFlags)))

		if err != nil {
			log.Printf("Error starting Elasticsearch: %s, will retry", err)
			time.Sleep(2 * time.Second)
			continue
		}

		break
	}

	log.Printf("Connected to Elasticsearch")

	service := elastic.NewBulkService(client)

	// Add the client as a subscriber
	receiveChan := make(chan *buffer.Event, esRecvBuffer)
	es.b.AddSubscriber(es.host, receiveChan)
	defer es.b.DelSubscriber(es.host)

	rateCounter := ratecounter.NewRateCounter(1 * time.Second)

	// Create indexer
	idx := &Indexer{service, es.config.IndexPrefix, es.config.IndexType, rateCounter, time.Now()}

	// Loop events and publish to elasticsearch
	tick := time.NewTicker(time.Duration(esFlushInterval) * time.Second)

	for {
		readInputChannel(idx, receiveChan)

		if len(tick.C) > 0 || len(es.term) > 0 {
			select {
			case <-tick.C:
				idx.flush()
			case <-es.term:
				tick.Stop()
				log.Println("Elasticsearch received term signal")
				break
			}
		}
	}

	return nil
}

func (es *ESServer) Stop() error {
	es.term <- true
	return nil
}
<|endoftext|>


<commit_before>package auth

import (
	"net/http"
	"strings"
)

// APIVersion represents a version of an API including its
// type and version number.
type APIVersion struct {
	// Type refers to the name of a specific API specification
	// such as "registry"
	Type string

	// Version is the vesion of the API specification implemented,
	// This may omit the revision number and only include
	// the major and minor version, such as "2.0"
	Version string
}

// String returns the string formatted API Version
func (v APIVersion) String() string {
	return v.Type + "/" + v.Version
}

// APIVersions gets the API versions out of an HTTP response using the provided
// version header as the key for the HTTP header.
func APIVersions(resp *http.Response, versionHeader string) []APIVersion {
	versions := []APIVersion{}
	if versionHeader != "" {
		for _, supportedVersions := range resp.Header[http.CanonicalHeaderKey(versionHeader)] {
			for _, version := range strings.Fields(supportedVersions) {
				versions = append(versions, ParseAPIVersion(version))
			}
		}
	}
	return versions
}

// ParseAPIVersion parses an API version string into an APIVersion
// Format (Expected, not enforced):
// API version string = <API type> '/' <API version>
// API type = [a-z][a-z0-9]*
// API version = [0-9]+(\.[0-9]+)?
// TODO(dmcgowan): Enforce format, add error condition, remove unknown type
func ParseAPIVersion(versionStr string) APIVersion {
	idx := strings.IndexRune(versionStr, '/')
	if idx == -1 {
		return APIVersion{
			Type:    "unknown",
			Version: versionStr,
		}
	}
	return APIVersion{
		Type:    strings.ToLower(versionStr[:idx]),
		Version: versionStr[idx+1:],
	}
}
<commit_msg>Fix typo in Version doc<commit_after>package auth

import (
	"net/http"
	"strings"
)

// APIVersion represents a version of an API including its
// type and version number.
type APIVersion struct {
	// Type refers to the name of a specific API specification
	// such as "registry"
	Type string

	// Version is the version of the API specification implemented,
	// This may omit the revision number and only include
	// the major and minor version, such as "2.0"
	Version string
}

// String returns the string formatted API Version
func (v APIVersion) String() string {
	return v.Type + "/" + v.Version
}

// APIVersions gets the API versions out of an HTTP response using the provided
// version header as the key for the HTTP header.
func APIVersions(resp *http.Response, versionHeader string) []APIVersion {
	versions := []APIVersion{}
	if versionHeader != "" {
		for _, supportedVersions := range resp.Header[http.CanonicalHeaderKey(versionHeader)] {
			for _, version := range strings.Fields(supportedVersions) {
				versions = append(versions, ParseAPIVersion(version))
			}
		}
	}
	return versions
}

// ParseAPIVersion parses an API version string into an APIVersion
// Format (Expected, not enforced):
// API version string = <API type> '/' <API version>
// API type = [a-z][a-z0-9]*
// API version = [0-9]+(\.[0-9]+)?
// TODO(dmcgowan): Enforce format, add error condition, remove unknown type
func ParseAPIVersion(versionStr string) APIVersion {
	idx := strings.IndexRune(versionStr, '/')
	if idx == -1 {
		return APIVersion{
			Type:    "unknown",
			Version: versionStr,
		}
	}
	return APIVersion{
		Type:    strings.ToLower(versionStr[:idx]),
		Version: versionStr[idx+1:],
	}
}
<|endoftext|>


<commit_before>package hamt32_test

import (
	"log"
	"math/rand"
	"testing"
	"time"

	"github.com/lleo/go-hamt-functional/hamt32"
	"github.com/lleo/go-hamt/stringkey"
	"github.com/lleo/stringutil"
)

func TestBuildHamt32(t *testing.T) {
	log.Println("TestBuildHamt32:")
	var h = hamt32.Hamt{}

	var added bool
	for _, kv := range hugeKvs {
		h, added = h.Put(kv.Key, kv.Val)
		if !added {
			t.Fatalf("failed to h.Put(%s, %v)", kv.Key, kv.Val)
		}
	}
	//log.Println(h.LongString(""))

	var val interface{}
	var removed bool
	for _, kv := range hugeKvs {
		h, val, removed = h.Del(kv.Key)
		if !removed {
			t.Fatalf("failed to h.Del(%s)", kv.Key)
		}
		if val != kv.Val {
			t.Fatalf("val,%d != kv.Val,%d", val, kv.Val)
		}
	}

	log.Printf("h = %s", h.LongString(""))

	if !h.IsEmpty() {
		t.Fatalf("!h.IsEmpty()")
	}
}

func BenchmarkHamt32Get(b *testing.B) {
	log.Printf("BenchmarkHamt32Get: b.N=%d", b.N)

	for i := 0; i < b.N; i++ {
		var j = int(rand.Int31()) % numHugeKvs
		var key = hugeKvs[j].Key
		var val = hugeKvs[j].Val
		var v, found = LookupHamt32.Get(key)
		if !found {
			b.Fatalf("H.Get(%s) not found", key)
		}
		if v != val {
			b.Fatalf("val,%v != hugeKvs[%d].val,%v", v, j, val)
		}
	}
}

func BenchmarkHamt32Put(b *testing.B) {
	log.Printf("BenchmarkHamt32Put: b.N=%d", b.N)

	var h = hamt32.Hamt{}
	var s = "aaa"
	for i := 0; i < b.N; i++ {
		key := stringkey.New(s)
		val := i
		h, _ = h.Put(key, val)
		s = stringutil.DigitalInc(s)
	}
}

func BenchmarkHamt32Del(b *testing.B) {
	log.Printf("BenchmarkHamt32Del: b.N=%d", b.N)

	// We rebuild the DeleteHamt32 datastructure because this Benchmark will probably be
	// rereun with different b.N values to get a better/more-accurate benchmark.

	StartTime["BenchmarkHamt32Del:rebuildDeleteHamt32"] = time.Now()

	//rebuildDeleteHamt32(hugeKvs)
	var h = DeleteHamt32 //its functional and I am an idiot

	RunTime["BenchmarkHamt32Del:rebuildDeleteHamt"] = time.Since(StartTime["BenchmarkHamt32Del:rebuildDeleteHamt32"])

	b.ResetTimer()

	StartTime["run BenchmarkHamt32Del"] = time.Now()
	for i := 0; i < b.N; i++ {
		kv := hugeKvs[i]
		key := kv.Key
		val := kv.Val

		var v interface{}
		var deleted bool
		h, v, deleted = h.Del(key)
		if !deleted {
			b.Fatalf("failed to find and delet key=%s", key)
		}
		if v != val {
			b.Fatalf("deleted key=%s but the value found was wrong v=%d, expected val=%d", key, v, val)
		}
	}

	if h.IsEmpty() {
		b.Fatal("DeleteHamt32.IsEmpty() => true; hence this wasn't a valid benchmark")
	}

	RunTime["run BenchmarkHamt32Del"] = time.Since(StartTime["run BenchmarkHamt32Del"])
}
<commit_msg>* Renamed numHugeKvs to numKvs * Renamed hugeKvs to KVS * Renamed LookupHamt32 to TestHamt32 * Deleted DeleteHamt32 * commented out calls to h.LongString("")<commit_after>package hamt32_test

import (
	"log"
	"math/rand"
	"testing"
	"time"

	"github.com/lleo/go-hamt-functional/hamt32"
	"github.com/lleo/go-hamt/stringkey"
	"github.com/lleo/stringutil"
)

func TestBuildHamt32(t *testing.T) {
	log.Println("TestBuildHamt32:")
	var h = hamt32.Hamt{}

	var added bool
	for _, kv := range KVS {
		h, added = h.Put(kv.Key, kv.Val)
		if !added {
			t.Fatalf("failed to h.Put(%s, %v)", kv.Key, kv.Val)
		}
	}

	//log.Println(h.LongString(""))

	var val interface{}
	var removed bool
	for _, kv := range KVS {
		//log.Printf("kv.Key = %s", kv.Key)
		h, val, removed = h.Del(kv.Key)
		if !removed {
			t.Fatalf("failed to h.Del(%s)", kv.Key)
		}
		if val != kv.Val {
			t.Fatalf("val,%d != kv.Val,%d", val, kv.Val)
		}
	}

	//log.Printf("h = %s", h.LongString(""))

	if !h.IsEmpty() {
		t.Fatalf("!h.IsEmpty()")
	}
}

func BenchmarkHamt32Get(b *testing.B) {
	log.Printf("BenchmarkHamt32Get: b.N=%d", b.N)

	for i := 0; i < b.N; i++ {
		//var j = int(rand.Int31()) % numKvs
		var j = rand.Int() % numKvs
		var key = KVS[j].Key
		var val = KVS[j].Val
		var v, found = TestHamt32.Get(key)
		if !found {
			b.Fatalf("H.Get(%s) not found", key)
		}
		if v != val {
			b.Fatalf("val,%v != KVS[%d].val,%v", v, j, val)
		}
	}
}

func BenchmarkHamt32Put(b *testing.B) {
	log.Printf("BenchmarkHamt32Put: b.N=%d", b.N)

	var h = hamt32.Hamt{}
	var s = "aaa"
	for i := 0; i < b.N; i++ {
		key := stringkey.New(s)
		val := i
		h, _ = h.Put(key, val)
		s = stringutil.DigitalInc(s)
	}
}

func BenchmarkHamt32Del(b *testing.B) {
	log.Printf("BenchmarkHamt32Del: b.N=%d", b.N)

	var h = TestHamt32

	var randomizedKVS = genRandomizedKvs(KVS)

	b.ResetTimer()

	StartTime["run BenchmarkHamt32Del"] = time.Now()
	for i := 0; i < b.N; i++ {
		kv := randomizedKVS[i%numKvs]
		key := kv.Key
		val := kv.Val

		var v interface{}
		var deleted bool
		h, v, deleted = h.Del(key)
		if !deleted {
			b.Fatalf("failed to find and delet key=%s", key)
		}
		if v != val {
			b.Fatalf("deleted key=%s but the value found was wrong v=%d, expected val=%d", key, v, val)
		}
	}

	if h.IsEmpty() {
		b.Fatal("TestHamt32.IsEmpty() => true; hence this wasn't a valid benchmark")
	}

	RunTime["run BenchmarkHamt32Del"] = time.Since(StartTime["run BenchmarkHamt32Del"])
}
<|endoftext|>


<commit_before>package v1

import (
	"encoding/json"
	"net/http"
	"strconv"
	"time"

	"github.com/freeusd/solebtc/models"
	"github.com/freeusd/solebtc/utils"
	"github.com/gin-gonic/gin"
)

// GetReward randomly gives users reward
func GetReward(
	getUserByID dependencyGetUserByID,
	getLatestTotalReward dependencyGetLatestTotalReward,
	getSystemConfig dependencyGetSystemConfig,
	getRewardRatesByType dependencyGetRewardRatesByType,
	createRewardIncome dependencyCreateRewardIncome,
	insertIncome dependencyInsertIncome,
	broadcast dependencyBroadcast,
) gin.HandlerFunc {
	return func(c *gin.Context) {
		authToken := c.MustGet("auth_token").(models.AuthToken)
		now := time.Now()

		// get user
		user, err := getUserByID(authToken.UserID)
		if err != nil {
			c.AbortWithError(http.StatusInternalServerError, err)
			return
		}

		// check last rewarded time
		if user.RewardedAt.Add(time.Second * time.Duration(user.RewardInterval)).After(now) {
			c.AbortWithStatus(statusCodeTooManyRequests)
			return
		}

		// get random reward
		latestTotalReward := getLatestTotalReward()
		rewardRateType := models.RewardRateTypeLess
		if latestTotalReward.IsSameDay(now) && latestTotalReward.Total > getSystemConfig().TotalRewardThreshold {
			rewardRateType = models.RewardRateTypeMore
		}
		rewardRates := getRewardRatesByType(rewardRateType)
		reward := utils.RandomReward(rewardRates)
		rewardReferer := reward * getSystemConfig().RefererRewardRate

		// create income reward
		income := models.Income{
			UserID:        user.ID,
			RefererID:     user.RefererID,
			Type:          models.IncomeTypeReward,
			Income:        reward,
			RefererIncome: rewardReferer,
		}
		if err := createRewardIncome(income, now); err != nil {
			c.AbortWithError(http.StatusInternalServerError, err)
			return
		}

		// cache delta income
		deltaIncome := struct {
			Address string    `json:"address"`
			Amount  float64   `json:"amount"`
			Type    string    `json:"type"`
			Time    time.Time `json:"time"`
		}{user.Address, reward, "reward", now}
		insertIncome(deltaIncome)

		// broadcast delta income to all clients
		msg, _ := json.Marshal(models.WebsocketMessage{DeltaIncome: deltaIncome})
		broadcast(msg)

		c.JSON(http.StatusOK, income)
	}
}

// RewardList returns user's reward list as response
func RewardList(
	getRewardIncomesSince dependencyGetRewardIncomesSince,
	getRewardIncomesUntil dependencyGetRewardIncomesUntil,
) gin.HandlerFunc {
	return func(c *gin.Context) {
		authToken := c.MustGet("auth_token").(models.AuthToken)

		// response
		getRewards(c, authToken.UserID, getRewardIncomesSince, getRewardIncomesUntil)
	}
}

// RefereeRewardList returns user's referee's reward list as response
func RefereeRewardList(
	getUserByID dependencyGetUserByID,
	getRewardIncomesSince dependencyGetRewardIncomesSince,
	getRewardIncomesUntil dependencyGetRewardIncomesUntil,
) gin.HandlerFunc {
	return func(c *gin.Context) {
		authToken := c.MustGet("auth_token").(models.AuthToken)

		// check if user is referer of :referee_id
		refereeID, _ := strconv.ParseInt(c.Param("referee_id"), 10, 64)
		referee, _ := getUserByID(refereeID)
		if referee.HasReferer() && referee.RefererID != authToken.UserID {
			c.AbortWithStatus(http.StatusForbidden)
			return
		}

		// response
		getRewards(c, refereeID, getRewardIncomesSince, getRewardIncomesUntil)
	}
}

// common get rewards logic
func getRewards(
	c *gin.Context,
	userID int64,
	getRewardIncomesSince dependencyGetRewardIncomesSince,
	getRewardIncomesUntil dependencyGetRewardIncomesUntil,
) {
	// parse pagination args
	isSince, separator, limit, err := parsePagination(c)
	if err != nil {
		c.AbortWithStatus(http.StatusBadRequest)
		return
	}

	// get result according to args
	t := time.Unix(separator, 0)
	result := []models.Income{}
	if isSince {
		result, err = getRewardIncomesSince(userID, t, limit)
	} else {
		result, err = getRewardIncomesUntil(userID, t, limit)
	}

	// response with result or error
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	c.JSON(http.StatusOK, result)
}
<commit_msg>Fix naming issue<commit_after>package v1

import (
	"encoding/json"
	"net/http"
	"strconv"
	"time"

	"github.com/freeusd/solebtc/models"
	"github.com/freeusd/solebtc/utils"
	"github.com/gin-gonic/gin"
)

// GetReward randomly gives users reward
func GetReward(
	getUserByID dependencyGetUserByID,
	getLatestTotalReward dependencyGetLatestTotalReward,
	getSystemConfig dependencyGetSystemConfig,
	getRewardRatesByType dependencyGetRewardRatesByType,
	createRewardIncome dependencyCreateRewardIncome,
	cacheIncome dependencyInsertIncome,
	broadcast dependencyBroadcast,
) gin.HandlerFunc {
	return func(c *gin.Context) {
		authToken := c.MustGet("auth_token").(models.AuthToken)
		now := time.Now()

		// get user
		user, err := getUserByID(authToken.UserID)
		if err != nil {
			c.AbortWithError(http.StatusInternalServerError, err)
			return
		}

		// check last rewarded time
		if user.RewardedAt.Add(time.Second * time.Duration(user.RewardInterval)).After(now) {
			c.AbortWithStatus(statusCodeTooManyRequests)
			return
		}

		// get random reward
		latestTotalReward := getLatestTotalReward()
		rewardRateType := models.RewardRateTypeLess
		if latestTotalReward.IsSameDay(now) && latestTotalReward.Total > getSystemConfig().TotalRewardThreshold {
			rewardRateType = models.RewardRateTypeMore
		}
		rewardRates := getRewardRatesByType(rewardRateType)
		reward := utils.RandomReward(rewardRates)
		rewardReferer := reward * getSystemConfig().RefererRewardRate

		// create income reward
		income := models.Income{
			UserID:        user.ID,
			RefererID:     user.RefererID,
			Type:          models.IncomeTypeReward,
			Income:        reward,
			RefererIncome: rewardReferer,
		}
		if err := createRewardIncome(income, now); err != nil {
			c.AbortWithError(http.StatusInternalServerError, err)
			return
		}

		// cache delta income
		deltaIncome := struct {
			Address string    `json:"address"`
			Amount  float64   `json:"amount"`
			Type    string    `json:"type"`
			Time    time.Time `json:"time"`
		}{user.Address, reward, "reward", now}
		cacheIncome(deltaIncome)

		// broadcast delta income to all clients
		msg, _ := json.Marshal(models.WebsocketMessage{DeltaIncome: deltaIncome})
		broadcast(msg)

		c.JSON(http.StatusOK, income)
	}
}

// RewardList returns user's reward list as response
func RewardList(
	getRewardIncomesSince dependencyGetRewardIncomesSince,
	getRewardIncomesUntil dependencyGetRewardIncomesUntil,
) gin.HandlerFunc {
	return func(c *gin.Context) {
		authToken := c.MustGet("auth_token").(models.AuthToken)

		// response
		getRewards(c, authToken.UserID, getRewardIncomesSince, getRewardIncomesUntil)
	}
}

// RefereeRewardList returns user's referee's reward list as response
func RefereeRewardList(
	getUserByID dependencyGetUserByID,
	getRewardIncomesSince dependencyGetRewardIncomesSince,
	getRewardIncomesUntil dependencyGetRewardIncomesUntil,
) gin.HandlerFunc {
	return func(c *gin.Context) {
		authToken := c.MustGet("auth_token").(models.AuthToken)

		// check if user is referer of :referee_id
		refereeID, _ := strconv.ParseInt(c.Param("referee_id"), 10, 64)
		referee, _ := getUserByID(refereeID)
		if referee.HasReferer() && referee.RefererID != authToken.UserID {
			c.AbortWithStatus(http.StatusForbidden)
			return
		}

		// response
		getRewards(c, refereeID, getRewardIncomesSince, getRewardIncomesUntil)
	}
}

// common get rewards logic
func getRewards(
	c *gin.Context,
	userID int64,
	getRewardIncomesSince dependencyGetRewardIncomesSince,
	getRewardIncomesUntil dependencyGetRewardIncomesUntil,
) {
	// parse pagination args
	isSince, separator, limit, err := parsePagination(c)
	if err != nil {
		c.AbortWithStatus(http.StatusBadRequest)
		return
	}

	// get result according to args
	t := time.Unix(separator, 0)
	result := []models.Income{}
	if isSince {
		result, err = getRewardIncomesSince(userID, t, limit)
	} else {
		result, err = getRewardIncomesUntil(userID, t, limit)
	}

	// response with result or error
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	c.JSON(http.StatusOK, result)
}
<|endoftext|>


<commit_before>package isogrids

import "testing"

func TestDistanceTo3rdPoint(t *testing.T) {

	got := distanceTo3rdPoint(4)
	expected := 4
	if got != expected {
		t.Errorf("expected %d got %d", expected, got)
	}
}
<commit_msg>add test to Right1stTriangle<commit_after>package isogrids

import "testing"

func TestDistanceTo3rdPoint(t *testing.T) {

	got := distanceTo3rdPoint(4)
	expected := 4
	if got != expected {
		t.Errorf("expected %d got %d", expected, got)
	}
}

func TestRight1stTriangle(t *testing.T) {

	x1, y1, x2, y2, x3, y3 := right1stTriangle(0, 0, 2, 4)
	got := []int{x1, y1, x2, y2, x3, y3}
	expected := []int{0, 0, 4, 1, 0, 2}

	if len(got) != len(expected) {
		t.Errorf("lengths of arrays should be equal")
	}

	for k, _ := range got {
		if got[k] != expected[k] {
			t.Errorf("expected %d got %d", expected[k], got[k])
		}
	}
}
<|endoftext|>


<commit_before>// (c) 2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package scheduler

import (
	"time"

	"github.com/ava-labs/avalanchego/snow/engine/common"
	"github.com/ava-labs/avalanchego/utils/logging"
)

const (
	fromVMSize = 1024
)

type Scheduler interface {
	Dispatch(startTime time.Time)
	SetBuildBlockTime(t time.Time)
	Close()
}

// Scheduler receives notifications from a VM that it wants its engine to
// call the VM's BuildBlock method, and delivers the notification to the
// engine only when the engine should call BuildBlock. Namely, when this node is
// allowed to propose a block under the congestion control mechanism.
type scheduler struct {
	log logging.Logger
	// TODO this isn't used. Do we need this?
	activationTime time.Time
	// The VM sends a message on this channel when it wants to tell the engine
	// that the engine should call the VM's BuildBlock method
	fromVM <-chan common.Message
	// The scheduler sends a message on this channel to notify the engine that
	// it should call its VM's BuildBlock method
	toEngine chan<- common.Message
	// When we receive a message on this channel, it means that we must refrain
	// from telling the engine to call its VM's BuildBlock method until the given time
	newBuildBlockTime chan time.Time
}

func New(log logging.Logger, toEngine chan<- common.Message, activationTime time.Time) (Scheduler, chan<- common.Message) {
	vmToEngine := make(chan common.Message, fromVMSize)
	return &scheduler{
		activationTime:    activationTime,
		fromVM:            vmToEngine,
		toEngine:          toEngine,
		newBuildBlockTime: make(chan time.Time),
	}, vmToEngine
}

func (s *scheduler) Dispatch(buildBlockTime time.Time) {
	timer := time.NewTimer(time.Until(buildBlockTime))
	var ok bool
waitloop:
	for {
		select {
		case <-timer.C: // It's time to tell the engine to try to build a block
		case buildBlockTime, ok = <-s.newBuildBlockTime:
			if !ok {
				// s.Close() was called
				timer.Stop()
				return
			}
			// The time at which we should notify the engine that
			// it should try to build a block has changed
			timer.Reset(time.Until(buildBlockTime))
			continue waitloop
		}

		// At this point, we know [timer.C] has been drained so it's
		// safe to call [timer.Reset] below
		for {
			gotNewBuildBlockTime := false
			select {
			case msg := <-s.fromVM:
				// Give the engine the message from the VM asking the engine to build a block
				select {
				case s.toEngine <- msg:
				default:
					// If the channel to the engine is full, drop the message from the VM to avoid deadlock
					s.log.Debug("dropping message from VM because channel to engine is full")
				}
			case buildBlockTime, ok = <-s.newBuildBlockTime:
				// The time at which we should notify the engine that
				// it should try to build a block has changed
				if !ok {
					// s.Close() was called
					return
				}
				gotNewBuildBlockTime = true
			}
			if gotNewBuildBlockTime {
				timer.Reset(time.Until(buildBlockTime))
				continue waitloop
			}
		}
	}
}

func (s *scheduler) SetBuildBlockTime(t time.Time) {
	s.newBuildBlockTime <- t
}

func (s *scheduler) Close() {
	close(s.newBuildBlockTime)
}
<commit_msg>cleanup<commit_after>// (c) 2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package scheduler

import (
	"time"

	"github.com/ava-labs/avalanchego/snow/engine/common"
	"github.com/ava-labs/avalanchego/utils/logging"
)

const (
	fromVMSize = 1024
)

type Scheduler interface {
	Dispatch(startTime time.Time)
	SetBuildBlockTime(t time.Time)
	Close()
}

// Scheduler receives notifications from a VM that it wants its engine to
// call the VM's BuildBlock method, and delivers the notification to the
// engine only when the engine should call BuildBlock. Namely, when this node is
// allowed to propose a block under the congestion control mechanism.
type scheduler struct {
	log logging.Logger
	// TODO this isn't used. Do we need this?
	activationTime time.Time
	// The VM sends a message on this channel when it wants to tell the engine
	// that the engine should call the VM's BuildBlock method
	fromVM <-chan common.Message
	// The scheduler sends a message on this channel to notify the engine that
	// it should call its VM's BuildBlock method
	toEngine chan<- common.Message
	// When we receive a message on this channel, it means that we must refrain
	// from telling the engine to call its VM's BuildBlock method until the given time
	newBuildBlockTime chan time.Time
}

func New(log logging.Logger, toEngine chan<- common.Message, activationTime time.Time) (Scheduler, chan<- common.Message) {
	vmToEngine := make(chan common.Message, fromVMSize)
	return &scheduler{
		activationTime:    activationTime,
		fromVM:            vmToEngine,
		toEngine:          toEngine,
		newBuildBlockTime: make(chan time.Time),
	}, vmToEngine
}

func (s *scheduler) Dispatch(buildBlockTime time.Time) {
	timer := time.NewTimer(time.Until(buildBlockTime))
	var ok bool
waitloop:
	for {
		select {
		case <-timer.C: // It's time to tell the engine to try to build a block
		case buildBlockTime, ok = <-s.newBuildBlockTime:
			if !ok {
				// s.Close() was called
				timer.Stop()
				return
			}
			// The time at which we should notify the engine that
			// it should try to build a block has changed
			timer.Reset(time.Until(buildBlockTime))
			continue waitloop
		}

		// At this point, we know [timer.C] has been drained so it's
		// safe to call [timer.Reset] below
		for {
			select {
			case msg := <-s.fromVM:
				// Give the engine the message from the VM asking the engine to build a block
				select {
				case s.toEngine <- msg:
				default:
					// If the channel to the engine is full, drop the message from the VM to avoid deadlock
					s.log.Debug("dropping message from VM because channel to engine is full")
				}
			case buildBlockTime, ok = <-s.newBuildBlockTime:
				// The time at which we should notify the engine that
				// it should try to build a block has changed
				if !ok {
					// s.Close() was called
					return
				}
				timer.Reset(time.Until(buildBlockTime))
				continue waitloop
			}
		}
	}
}

func (s *scheduler) SetBuildBlockTime(t time.Time) {
	s.newBuildBlockTime <- t
}

func (s *scheduler) Close() {
	close(s.newBuildBlockTime)
}
<|endoftext|>


<commit_before>package gendriver

import (
	"bytes"

	"testing"
)

func TestDefaultRenderer(t *testing.T) {

	tests := []struct{
		FileName       string
		FileTmpl       string
		RendererParams RendererParams
		Expected       string
	}{
		{
			FileName: "apple.go",
			FileTmpl:
`package {{.Pkg}}

type Type struct {
	value {{.Type}}
}
`,
			RendererParams: RendererParams{
				Pkg:  "itemid",
				Type: "int64",
			},
			Expected:
`package itemid

type Type struct {
	value int64
}
`,
		},

		{
			FileName: "banana_test.go",
			FileTmpl:
`package {{.Pkg}}

import (
	"testing"
)

func TestType{{.Type}}(t *testing.T) {

	t.Errorf("Uh oh!")

}
`,
			RendererParams: RendererParams{
				Pkg:  "somethingid",
				Type: "string",
			},
			Expected:
`package somethingid

import (
	"testing"
)

func TestTypestring(t *testing.T) {

	t.Errorf("Uh oh!")

}
`,
		},
	}

	for testNumber, test := range tests {

		renderer := DefaultRenderer{
			FileName: test.FileName,
			FileTmpl: test.FileTmpl,
		}

		actualFileName, actualWriterTo, err := renderer.WriterTo(test.RendererParams)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		if expected, actual := test.FileName, actualFileName; expected != actual {
			t.Errorf("For test #%d, expected %q, but actually got %q.", testNumber, expected, actual)
			continue
		}

		{
			var buffer bytes.Buffer

			n, err := actualWriterTo.WriteTo(&buffer)
			if nil != err {
				t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
				continue
			}

			if expected, actual := n, int64(len(test.Expected)); expected != actual {
				t.Errorf("For test #%d, expected %d, but actually got %d.", testNumber, expected, actual)
				continue
			}
			if expected, actual := test.Expected, buffer.String(); expected != actual {
				t.Errorf("For test #%d...", testNumber)
				t.Errorf("EXPECTED: %q", expected)
				t.Errorf("ACTUAL:   %q", actual)
				continue
			}
		}
	}
}
<commit_msg>udpated tests<commit_after>package gendriver

import (
	"bytes"

	"testing"
)

func TestDefaultRenderer(t *testing.T) {

	tests := []struct{
		FileName       string
		FileTmpl       string
		RendererParams RendererParams
		Expected       string
	}{
		{
			FileName: "apple.go",
			FileTmpl:
`type Type struct {
	value {{.Type}}
}
`,
			RendererParams: RendererParams{
				Pkg:  "itemid",
				Type: "int64",
			},
			Expected:
`package itemid

/*
 * CODE GENERATED AUTOMATICALLY WITH https://github.com/reiver/gogen-optiontype
 * THIS FILE SHOULD NOT BE EDITED BY HAND
 */

type Type struct {
	value int64
}
`,
		},

		{
			FileName: "banana_test.go",
			FileTmpl:
`import (
	"testing"
)

func TestType{{.Type}}(t *testing.T) {

	t.Errorf("Uh oh!")

}
`,
			RendererParams: RendererParams{
				Pkg:  "somethingid",
				Type: "string",
			},
			Expected:
`package somethingid

/*
 * CODE GENERATED AUTOMATICALLY WITH https://github.com/reiver/gogen-optiontype
 * THIS FILE SHOULD NOT BE EDITED BY HAND
 */

import (
	"testing"
)

func TestTypestring(t *testing.T) {

	t.Errorf("Uh oh!")

}
`,
		},
	}

	for testNumber, test := range tests {

		renderer := DefaultRenderer{
			FileName: test.FileName,
			FileTmpl: test.FileTmpl,
		}

		actualFileName, actualWriterTo, err := renderer.WriterTo(test.RendererParams)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		if expected, actual := test.FileName, actualFileName; expected != actual {
			t.Errorf("For test #%d, expected %q, but actually got %q.", testNumber, expected, actual)
			continue
		}

		{
			var buffer bytes.Buffer

			n, err := actualWriterTo.WriteTo(&buffer)
			if nil != err {
				t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
				continue
			}

			if expected, actual := n, int64(len(test.Expected)); expected != actual {
				t.Errorf("For test #%d, expected %d, but actually got %d.", testNumber, expected, actual)
				t.Errorf("EXPECTED: %d", expected)
				t.Errorf("ACTUAL:   %d", actual)

				t.Errorf("EXPECTED:\n%s", test.Expected)
				t.Errorf("ACTUAL:\n%s", buffer.String())
				continue
			}
			if expected, actual := test.Expected, buffer.String(); expected != actual {
				t.Errorf("For test #%d...", testNumber)
				t.Errorf("EXPECTED:\n%v", expected)
				t.Errorf("ACTUAL:\n%v", actual)
				continue
			}
		}
	}
}
<|endoftext|>


<commit_before>package docker

import (
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	"github.com/iron-io/worker/runner/agent"
	"golang.org/x/net/context"
)

// wrap docker client calls so we can retry 500s, kind of sucks but fsouza doesn't
// bake in retries we can use internally, could contribute it at some point, would
// be much more convenient if we didn't have to do this, but it's better than ad hoc retries.
// also adds timeouts to many operations, varying by operation
// TODO could generate this, maybe not worth it, may not change often
type dockerClient interface {
	// Each of these are github.com/fsouza/go-dockerclient methods

	AttachToContainerNonBlocking(opts docker.AttachToContainerOptions) (docker.CloseWaiter, error)
	WaitContainer(ctx context.Context, id string) (int, error)
	StartContainer(id string, hostConfig *docker.HostConfig) error
	CreateContainer(opts docker.CreateContainerOptions) (*docker.Container, error)
	RemoveContainer(opts docker.RemoveContainerOptions) error
	PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) error
	InspectImage(name string) (*docker.Image, error)
	InspectContainer(id string) (*docker.Container, error)
	StopContainer(id string, timeout uint) error
	Stats(opts docker.StatsOptions) error
}

// TODO: switch to github.com/docker/engine-api
func newClient() dockerClient {
	// docker, err := docker.NewClient(conf.Docker)
	client, err := docker.NewClientFromEnv()
	if err != nil {
		logrus.WithError(err).Fatal("couldn't create docker client")
	}

	// NOTE add granularity to things like pull, should not effect
	// hijacked / streaming endpoints
	client.SetTimeout(120 * time.Second)

	return &dockerWrap{client}
}

type dockerWrap struct {
	docker *docker.Client
}

func retry(f func() error) {
	var b agent.Backoff
	then := time.Now()
	limit := 10 * time.Minute
	for time.Since(then) < limit {
		err := f()
		if agent.IsTemporary(err) || isDocker500(err) {
			logrus.WithError(err).Warn("docker temporary error, retrying")
			b.Sleep()
			continue
		}
		return
	}
	logrus.Warnf("retrying on docker errors exceeded %s, restart docker or rotate this instance?", limit)
}

func isDocker500(err error) bool {
	derr, ok := err.(*docker.Error)
	return ok && derr.Status >= 500
}

func (d *dockerWrap) AttachToContainerNonBlocking(opts docker.AttachToContainerOptions) (w docker.CloseWaiter, err error) {
	retry(func() error {
		w, err = d.docker.AttachToContainerNonBlocking(opts)
		return err
	})
	return w, err
}

func (d *dockerWrap) WaitContainer(ctx context.Context, id string) (code int, err error) {
	// special one, since fsouza doesn't have context on this one and tasks can
	// take longer than 20 minutes
	for {
		// backup bail mechanism so this doesn't sit here forever
		select {
		case <-ctx.Done():
		default:
		}

		retry(func() error {
			code, err = d.docker.WaitContainer(id)
			return err
		})
		err = filterNoSuchContainer(err)
		if err == nil {
			break
		}
		logrus.WithError(err).Warn("retrying wait container (this is ok)")
	}
	return code, err
}

func filterNoSuchContainer(err error) error {
	if err == nil {
		return nil
	}
	_, containerNotFound := err.(*docker.NoSuchContainer)
	dockerErr, ok := err.(*docker.Error)
	if containerNotFound || (ok && dockerErr.Status == 404) {
		return nil
	}
	return err
}

func filterNotRunning(err error) error {
	if err == nil {
		return nil
	}

	_, containerNotRunning := err.(*docker.ContainerNotRunning)
	dockerErr, ok := err.(*docker.Error)
	if containerNotRunning || (ok && dockerErr.Status == 304) {
		return nil
	}

	return err
}

func (d *dockerWrap) StartContainer(id string, hostConfig *docker.HostConfig) (err error) {
	retry(func() error {
		err = d.docker.StartContainer(id, hostConfig)
		return err
	})
	return err
}

func (d *dockerWrap) CreateContainer(opts docker.CreateContainerOptions) (c *docker.Container, err error) {
	retry(func() error {
		c, err = d.docker.CreateContainer(opts)
		return err
	})
	return c, err
}

func (d *dockerWrap) RemoveContainer(opts docker.RemoveContainerOptions) (err error) {
	retry(func() error {
		err = d.docker.RemoveContainer(opts)
		return err
	})
	return filterNoSuchContainer(err)
}

func (d *dockerWrap) PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) (err error) {
	retry(func() error {
		err = d.docker.PullImage(opts, auth)
		return err
	})
	return err
}

func (d *dockerWrap) InspectImage(name string) (i *docker.Image, err error) {
	retry(func() error {
		i, err = d.docker.InspectImage(name)
		return err
	})
	return i, err
}

func (d *dockerWrap) InspectContainer(id string) (c *docker.Container, err error) {
	retry(func() error {
		c, err = d.docker.InspectContainer(id)
		return err
	})
	return c, err
}

func (d *dockerWrap) StopContainer(id string, timeout uint) (err error) {
	retry(func() error {
		err = d.docker.StopContainer(id, timeout)
		return err
	})
	return filterNotRunning(filterNoSuchContainer(err))
}

func (d *dockerWrap) Stats(opts docker.StatsOptions) (err error) {
	retry(func() error {
		err = d.docker.Stats(opts)
		return err
	})
	return err
}
<commit_msg>found a new error that's kind of ok in retries... there will be more<commit_after>package docker

import (
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	"github.com/iron-io/worker/runner/agent"
	"golang.org/x/net/context"
)

// wrap docker client calls so we can retry 500s, kind of sucks but fsouza doesn't
// bake in retries we can use internally, could contribute it at some point, would
// be much more convenient if we didn't have to do this, but it's better than ad hoc retries.
// also adds timeouts to many operations, varying by operation
// TODO could generate this, maybe not worth it, may not change often
type dockerClient interface {
	// Each of these are github.com/fsouza/go-dockerclient methods

	AttachToContainerNonBlocking(opts docker.AttachToContainerOptions) (docker.CloseWaiter, error)
	WaitContainer(ctx context.Context, id string) (int, error)
	StartContainer(id string, hostConfig *docker.HostConfig) error
	CreateContainer(opts docker.CreateContainerOptions) (*docker.Container, error)
	RemoveContainer(opts docker.RemoveContainerOptions) error
	PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) error
	InspectImage(name string) (*docker.Image, error)
	InspectContainer(id string) (*docker.Container, error)
	StopContainer(id string, timeout uint) error
	Stats(opts docker.StatsOptions) error
}

// TODO: switch to github.com/docker/engine-api
func newClient() dockerClient {
	// docker, err := docker.NewClient(conf.Docker)
	client, err := docker.NewClientFromEnv()
	if err != nil {
		logrus.WithError(err).Fatal("couldn't create docker client")
	}

	// NOTE add granularity to things like pull, should not effect
	// hijacked / streaming endpoints
	client.SetTimeout(120 * time.Second)

	return &dockerWrap{client}
}

type dockerWrap struct {
	docker *docker.Client
}

func retry(f func() error) {
	var b agent.Backoff
	then := time.Now()
	limit := 10 * time.Minute
	for time.Since(then) < limit {
		err := filter(f())
		if agent.IsTemporary(err) || isDocker500(err) {
			logrus.WithError(err).Warn("docker temporary error, retrying")
			b.Sleep()
			continue
		}
		return
	}
	logrus.Warnf("retrying on docker errors exceeded %s, restart docker or rotate this instance?", limit)
}

func isDocker500(err error) bool {
	derr, ok := err.(*docker.Error)
	return ok && derr.Status >= 500
}

// some 500s are totally cool
func filter(err error) error {
	// "API error (500): {\"message\":\"service endpoint with name task-57d722ecdecb9e7be16aff17 already exists\"}\n" -> ok since container exists
	switch {
	default:
		return err
	case err == nil:
		return err
	case strings.Contains(err.Error(), "service endpoint with name"):
	}
	return nil
}

func (d *dockerWrap) AttachToContainerNonBlocking(opts docker.AttachToContainerOptions) (w docker.CloseWaiter, err error) {
	retry(func() error {
		w, err = d.docker.AttachToContainerNonBlocking(opts)
		return err
	})
	return w, err
}

func (d *dockerWrap) WaitContainer(ctx context.Context, id string) (code int, err error) {
	// special one, since fsouza doesn't have context on this one and tasks can
	// take longer than 20 minutes
	for {
		// backup bail mechanism so this doesn't sit here forever
		select {
		case <-ctx.Done():
		default:
		}

		retry(func() error {
			code, err = d.docker.WaitContainer(id)
			return err
		})
		err = filterNoSuchContainer(err)
		if err == nil {
			break
		}
		logrus.WithError(err).Warn("retrying wait container (this is ok)")
	}
	return code, err
}

func filterNoSuchContainer(err error) error {
	if err == nil {
		return nil
	}
	_, containerNotFound := err.(*docker.NoSuchContainer)
	dockerErr, ok := err.(*docker.Error)
	if containerNotFound || (ok && dockerErr.Status == 404) {
		return nil
	}
	return err
}

func filterNotRunning(err error) error {
	if err == nil {
		return nil
	}

	_, containerNotRunning := err.(*docker.ContainerNotRunning)
	dockerErr, ok := err.(*docker.Error)
	if containerNotRunning || (ok && dockerErr.Status == 304) {
		return nil
	}

	return err
}

func (d *dockerWrap) StartContainer(id string, hostConfig *docker.HostConfig) (err error) {
	retry(func() error {
		err = d.docker.StartContainer(id, hostConfig)
		return err
	})
	return err
}

func (d *dockerWrap) CreateContainer(opts docker.CreateContainerOptions) (c *docker.Container, err error) {
	retry(func() error {
		c, err = d.docker.CreateContainer(opts)
		return err
	})
	return c, err
}

func (d *dockerWrap) RemoveContainer(opts docker.RemoveContainerOptions) (err error) {
	retry(func() error {
		err = d.docker.RemoveContainer(opts)
		return err
	})
	return filterNoSuchContainer(err)
}

func (d *dockerWrap) PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) (err error) {
	retry(func() error {
		err = d.docker.PullImage(opts, auth)
		return err
	})
	return err
}

func (d *dockerWrap) InspectImage(name string) (i *docker.Image, err error) {
	retry(func() error {
		i, err = d.docker.InspectImage(name)
		return err
	})
	return i, err
}

func (d *dockerWrap) InspectContainer(id string) (c *docker.Container, err error) {
	retry(func() error {
		c, err = d.docker.InspectContainer(id)
		return err
	})
	return c, err
}

func (d *dockerWrap) StopContainer(id string, timeout uint) (err error) {
	retry(func() error {
		err = d.docker.StopContainer(id, timeout)
		return err
	})
	return filterNotRunning(filterNoSuchContainer(err))
}

func (d *dockerWrap) Stats(opts docker.StatsOptions) (err error) {
	retry(func() error {
		err = d.docker.Stats(opts)
		return err
	})
	return err
}
<|endoftext|>


<commit_before>// Copyright 2013, Friedrich Paetzke. All rights reserved.

// This package  provides wrapper functions for dot.
//
// Example:
/*
	dotter, err := godot.NewDotter(godot.OUT_PDF, godot.GRAPH_DIRECTED, "test.pdf")
	defer dotter.Close()
	if err != nil {
		panic(err)
	}

	dotter.SetLink("val", "val")
	dotter.SetLabel("val", "LABEL!!")

	dotter.SetLink("val", "val1")
	dotter.SetLink("val", "val1")
	dotter.SetLink("val", "val1")

	dotter.SetLabel("val3", "sLABEL!!")

	dotter.SetNodeShape("val3", godot.SHAPE_BOX)
	dotter.SetNodeShape("val34", godot.SHAPE_BOX)
*/
package godot

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
)

type GraphType string
type NodeShape string
type Program string
type OutputType string

const (
	GRAPH_DIRECTED   GraphType = "digraph"
	GRAPH_UNDIRECTED GraphType = "graph"
)

const (
	SHAPE_BOX       NodeShape = "BOX"
	SHAPE_CIRCLE    NodeShape = "CIRCLE"
	SHAPE_FOLDER    NodeShape = "FOLDER"
	SHAPE_PLAINTEXT NodeShape = "PLAINTEXT"
	SHAPE_TRIANGLE  NodeShape = "TRIANGLE"
)

const (
	PROG_CIRCO Program = "circo"
	PROG_DOT   Program = "dot"
	PROG_FDP   Program = "fdp"
	PROG_NEATO Program = "neato"
	PROG_SFDP  Program = "sfdp"
	PROG_TWOPI Program = "twopi"
)

const (
	OUT_BMP OutputType = "bmp"
	OUT_DOT OutputType = "dot"
	OUT_JPG OutputType = "jpg"
	OUT_PDF OutputType = "pdf"
	OUT_PNG OutputType = "png"
	OUT_PS  OutputType = "ps"
	OUT_SVG OutputType = "svg"
)

type Dotter struct {
	instance   *exec.Cmd
	stdin      io.WriteCloser
	graphType  GraphType
	isStrict   bool
	isFirstCmd bool
}

func esc(node string) string {
	node = strings.Replace(node, ".", "DOT", -1)
	node = strings.Replace(node, "/", "SLASH", -1)
	node = strings.Replace(node, "-", "HYPHEN", -1)
	return node
}

func (dotter *Dotter) sendCmd(format string, args ...interface{}) error {
	if dotter.isFirstCmd {
		dotter.isFirstCmd = false
		if dotter.isStrict {
			dotter.sendCmd("strict")
		}
		dotter.sendCmd(string(dotter.graphType) + "{")
	}

	cmd := fmt.Sprintf(format, args...) + "\n"
	_, err := io.WriteString(dotter.stdin, cmd)
	return err
}

// Creates a New Dotter.
//
// Parameters:
//
// - isStrict: if true, multiple edges won't be displayed.
//
// - writeToFile: if true, output will be written to fname. Otherwise to stdout.
//
// - fname: filename. if fname equals "", dot will make up a filename - usally
// noname.dot.*
func NewDotterEx(oType OutputType, prog Program, gType GraphType,
	isStrict, writeToFile bool, fname string) (*Dotter, error) {

	dotPath, err := exec.LookPath(string(prog))
	if err != nil {
		panic(err)
	}

	otype := "-T" + string(oType)

	var cmd *exec.Cmd
	if writeToFile {
		ofile := "-O"
		if fname != "" {
			ofile = "-o" + fname
		}
		cmd = exec.Command(dotPath, otype, ofile)
	} else {
		cmd = exec.Command(dotPath, otype)
		stdout, err := cmd.StdoutPipe()
		if err != nil {
			return nil, err
		}
		go io.Copy(os.Stdin, stdout)
	}

	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	dotter := &Dotter{}
	dotter.instance = cmd
	dotter.stdin = stdin
	dotter.graphType = gType
	dotter.isFirstCmd = true
	dotter.isStrict = isStrict

	return dotter, cmd.Start()
}

// Convenience Wrapper for NewDotterEx(). Makes proper assumptions.
// For more see NewDotterEx().
func NewDotter(oType OutputType, gType GraphType, fname string) (*Dotter, error) {
	return NewDotterEx(oType, PROG_DOT, gType, true, true, fname)
}

func (dotter *Dotter) Close() error {
	dotter.sendCmd("}")
	dotter.stdin.Close()
	return dotter.instance.Wait()
}

func (dotter *Dotter) SetLink(from, to string) error {
	link := "%s -- %s"
	if dotter.graphType == GRAPH_DIRECTED {
		link = "%s -> %s"
	}
	return dotter.sendCmd(link, esc(from), esc(to))
}

func (dotter *Dotter) SetLabel(node, label string) error {
	return dotter.sendCmd(`%s [label="%s"]`, esc(node), label)
}

func (dotter *Dotter) SetNodeShape(node string, shape NodeShape) error {
	return dotter.sendCmd(`%s [shape="%s"]`, esc(node), shape)
}
<commit_msg>Write to stdout, not stdin<commit_after>// Copyright 2013, Friedrich Paetzke. All rights reserved.

// This package  provides wrapper functions for dot.
//
// Example:
/*
	dotter, err := godot.NewDotter(godot.OUT_PDF, godot.GRAPH_DIRECTED, "test.pdf")
	defer dotter.Close()
	if err != nil {
		panic(err)
	}

	dotter.SetLink("val", "val")
	dotter.SetLabel("val", "LABEL!!")

	dotter.SetLink("val", "val1")
	dotter.SetLink("val", "val1")
	dotter.SetLink("val", "val1")

	dotter.SetLabel("val3", "sLABEL!!")

	dotter.SetNodeShape("val3", godot.SHAPE_BOX)
	dotter.SetNodeShape("val34", godot.SHAPE_BOX)
*/
package godot

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
)

type GraphType string
type NodeShape string
type Program string
type OutputType string

const (
	GRAPH_DIRECTED   GraphType = "digraph"
	GRAPH_UNDIRECTED GraphType = "graph"
)

const (
	SHAPE_BOX       NodeShape = "BOX"
	SHAPE_CIRCLE    NodeShape = "CIRCLE"
	SHAPE_FOLDER    NodeShape = "FOLDER"
	SHAPE_PLAINTEXT NodeShape = "PLAINTEXT"
	SHAPE_TRIANGLE  NodeShape = "TRIANGLE"
)

const (
	PROG_CIRCO Program = "circo"
	PROG_DOT   Program = "dot"
	PROG_FDP   Program = "fdp"
	PROG_NEATO Program = "neato"
	PROG_SFDP  Program = "sfdp"
	PROG_TWOPI Program = "twopi"
)

const (
	OUT_BMP OutputType = "bmp"
	OUT_DOT OutputType = "dot"
	OUT_JPG OutputType = "jpg"
	OUT_PDF OutputType = "pdf"
	OUT_PNG OutputType = "png"
	OUT_PS  OutputType = "ps"
	OUT_SVG OutputType = "svg"
)

type Dotter struct {
	instance   *exec.Cmd
	stdin      io.WriteCloser
	graphType  GraphType
	isStrict   bool
	isFirstCmd bool
}

func esc(node string) string {
	node = strings.Replace(node, ".", "DOT", -1)
	node = strings.Replace(node, "/", "SLASH", -1)
	node = strings.Replace(node, "-", "HYPHEN", -1)
	return node
}

func (dotter *Dotter) sendCmd(format string, args ...interface{}) error {
	if dotter.isFirstCmd {
		dotter.isFirstCmd = false
		if dotter.isStrict {
			dotter.sendCmd("strict")
		}
		dotter.sendCmd(string(dotter.graphType) + "{")
	}

	cmd := fmt.Sprintf(format, args...) + "\n"
	_, err := io.WriteString(dotter.stdin, cmd)
	return err
}

// Creates a New Dotter.
//
// Parameters:
//
// - isStrict: if true, multiple edges won't be displayed.
//
// - writeToFile: if true, output will be written to fname. Otherwise to stdout.
//
// - fname: filename. if fname equals "", dot will make up a filename - usally
// noname.dot.*
func NewDotterEx(oType OutputType, prog Program, gType GraphType,
	isStrict, writeToFile bool, fname string) (*Dotter, error) {

	dotPath, err := exec.LookPath(string(prog))
	if err != nil {
		panic(err)
	}

	otype := "-T" + string(oType)

	var cmd *exec.Cmd
	if writeToFile {
		ofile := "-O"
		if fname != "" {
			ofile = "-o" + fname
		}
		cmd = exec.Command(dotPath, otype, ofile)
	} else {
		cmd = exec.Command(dotPath, otype)
		stdout, err := cmd.StdoutPipe()
		if err != nil {
			return nil, err
		}
		go io.Copy(os.Stdout, stdout)
	}

	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	dotter := &Dotter{}
	dotter.instance = cmd
	dotter.stdin = stdin
	dotter.graphType = gType
	dotter.isFirstCmd = true
	dotter.isStrict = isStrict

	return dotter, cmd.Start()
}

// Convenience Wrapper for NewDotterEx(). Makes proper assumptions.
// For more see NewDotterEx().
func NewDotter(oType OutputType, gType GraphType, fname string) (*Dotter, error) {
	return NewDotterEx(oType, PROG_DOT, gType, true, true, fname)
}

func (dotter *Dotter) Close() error {
	dotter.sendCmd("}")
	dotter.stdin.Close()
	return dotter.instance.Wait()
}

func (dotter *Dotter) SetLink(from, to string) error {
	link := "%s -- %s"
	if dotter.graphType == GRAPH_DIRECTED {
		link = "%s -> %s"
	}
	return dotter.sendCmd(link, esc(from), esc(to))
}

func (dotter *Dotter) SetLabel(node, label string) error {
	return dotter.sendCmd(`%s [label="%s"]`, esc(node), label)
}

func (dotter *Dotter) SetNodeShape(node string, shape NodeShape) error {
	return dotter.sendCmd(`%s [shape="%s"]`, esc(node), shape)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	. "github.com/abhiyerra/workmachine/crowdflow"
	"os"
)

type ImageTagging struct {
	ImageUrl             InputField  `work_desc:"Use this image to fill the information below." work_id:"image_url" work_type:"image"`
	Tags                 OutputField `work_desc:"List all the relevent tags separated by a comma for the image. Ex. trees, castle, person" work_id:"tags"`
	TextInImage          OutputField `work_desc:"Put any caption that appears of the image here. Put one item per line if there are multiple." work_id:"text_in_image" work_type:"long_text"`
	IsCorrectOrientation OutputField `work_desc:"Is the image in the correct orientation?" work_id:"is_correct_orientation" work_type:"checkbox"`
	IsLandscape          OutputField `work_desc:"Is the image of a landscape (a non urban setting)?" work_id:"is_landscape" work_type:"checkbox"`
	IsPattern            OutputField `work_desc:"Is the image of a pattern?" work_id:"is_pattern" work_type:"checkbox"`
	IsPerson             OutputField `work_desc:"Does the image contain people?" work_id:"is_person" work_type:"checkbox"`
	TraditionalClothing  OutputField `work_desc:"If the image has people are they wearing traditional clothes?" work_id:"traditional_clothing" work_type:"checkbox"`
	IsMap                OutputField `work_desc:"Is the image a map?" work_id:"is_map" work_type:"checkbox"`
	IsDiagram            OutputField `work_desc:"Is the image a diagram?" work_id:"is_diagram" work_type:"checkbox"`
}

func imageUrls(in_file string) (images []ImageTagging) {
	file, err := os.Open(in_file)
	if err != nil {
		panic(err)
	}

	reader := csv.NewReader(file)
	records, err := reader.ReadAll()

	for _, i := range records {
		//		fmt.Printf("%s\n", i[1])
		images = append(images, ImageTagging{ImageUrl: InputField(i[1])})
	}

	if err != nil {
		panic(err)
	}

	return
}

func main() {
	var in_file string
	flag.StringVar(&in_file, "in_file", "", "input file")
	flag.Parse()

	if in_file == "" {
		fmt.Println("No in file")
		os.Exit(1)
	}

	results_filename := fmt.Sprintf("%s_out.csv", in_file)
	results_file, err := os.OpenFile(results_filename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0660)
	if err != nil {
		panic(err)
	}
	defer results_file.Close()

	image_urls := imageUrls(in_file)

	description := `
Look at the image and fill out the appropriate fields. We want to be able to tag all the images correctly. Fill out any appropriate tag that you see.
 Here are further Instructions and FAQ`

	image_tasks := Task{
		Title:       "Tag the appropriate images",
		Description: description,
		Write:       CsvJobWriter(results_file),
		Tasks:       image_urls,
	}

	fmt.Printf("Loaded %d images and starting\n", len(image_urls))
	serve := HtmlServe{}
	go HtmlServer()

	fmt.Println("Serving")
	var backend Assigner = serve
	NewBatch(image_tasks).Run(backend)

}
<commit_msg>Updated the code to work with new crowdflow code.<commit_after>package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	. "github.com/abhiyerra/workmachine/crowdflow"
	"os"
)

type ImageTagging struct {
	ImageUrl             InputField  `work_desc:"Use this image to fill the information below." work_id:"image_url" work_type:"image"`
	Tags                 OutputField `work_desc:"List all the relevent tags separated by a comma for the image. Ex. trees, castle, person" work_id:"tags"`
	TextInImage          OutputField `work_desc:"Put any caption that appears of the image here. Put one item per line if there are multiple." work_id:"text_in_image" work_type:"long_text"`
	IsCorrectOrientation OutputField `work_desc:"Is the image in the correct orientation?" work_id:"is_correct_orientation" work_type:"checkbox"`
	IsLandscape          OutputField `work_desc:"Is the image of a landscape (a non urban setting)?" work_id:"is_landscape" work_type:"checkbox"`
	IsPattern            OutputField `work_desc:"Is the image of a pattern?" work_id:"is_pattern" work_type:"checkbox"`
	IsPerson             OutputField `work_desc:"Does the image contain people?" work_id:"is_person" work_type:"checkbox"`
	TraditionalClothing  OutputField `work_desc:"If the image has people are they wearing traditional clothes?" work_id:"traditional_clothing" work_type:"checkbox"`
	IsMap                OutputField `work_desc:"Is the image a map?" work_id:"is_map" work_type:"checkbox"`
	IsDiagram            OutputField `work_desc:"Is the image a diagram?" work_id:"is_diagram" work_type:"checkbox"`
}

func imageUrls(in_file string) (images []ImageTagging) {
	file, err := os.Open(in_file)
	if err != nil {
		panic(err)
	}

	reader := csv.NewReader(file)
	records, err := reader.ReadAll()

	for _, i := range records {
		//		fmt.Printf("%s\n", i[1])
		images = append(images, ImageTagging{ImageUrl: InputField(i[1])})
	}

	if err != nil {
		panic(err)
	}

	return
}

func main() {
	var in_file string
	flag.StringVar(&in_file, "in_file", "", "input file")
	flag.Parse()

	if in_file == "" {
		fmt.Println("No in file")
		os.Exit(1)
	}

	results_filename := fmt.Sprintf("%s_out.csv", in_file)
	results_file, err := os.OpenFile(results_filename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0660)
	if err != nil {
		panic(err)
	}
	defer results_file.Close()

	image_urls := imageUrls(in_file)

	description := `
Look at the image and fill out the appropriate fields. We want to be able to tag all the images correctly. Fill out any appropriate tag that you see.
 Here are further Instructions and FAQ`

	image_tasks := TaskDesc{
		Title:       "Tag the appropriate images",
		Description: description,
		Write:       CsvJobWriter(results_file),
		Tasks:       image_urls,
	}

	fmt.Printf("Loaded %d images and starting\n", len(image_urls))
	serve := SplitHtmlServe{}
	go HtmlServer()

	fmt.Println("Serving")
	var backend SplitAssigner = serve
	NewBatch(image_tasks).RunSplit(backend)

}
<|endoftext|>


<commit_before>package customer

import (
	"strconv"

	"movie"
	"rental"
)

// Customer store customer information
type Customer struct {
	name    string
	rentals []*rental.Rental
}

// NewCustomer create new customer with name
func NewCustomer(name string) *Customer {
	return &Customer{
		name:    name,
		rentals: make([]*rental.Rental, 0, 3),
	}
}

// AddRental adds rental r to customer rentals
func (c *Customer) AddRental(r *rental.Rental) {
	c.rentals = append(c.rentals, r)
}

// Name return customer's name
func (c *Customer) Name() string {
	return c.name
}

// Statement return string statement of customer order
func (c *Customer) Statement() string {
	var (
		totalAmount          float64
		frequentRenterPoints int
	)
	result := "Rental Record for " + c.Name() + "\n"
	for _, each := range c.rentals {
		thisAmount := amountFor(each)

		// add frequentrenterpoints renter points
		frequentRenterPoints++
		// add bonus for a two day new release rental
		if each.Movie().PriceCode == movie.NewRelease &&
			each.DaysRented() > 1 {
			frequentRenterPoints++
		}

		// show figures for this rental
		result += "\t" + each.Movie().Title() + "\t" +
			strconv.FormatFloat(thisAmount, 'f', -1, 64) + "\n"
		totalAmount += thisAmount
	}
	// add footer lines
	result += "Amount owed is " +
		strconv.FormatFloat(totalAmount, 'f', -1, 64) + "\n"
	result += "You earned " + strconv.Itoa(frequentRenterPoints) +
		" frequentrenterpoints renter points"
	return result
}

func amountFor(each *rental.Rental) float64 {
	var thisAmount float64
	// determine amounts for each line
	switch each.Movie().PriceCode {
	case movie.Regalur:
		thisAmount += 2
		if each.DaysRented() > 2 {
			thisAmount += float64(each.DaysRented()-2) * 1.5
		}
	case movie.NewRelease:
		thisAmount += float64(each.DaysRented() * 3)
	case movie.Childrens:
		thisAmount += 1.5
		if each.DaysRented() > 3 {
			thisAmount += float64(each.DaysRented()-3) * 1.5
		}
	}
	return thisAmount
}
<commit_msg>refactor(VideoStore) renaming amountFor local vars<commit_after>package customer

import (
	"strconv"

	"movie"
	"rental"
)

// Customer store customer information
type Customer struct {
	name    string
	rentals []*rental.Rental
}

// NewCustomer create new customer with name
func NewCustomer(name string) *Customer {
	return &Customer{
		name:    name,
		rentals: make([]*rental.Rental, 0, 3),
	}
}

// AddRental adds rental r to customer rentals
func (c *Customer) AddRental(r *rental.Rental) {
	c.rentals = append(c.rentals, r)
}

// Name return customer's name
func (c *Customer) Name() string {
	return c.name
}

// Statement return string statement of customer order
func (c *Customer) Statement() string {
	var (
		totalAmount          float64
		frequentRenterPoints int
	)
	result := "Rental Record for " + c.Name() + "\n"
	for _, each := range c.rentals {
		thisAmount := amountFor(each)

		// add frequentrenterpoints renter points
		frequentRenterPoints++
		// add bonus for a two day new release rental
		if each.Movie().PriceCode == movie.NewRelease &&
			each.DaysRented() > 1 {
			frequentRenterPoints++
		}

		// show figures for this rental
		result += "\t" + each.Movie().Title() + "\t" +
			strconv.FormatFloat(thisAmount, 'f', -1, 64) + "\n"
		totalAmount += thisAmount
	}
	// add footer lines
	result += "Amount owed is " +
		strconv.FormatFloat(totalAmount, 'f', -1, 64) + "\n"
	result += "You earned " + strconv.Itoa(frequentRenterPoints) +
		" frequentrenterpoints renter points"
	return result
}

func amountFor(aRental *rental.Rental) float64 {
	var result float64
	// determine amounts for each line
	switch aRental.Movie().PriceCode {
	case movie.Regalur:
		result += 2
		if aRental.DaysRented() > 2 {
			result += float64(aRental.DaysRented()-2) * 1.5
		}
	case movie.NewRelease:
		result += float64(aRental.DaysRented() * 3)
	case movie.Childrens:
		result += 1.5
		if aRental.DaysRented() > 3 {
			result += float64(aRental.DaysRented()-3) * 1.5
		}
	}
	return result
}
<|endoftext|>


<commit_before>package lib

import (
	"fmt"
	"strconv"

	"github.com/3d0c/gmf"
	"github.com/flavioribeiro/snickers/db"
	"github.com/flavioribeiro/snickers/types"
)

func addStream(codecName string, oc *gmf.FmtCtx, ist *gmf.Stream) (int, int, error) {
	var cc *gmf.CodecCtx
	var ost *gmf.Stream

	codec, err := gmf.FindEncoder(codecName)
	if err != nil {
		return 0, 0, err
	}

	if ost = oc.NewStream(codec); ost == nil {
		fmt.Println("unable to create stream in output context")
	}
	defer gmf.Release(ost)

	if cc = gmf.NewCodecCtx(codec); cc == nil {
		fmt.Println("unable to create codec context")
	}

	defer gmf.Release(cc)

	if oc.IsGlobalHeader() {
		cc.SetFlag(gmf.CODEC_FLAG_GLOBAL_HEADER)
	}

	if codec.IsExperimental() {
		cc.SetStrictCompliance(gmf.FF_COMPLIANCE_EXPERIMENTAL)
	}

	if cc.Type() == gmf.AVMEDIA_TYPE_AUDIO {
		cc.SetSampleFmt(ist.CodecCtx().SampleFmt())
		cc.SetSampleRate(ist.CodecCtx().SampleRate())
		cc.SetChannels(ist.CodecCtx().Channels())
		cc.SelectChannelLayout()
		cc.SelectSampleRate()

	}

	if cc.Type() == gmf.AVMEDIA_TYPE_VIDEO {
		cc.SetTimeBase(gmf.AVR{Num: 1, Den: 25})
		cc.SetProfile(gmf.FF_PROFILE_MPEG4_SIMPLE)
		cc.SetDimension(ist.CodecCtx().Width(), ist.CodecCtx().Height())
		cc.SetPixFmt(ist.CodecCtx().PixFmt())
	}

	if err := cc.Open(nil); err != nil {
		fmt.Println(err.Error())
	}

	ost.SetCodecCtx(cc)

	return ist.Index(), ost.Index(), nil
}

// FFMPEGEncode function is responsible for encoding the file
func FFMPEGEncode(jobID string) error {
	gmf.LogSetLevel(gmf.AV_LOG_FATAL)
	dbInstance, _ := db.GetDatabase()
	job, _ := dbInstance.RetrieveJob(jobID)
	srcFileName := job.LocalSource
	dstFileName := job.LocalDestination
	stMap := make(map[int]int, 0)
	var lastDelta int64

	inputCtx, err := gmf.NewInputCtx(srcFileName)
	if err != nil {
		job.Status = types.JobError
		job.Details = err.Error()
		dbInstance.UpdateJob(job.ID, job)
		return err
	}
	defer inputCtx.CloseInputAndRelease()

	outputCtx, err := gmf.NewOutputCtx(dstFileName)
	if err != nil {
		job.Status = types.JobError
		job.Details = err.Error()
		dbInstance.UpdateJob(job.ID, job)
		return err
	}
	defer outputCtx.CloseOutputAndRelease()

	job.Status = types.JobEncoding
	job.Details = "0%"
	dbInstance.UpdateJob(job.ID, job)

	srcVideoStream, _ := inputCtx.GetBestStream(gmf.AVMEDIA_TYPE_VIDEO)
	i, o, _ := addStream("mpeg4", outputCtx, srcVideoStream)
	stMap[i] = o

	srcAudioStream, _ := inputCtx.GetBestStream(gmf.AVMEDIA_TYPE_AUDIO)
	i, o, _ = addStream("aac", outputCtx, srcAudioStream)
	stMap[i] = o

	if err := outputCtx.WriteHeader(); err != nil {
		return err
	}
	totalFrames := float64(srcVideoStream.NbFrames() + srcAudioStream.NbFrames())
	framesCount := float64(0)

	for packet := range inputCtx.GetNewPackets() {
		ist, err := inputCtx.GetStream(packet.StreamIndex())
		if err != nil {
			return err
		}
		ost, err := outputCtx.GetStream(stMap[ist.Index()])
		if err != nil {
			return err
		}

		for frame := range packet.Frames(ist.CodecCtx()) {
			if ost.IsAudio() {
				fsTb := gmf.AVR{Num: 1, Den: ist.CodecCtx().SampleRate()}
				outTb := gmf.AVR{Num: 1, Den: ist.CodecCtx().SampleRate()}

				frame.SetPts(packet.Pts())

				pts := gmf.RescaleDelta(ist.TimeBase(), frame.Pts(), fsTb.AVRational(), frame.NbSamples(), &lastDelta, outTb.AVRational())

				frame.
					SetNbSamples(ost.CodecCtx().FrameSize()).
					SetFormat(ost.CodecCtx().SampleFmt()).
					SetChannelLayout(ost.CodecCtx().ChannelLayout()).
					SetPts(pts)
			} else {
				frame.SetPts(ost.Pts)
			}

			if p, ready, _ := frame.EncodeNewPacket(ost.CodecCtx()); ready {
				if p.Pts() != gmf.AV_NOPTS_VALUE {
					p.SetPts(gmf.RescaleQ(p.Pts(), ost.CodecCtx().TimeBase(), ost.TimeBase()))
				}

				if p.Dts() != gmf.AV_NOPTS_VALUE {
					p.SetDts(gmf.RescaleQ(p.Dts(), ost.CodecCtx().TimeBase(), ost.TimeBase()))
				}

				p.SetStreamIndex(ost.Index())

				if err := outputCtx.WritePacket(p); err != nil {
					return err
				}
				gmf.Release(p)
			}

			ost.Pts++
			framesCount++
			percentage := string(strconv.FormatInt(int64(framesCount/totalFrames*100), 10) + "%")
			if percentage != job.Details {
				job.Details = percentage
				dbInstance.UpdateJob(job.ID, job)
			}
		}
		gmf.Release(packet)
	}

	for i := 0; i < outputCtx.StreamsCnt(); i++ {
		ist, err := inputCtx.GetStream(0)
		if err != nil {
			return err
		}
		ost, err := outputCtx.GetStream(stMap[ist.Index()])
		if err != nil {
			return err
		}

		frame := gmf.NewFrame()

		for {
			if p, ready, _ := frame.FlushNewPacket(ost.CodecCtx()); ready {
				if p.Pts() != gmf.AV_NOPTS_VALUE {
					p.SetPts(gmf.RescaleQ(p.Pts(), ost.CodecCtx().TimeBase(), ost.TimeBase()))
				}

				if p.Dts() != gmf.AV_NOPTS_VALUE {
					p.SetDts(gmf.RescaleQ(p.Dts(), ost.CodecCtx().TimeBase(), ost.TimeBase()))
				}

				p.SetStreamIndex(ost.Index())

				if err := outputCtx.WritePacket(p); err != nil {
					return err
				}
				gmf.Release(p)
			} else {
				gmf.Release(p)
				break
			}

			ost.Pts++
		}

		gmf.Release(frame)
	}
	if job.Details != "100%" {
		job.Details = "100%"
		dbInstance.UpdateJob(job.ID, job)
	}

	return nil
}
<commit_msg>ffmpeg_encoder: raise errors (close #43)<commit_after>package lib

import (
	"errors"
	"strconv"

	"github.com/3d0c/gmf"
	"github.com/flavioribeiro/snickers/db"
	"github.com/flavioribeiro/snickers/types"
)

func addStream(codecName string, oc *gmf.FmtCtx, ist *gmf.Stream) (int, int, error) {
	var cc *gmf.CodecCtx
	var ost *gmf.Stream

	codec, err := gmf.FindEncoder(codecName)
	if err != nil {
		return 0, 0, err
	}

	if ost = oc.NewStream(codec); ost == nil {
		return 0, 0, errors.New("unable to create stream in output context")
	}
	defer gmf.Release(ost)

	if cc = gmf.NewCodecCtx(codec); cc == nil {
		return 0, 0, errors.New("unable to create codec context")
	}

	defer gmf.Release(cc)

	if oc.IsGlobalHeader() {
		cc.SetFlag(gmf.CODEC_FLAG_GLOBAL_HEADER)
	}

	if codec.IsExperimental() {
		cc.SetStrictCompliance(gmf.FF_COMPLIANCE_EXPERIMENTAL)
	}

	if cc.Type() == gmf.AVMEDIA_TYPE_AUDIO {
		cc.SetSampleFmt(ist.CodecCtx().SampleFmt())
		cc.SetSampleRate(ist.CodecCtx().SampleRate())
		cc.SetChannels(ist.CodecCtx().Channels())
		cc.SelectChannelLayout()
		cc.SelectSampleRate()

	}

	if cc.Type() == gmf.AVMEDIA_TYPE_VIDEO {
		cc.SetTimeBase(gmf.AVR{Num: 1, Den: 25})
		cc.SetProfile(gmf.FF_PROFILE_MPEG4_SIMPLE)
		cc.SetDimension(ist.CodecCtx().Width(), ist.CodecCtx().Height())
		cc.SetPixFmt(ist.CodecCtx().PixFmt())
	}

	if err := cc.Open(nil); err != nil {
		return 0, 0, err
	}

	ost.SetCodecCtx(cc)

	return ist.Index(), ost.Index(), nil
}

// FFMPEGEncode function is responsible for encoding the file
func FFMPEGEncode(jobID string) error {
	gmf.LogSetLevel(gmf.AV_LOG_FATAL)
	dbInstance, _ := db.GetDatabase()
	job, _ := dbInstance.RetrieveJob(jobID)
	srcFileName := job.LocalSource
	dstFileName := job.LocalDestination
	stMap := make(map[int]int, 0)
	var lastDelta int64

	inputCtx, err := gmf.NewInputCtx(srcFileName)
	if err != nil {
		job.Status = types.JobError
		job.Details = err.Error()
		dbInstance.UpdateJob(job.ID, job)
		return err
	}
	defer inputCtx.CloseInputAndRelease()

	outputCtx, err := gmf.NewOutputCtx(dstFileName)
	if err != nil {
		job.Status = types.JobError
		job.Details = err.Error()
		dbInstance.UpdateJob(job.ID, job)
		return err
	}
	defer outputCtx.CloseOutputAndRelease()

	job.Status = types.JobEncoding
	job.Details = "0%"
	dbInstance.UpdateJob(job.ID, job)

	srcVideoStream, _ := inputCtx.GetBestStream(gmf.AVMEDIA_TYPE_VIDEO)
	i, o, err := addStream("mpeg4", outputCtx, srcVideoStream)
	if err != nil {
		return err
	}
	stMap[i] = o

	srcAudioStream, err := inputCtx.GetBestStream(gmf.AVMEDIA_TYPE_AUDIO)
	if err != nil {
		return err
	}

	i, o, err = addStream("aac", outputCtx, srcAudioStream)
	if err != nil {
		return err
	}
	stMap[i] = o

	if err := outputCtx.WriteHeader(); err != nil {
		return err
	}
	totalFrames := float64(srcVideoStream.NbFrames() + srcAudioStream.NbFrames())
	framesCount := float64(0)

	for packet := range inputCtx.GetNewPackets() {
		ist, err := inputCtx.GetStream(packet.StreamIndex())
		if err != nil {
			return err
		}
		ost, err := outputCtx.GetStream(stMap[ist.Index()])
		if err != nil {
			return err
		}

		for frame := range packet.Frames(ist.CodecCtx()) {
			if ost.IsAudio() {
				fsTb := gmf.AVR{Num: 1, Den: ist.CodecCtx().SampleRate()}
				outTb := gmf.AVR{Num: 1, Den: ist.CodecCtx().SampleRate()}

				frame.SetPts(packet.Pts())

				pts := gmf.RescaleDelta(ist.TimeBase(), frame.Pts(), fsTb.AVRational(), frame.NbSamples(), &lastDelta, outTb.AVRational())

				frame.
					SetNbSamples(ost.CodecCtx().FrameSize()).
					SetFormat(ost.CodecCtx().SampleFmt()).
					SetChannelLayout(ost.CodecCtx().ChannelLayout()).
					SetPts(pts)
			} else {
				frame.SetPts(ost.Pts)
			}

			if p, ready, _ := frame.EncodeNewPacket(ost.CodecCtx()); ready {
				if p.Pts() != gmf.AV_NOPTS_VALUE {
					p.SetPts(gmf.RescaleQ(p.Pts(), ost.CodecCtx().TimeBase(), ost.TimeBase()))
				}

				if p.Dts() != gmf.AV_NOPTS_VALUE {
					p.SetDts(gmf.RescaleQ(p.Dts(), ost.CodecCtx().TimeBase(), ost.TimeBase()))
				}

				p.SetStreamIndex(ost.Index())

				if err := outputCtx.WritePacket(p); err != nil {
					return err
				}
				gmf.Release(p)
			}

			ost.Pts++
			framesCount++
			percentage := string(strconv.FormatInt(int64(framesCount/totalFrames*100), 10) + "%")
			if percentage != job.Details {
				job.Details = percentage
				dbInstance.UpdateJob(job.ID, job)
			}
		}
		gmf.Release(packet)
	}

	for i := 0; i < outputCtx.StreamsCnt(); i++ {
		ist, err := inputCtx.GetStream(0)
		if err != nil {
			return err
		}
		ost, err := outputCtx.GetStream(stMap[ist.Index()])
		if err != nil {
			return err
		}

		frame := gmf.NewFrame()

		for {
			if p, ready, _ := frame.FlushNewPacket(ost.CodecCtx()); ready {
				if p.Pts() != gmf.AV_NOPTS_VALUE {
					p.SetPts(gmf.RescaleQ(p.Pts(), ost.CodecCtx().TimeBase(), ost.TimeBase()))
				}

				if p.Dts() != gmf.AV_NOPTS_VALUE {
					p.SetDts(gmf.RescaleQ(p.Dts(), ost.CodecCtx().TimeBase(), ost.TimeBase()))
				}

				p.SetStreamIndex(ost.Index())

				if err := outputCtx.WritePacket(p); err != nil {
					return err
				}
				gmf.Release(p)
			} else {
				gmf.Release(p)
				break
			}

			ost.Pts++
		}

		gmf.Release(frame)
	}
	if job.Details != "100%" {
		job.Details = "100%"
		dbInstance.UpdateJob(job.ID, job)
	}

	return nil
}
<|endoftext|>