<commit_before>// This package provides a simple LRU cache. It is based on the
// LRU implementation in groupcache:
// https://github.com/golang/groupcache/tree/master/lru
package lru

import (
	"container/list"
	"errors"
	"sync"
)

// Cache is a thread-safe fixed size LRU cache.
type Cache struct {
	size      int
	evictList *list.List
	items     map[interface{}]*list.Element
	lock      sync.Mutex
}

// entry is used to hold a value in the evictList
type entry struct {
	key   interface{}
	value interface{}
}

// New creates an LRU of the given size
func New(size int) (*Cache, error) {
	if size <= 0 {
		return nil, errors.New("Must provide a positive size")
	}
	c := &Cache{
		size:      size,
		evictList: list.New(),
		items:     make(map[interface{}]*list.Element, size),
	}
	return c, nil
}

// Purge is used to completely clear the cache
func (c *Cache) Purge() {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.evictList = list.New()
	c.items = make(map[interface{}]*list.Element, c.size)
}

// Add adds a value to the cache.
func (c *Cache) Add(key, value interface{}) {
	c.lock.Lock()
	defer c.lock.Unlock()

	// Check for existing item
	if ent, ok := c.items[key]; ok {
		c.evictList.MoveToFront(ent)
		ent.Value.(*entry).value = value
		return
	}

	// Add new item
	ent := &entry{key, value}
	entry := c.evictList.PushFront(ent)
	c.items[key] = entry

	// Verify size not exceeded
	if c.evictList.Len() > c.size {
		c.removeOldest()
	}
}

// Get looks up a key's value from the cache.
func (c *Cache) Get(key interface{}) (value interface{}, ok bool) {
	c.lock.Lock()
	defer c.lock.Unlock()

	if ent, ok := c.items[key]; ok {
		c.evictList.MoveToFront(ent)
		return ent.Value.(*entry).value, true
	}
	return
}

// Remove removes the provided key from the cache.
func (c *Cache) Remove(key interface{}) {
	c.lock.Lock()
	defer c.lock.Unlock()

	if ent, ok := c.items[key]; ok {
		c.removeElement(ent)
	}
}

// RemoveOldest removes the oldest item from the cache.
func (c *Cache) RemoveOldest() {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.removeOldest()
}

// Returns the keys of itmes in the cache.
func (c *Cache) Keys() []interface{} {
	c.lock.Lock()
	defer c.lock.Unlock()

	keys := make([]interface{}, c.size)
	i := 0
	for k := range c.items {
		keys[i] = k
		i++
	}

	return keys
}

// removeOldest removes the oldest item from the cache.
func (c *Cache) removeOldest() {
	ent := c.evictList.Back()
	if ent != nil {
		c.removeElement(ent)
	}
}

// removeElement is used to remove a given list element from the cache
func (c *Cache) removeElement(e *list.Element) {
	c.evictList.Remove(e)
	kv := e.Value.(*entry)
	delete(c.items, kv.key)
}

// Len returns the number of items in the cache.
func (c *Cache) Len() int {
	c.lock.Lock()
	defer c.lock.Unlock()
	return c.evictList.Len()
}
<commit_msg>Doc update<commit_after>// This package provides a simple LRU cache. It is based on the
// LRU implementation in groupcache:
// https://github.com/golang/groupcache/tree/master/lru
package lru

import (
	"container/list"
	"errors"
	"sync"
)

// Cache is a thread-safe fixed size LRU cache.
type Cache struct {
	size      int
	evictList *list.List
	items     map[interface{}]*list.Element
	lock      sync.Mutex
}

// entry is used to hold a value in the evictList
type entry struct {
	key   interface{}
	value interface{}
}

// New creates an LRU of the given size
func New(size int) (*Cache, error) {
	if size <= 0 {
		return nil, errors.New("Must provide a positive size")
	}
	c := &Cache{
		size:      size,
		evictList: list.New(),
		items:     make(map[interface{}]*list.Element, size),
	}
	return c, nil
}

// Purge is used to completely clear the cache
func (c *Cache) Purge() {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.evictList = list.New()
	c.items = make(map[interface{}]*list.Element, c.size)
}

// Add adds a value to the cache.
func (c *Cache) Add(key, value interface{}) {
	c.lock.Lock()
	defer c.lock.Unlock()

	// Check for existing item
	if ent, ok := c.items[key]; ok {
		c.evictList.MoveToFront(ent)
		ent.Value.(*entry).value = value
		return
	}

	// Add new item
	ent := &entry{key, value}
	entry := c.evictList.PushFront(ent)
	c.items[key] = entry

	// Verify size not exceeded
	if c.evictList.Len() > c.size {
		c.removeOldest()
	}
}

// Get looks up a key's value from the cache.
func (c *Cache) Get(key interface{}) (value interface{}, ok bool) {
	c.lock.Lock()
	defer c.lock.Unlock()

	if ent, ok := c.items[key]; ok {
		c.evictList.MoveToFront(ent)
		return ent.Value.(*entry).value, true
	}
	return
}

// Remove removes the provided key from the cache.
func (c *Cache) Remove(key interface{}) {
	c.lock.Lock()
	defer c.lock.Unlock()

	if ent, ok := c.items[key]; ok {
		c.removeElement(ent)
	}
}

// RemoveOldest removes the oldest item from the cache.
func (c *Cache) RemoveOldest() {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.removeOldest()
}

// Keys returns a slice of the keys in the cache.
func (c *Cache) Keys() []interface{} {
	c.lock.Lock()
	defer c.lock.Unlock()

	keys := make([]interface{}, c.size)
	i := 0
	for k := range c.items {
		keys[i] = k
		i++
	}

	return keys
}

// removeOldest removes the oldest item from the cache.
func (c *Cache) removeOldest() {
	ent := c.evictList.Back()
	if ent != nil {
		c.removeElement(ent)
	}
}

// removeElement is used to remove a given list element from the cache
func (c *Cache) removeElement(e *list.Element) {
	c.evictList.Remove(e)
	kv := e.Value.(*entry)
	delete(c.items, kv.key)
}

// Len returns the number of items in the cache.
func (c *Cache) Len() int {
	c.lock.Lock()
	defer c.lock.Unlock()
	return c.evictList.Len()
}
<|endoftext|>


<commit_before>package zero

import (
	"fmt"
	"time"
	"strconv"
	"strings"

	"github.com/tj/go-debug"
)

type numVal struct {
	Time time.Time
	Value int64
}

type Num struct {
	store map[Key]numVal
	debug debug.DebugFunction
}

func (n *Num) Add(key Key, value int64) {
	n.debug("adding key %v with value %d", key, value)
	n.store[key] = numVal{
		Time: time.Now(),
		Value: value,
	}
}

func (n *Num) Show(key Key) (int64, error) {
	n.debug("displaying value for key %v", key)
	if v, ok := n.store[key]; ok {
		n.debug("key found")
		return v.Value, nil
	}

	return 0, fmt.Errorf("key %s not found", key)
}

func (n *Num) All() [][]string {
	n.debug("displaying all values (%d)", len(n.store))
	vals := [][]string{}
	for k, v := range n.store {
		vals = append(vals, []string{
			string(k),
			strings.TrimSpace(strconv.FormatInt(v.Value, 10)),
			"number",
			v.Time.Format(time.RFC822),
		})
	}

	return vals
}

func (n *Num) Keys() []Key {
	n.debug("getting all keys for store (len: %d)", len(n.store))
	keys := make([]Key, 0, len(n.store))
	for k := range n.store {
		keys = append(keys, k)
	}

	return keys
}

func (n *Num) Count() int {
	n.debug("getting count of items in store")
	return len(n.store)
}

func (n *Num) Del(key Key) error {
	n.debug("removing value for key %v", key)
	if _, ok := n.store[key]; ok {
		n.debug("key found")
		delete(n.store, key)
		return nil
	}

	return fmt.Errorf("key %s not found", key)
}

func NewNum() *Num {
	return &Num{
		store: make(map[Key]numVal),
		debug: debug.Debug("mini:number"),
	}
}
<commit_msg>Create type for numeric store<commit_after>package zero

import (
	"fmt"
	"time"
	"strconv"
	"strings"

	"github.com/tj/go-debug"
)

type numVal struct {
	Time time.Time
	Value int64
}

type numStore map[Key]numVal

type Num struct {
	store numStore
	debug debug.DebugFunction
}

func (n *Num) Add(key Key, value int64) {
	n.debug("adding key %v with value %d", key, value)
	n.store[key] = numVal{
		Time: time.Now(),
		Value: value,
	}
}

func (n *Num) Show(key Key) (int64, error) {
	n.debug("displaying value for key %v", key)
	if v, ok := n.store[key]; ok {
		n.debug("key found")
		return v.Value, nil
	}

	return 0, fmt.Errorf("key %s not found", key)
}

func (n *Num) All() [][]string {
	n.debug("displaying all values (%d)", len(n.store))
	vals := [][]string{}
	for k, v := range n.store {
		vals = append(vals, []string{
			string(k),
			strings.TrimSpace(strconv.FormatInt(v.Value, 10)),
			"number",
			v.Time.Format(time.RFC822),
		})
	}

	return vals
}

func (n *Num) Keys() []Key {
	n.debug("getting all keys for store (len: %d)", len(n.store))
	keys := make([]Key, 0, len(n.store))
	for k := range n.store {
		keys = append(keys, k)
	}

	return keys
}

func (n *Num) Count() int {
	n.debug("getting count of items in store")
	return len(n.store)
}

func (n *Num) Del(key Key) error {
	n.debug("removing value for key %v", key)
	if _, ok := n.store[key]; ok {
		n.debug("key found")
		delete(n.store, key)
		return nil
	}

	return fmt.Errorf("key %s not found", key)
}

func NewNum() *Num {
	return &Num{
		store: make(map[Key]numVal),
		debug: debug.Debug("mini:number"),
	}
}
<|endoftext|>


<commit_before>package main

import "fmt"
//import "bitbucket.org/davidwallace/go-opc/colorutils"

func main() {

    const N_PIXELS = 10

    var array = make([]int, N_PIXELS*3)

    var pct, r, g, b float64
    for ii := 0; ii < N_PIXELS; ii++ {
        pct = float64(ii) / N_PIXELS
        r = pct
        g = pct
        b = pct
        array[ii*3 + 0] = int(r*255)
        array[ii*3 + 1] = int(g*255)
        array[ii*3 + 2] = int(b*255)
    }
    for ii,v := range array {
        fmt.Printf("array[%d] = %d\n", ii, v)
    }

}
<commit_msg>print fps<commit_after>package main

import "fmt"
import "time"
import "bitbucket.org/davidwallace/go-opc/colorutils"

func main() {

	const N_PIXELS = 1000
	var array = make([]byte, N_PIXELS*3)

	var pct, r, g, b, t float64
	var last_print = float64(time.Now().UnixNano()) / 1.0e9
	var frames = 0
	for true {
		t = float64(time.Now().UnixNano()) / 1.0e9
		frames += 1
		if t > last_print+1 {
			last_print = t
			fmt.Printf("%f ms (%d fps)\n", 1000.0/float64(frames), frames)
			frames = 0
		}
		for ii := 0; ii < N_PIXELS; ii++ {
			pct = float64(ii) / N_PIXELS

			r = pct
			g = pct
			b = pct

			array[ii*3+0] = colorutils.FloatToByte(r)
			array[ii*3+1] = colorutils.FloatToByte(g)
			array[ii*3+2] = colorutils.FloatToByte(b)
		}

		//for ii, v := range array {
		//    fmt.Printf("array[%d] = %d\n", ii, v)
		//}
	}

}
<|endoftext|>


<commit_before>package borm

import (
	"errors"
	"fmt"
	"reflect"
	"time"

	"github.com/boltdb/bolt"
	"github.com/vtg/pubsub"
)

// Events listener
var Events = *pubsub.Hub

// Params for List query
type Params struct {
	Offset  int
	Limit   int
	Reverse bool
}

func init() {
	Events.Start(4)
}

// DB
type DB struct {
	File string
	Log  bool

	db   *bolt.DB
	open bool
}

// Open opens database
func Open(dbfile string) (db DB, err error) {
	db.db, err = bolt.Open(dbfile, 0600, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		return
	}
	db.open = true
	db.File = dbfile
	return
}

// Close closing database
func (db *DB) Close() {
	db.open = false
	db.db.Close()
}

// Find returns model from database
// 		m := Model{}
// 		db.Find([]string{"bucket"}, &m)
func (db *DB) Find(buckets []string, id string, i interface{}) error {
	l := logit(db.Log, "FIND", buckets, id, nil)
	err := db.find(buckets, id, i)
	return l.done(err)
}

func (db *DB) find(buckets []string, id string, i interface{}) error {
	if err := db.check(buckets); err != nil {
		return err
	}

	return db.db.View(func(tx *bolt.Tx) error {
		var err error
		b := getBucket(tx, buckets)
		if b == nil {
			return errors.New("Bucket not found")
		}

		k := []byte(id)
		if err = unmarshal(b.Get(k), i); err != nil {
			return err
		}
		return nil
	})
}

// GET returns value by key
// 		val, err := db.FindValue([]string{"bucket"}, "1")
func (db *DB) Get(buckets []string, key string) ([]byte, error) {
	l := logit(db.Log, "GET", buckets, key, nil)
	v, err := db.get(buckets, key)
	return v, l.done(err)
}

func (db *DB) get(buckets []string, key string) ([]byte, error) {
	if err := db.check(buckets); err != nil {
		return nil, err
	}

	var v []byte
	err := db.db.View(func(tx *bolt.Tx) error {
		b := getBucket(tx, buckets)
		if b == nil {
			return errors.New("Bucket not found")
		}
		v = b.Get([]byte(key))
		return nil
	})
	return v, err
}

// Save saves model into database
// 		m := Model{Name: "Model Name"}
// 		db.Save([]string{"bucket"}, &m)
func (db *DB) Save(buckets []string, m mod) error {
	l := logit(db.Log, "SAVE", buckets, "", m)
	err := db.save(buckets, m)
	return l.done(err)
}

func (db *DB) save(buckets []string, m mod) error {
	if err := db.check(buckets); err != nil {
		return err
	}

	return db.db.Update(func(tx *bolt.Tx) error {
		b, err := createBucket(tx, buckets)
		if err != nil {
			return fmt.Errorf("create bucket: %s", err)
		}

		id, newItem := checkID(b, m)

		enc, err := marshal(m)
		if err != nil {
			return fmt.Errorf("could not encode %s: %s", id, err)
		}

		if err := b.Put([]byte(id), enc); err != nil {
			return err
		}

		if newItem {
			addEvent("Created", m)
		} else {
			addEvent("Updated", m)
		}
		return nil
	})
}

// SaveValue saves key/value pair into database
func (db *DB) SaveValue(buckets []string, id string, val []byte) error {
	l := logit(db.Log, "SAVE-VALUE", buckets, "", val)
	err := db.saveValue(buckets, id, val)
	return l.done(err)
}

func (db *DB) saveValue(buckets []string, id string, val []byte) error {
	if err := db.check(buckets); err != nil {
		return err
	}

	return db.db.Update(func(tx *bolt.Tx) error {
		b, err := createBucket(tx, buckets)
		if err != nil {
			return fmt.Errorf("create bucket: %s", err)
		}
		return b.Put([]byte(id), val)
	})
}

// Delete deletes model from database
// 		m := Model{}
// 		db.Find([]string{"bucket"}, &m)
// 		db.Delete([]string{"bucket"}, &m)
func (db *DB) Delete(buckets []string, m mod) error {
	err := db.DeleteKeys(buckets, []string{m.GetID()})
	if err == nil {
		addEvent("Deleted", m)
	}
	return err
}

// DeleteKeys deletes records from database by keys
// 		db.DeleteKeys([]string{"bucket"}, []string{"1","2","3"})
func (db *DB) DeleteKeys(buckets []string, keys []string) error {
	l := logit(db.Log, "Delete", buckets, "", keys)
	err := db.deleteKeys(buckets, keys)
	return l.done(err)
}

func (db *DB) deleteKeys(buckets []string, keys []string) error {
	if err := db.check(buckets); err != nil {
		return err
	}

	return db.db.Update(func(tx *bolt.Tx) error {
		b := getBucket(tx, buckets)
		if b == nil {
			return fmt.Errorf("Bucket not found")
		}
		for _, v := range keys {
			b.Delete([]byte(v))
		}
		return nil
	})
}

// DeleteBuckets deletes records from database by keys
// 		db.DeleteBuckets([]string{"bucket"}, []string{"1","2","3"})
func (db *DB) DeleteBuckets(buckets []string, keys []string) error {
	l := logit(db.Log, "DELETE BUCKET", buckets, "", keys)
	err := db.deleteBuckets(buckets, keys)
	return l.done(err)
}

func (db *DB) deleteBuckets(buckets []string, keys []string) error {
	if err := db.check(buckets); err != nil {
		return err
	}

	return db.db.Update(func(tx *bolt.Tx) error {
		b := getBucket(tx, buckets)
		if b == nil {
			return fmt.Errorf("Bucket not found")
		}
		for _, v := range keys {
			if err := b.DeleteBucket([]byte(v)); err != nil {
				return err
			}
		}
		return nil
	})
}

// List fills models slice with records from database
// 		m := []Model{}
// 		db.List([]string{"bucket"}, &m)
// load with params
// 		db.List([]string{"bucket"}, &m, Params{Offset: 10, Limit: 30})
func (db *DB) List(buckets []string, dest interface{}, params ...Params) error {
	l := logit(db.Log, "LIST", buckets, "", params)
	err := db.list(buckets, dest, params...)
	return l.done(err)
}

func (db *DB) list(buckets []string, dest interface{}, params ...Params) error {
	if err := db.check(buckets); err != nil {
		return err
	}

	opts := parseParams(params)

	return db.db.View(func(tx *bolt.Tx) error {
		b := getBucket(tx, buckets)
		if b == nil {
			return fmt.Errorf("Bucket not found")
		}

		v := reflect.ValueOf(dest)
		if v.Kind() != reflect.Ptr {
			return errors.New("expected pointer but value passed")
		}
		if v.IsNil() {
			return errors.New("nil pointer passed")
		}

		d := reflect.Indirect(v)
		slice, err := baseType(v.Type(), reflect.Slice)
		if err != nil {
			return err
		}

		ptr := slice.Elem().Kind() == reflect.Ptr
		tp := deref(slice.Elem())

		c := b.Cursor()
		i := 0
		for k, v := cursorStart(c, opts.Reverse); k != nil; k, v = cursorNext(c, opts.Reverse) {
			if v == nil || i < opts.Offset || i > opts.Offset+opts.Limit {
				continue
			}
			i++
			item := reflect.New(tp)
			if err := unmarshal(v, item.Interface()); err != nil && err.Error() != "unexpected end of JSON input" {
				return err
			}

			if ptr {
				d.Set(reflect.Append(d, item))
			} else {
				d.Set(reflect.Append(d, reflect.Indirect(item)))
			}
		}
		return nil
	})
}

// ListKeys fills models slice with records by keys provided
// 		m := []Model{}
// 		db.ListKeys([]string{"bucket"}, [][]byte{[]byte("1"),[]byte("2")}, &m)
func (db *DB) ListKeys(buckets []string, keys [][]byte, dest interface{}) error {
	l := logit(db.Log, "LISTKEYS", buckets, "", nil)
	err := db.listKeys(buckets, keys, dest)
	return l.done(err)
}

func (db *DB) listKeys(buckets []string, keys [][]byte, dest interface{}) error {
	if err := db.check(buckets); err != nil {
		return err
	}

	return db.db.View(func(tx *bolt.Tx) error {
		b := getBucket(tx, buckets)
		if b == nil {
			return fmt.Errorf("Bucket not found")
		}

		v := reflect.ValueOf(dest)
		if v.Kind() != reflect.Ptr {
			return errors.New("expected pointer but value passed")
		}
		if v.IsNil() {
			return errors.New("nil pointer passed")
		}

		d := reflect.Indirect(v)
		slice, err := baseType(v.Type(), reflect.Slice)
		if err != nil {
			return err
		}

		ptr := slice.Elem().Kind() == reflect.Ptr
		tp := deref(slice.Elem())

		for _, key := range keys {
			v := b.Get(key)
			if v == nil {
				continue
			}
			item := reflect.New(tp)
			if err := unmarshal(v, item.Interface()); err != nil && err.Error() != "unexpected end of JSON input" {
				return err
			}

			if ptr {
				d.Set(reflect.Append(d, item))
			} else {
				d.Set(reflect.Append(d, reflect.Indirect(item)))
			}
		}
		return nil
	})
}

// ListItems returns raw records from database
func (db *DB) ListItems(buckets []string, params ...Params) (map[string][]byte, error) {
	l := logit(db.Log, "LIST", buckets, "", params)
	res := make(map[string][]byte)
	err := db.listItems(buckets, res, params...)
	return res, l.done(err)
}

func (db *DB) listItems(buckets []string, res map[string][]byte, params ...Params) error {
	if err := db.check(buckets); err != nil {
		return err
	}

	opts := parseParams(params)

	err := db.db.View(func(tx *bolt.Tx) error {
		b := getBucket(tx, buckets)
		if b == nil {
			return fmt.Errorf("Bucket not found")
		}

		i := 0
		c := b.Cursor()
		for k, v := cursorStart(c, opts.Reverse); k != nil; k, v = cursorNext(c, opts.Reverse) {
			if i < opts.Offset || i > opts.Offset+opts.Limit {
				continue
			}
			i++
			res[string(k)] = v
		}

		return nil
	})
	return err
}

// Values returns values from bucket
func (db *DB) Values(buckets []string, params ...Params) ([][]byte, error) {
	l := logit(db.Log, "VALUES", buckets, "", params)
	res, err := db.values(buckets, params...)
	return res, l.done(err)
}

func (db *DB) values(buckets []string, params ...Params) (res [][]byte, err error) {
	if err = db.check(buckets); err != nil {
		return
	}

	opts := parseParams(params)

	err = db.db.View(func(tx *bolt.Tx) error {
		b := getBucket(tx, buckets)
		if b == nil {
			return fmt.Errorf("Bucket not found")
		}

		i := 0
		c := b.Cursor()
		for k, v := cursorStart(c, opts.Reverse); k != nil; k, v = cursorNext(c, opts.Reverse) {
			if i < opts.Offset || i > opts.Offset+opts.Limit {
				continue
			}
			i++
			res = append(res, v)
		}

		return nil
	})
	return
}

// Count returns number of records in bucket
func (db *DB) Count(buckets []string) int {
	res := 0

	if err := db.check(buckets); err != nil {
		return res
	}

	db.db.View(func(tx *bolt.Tx) error {
		b := getBucket(tx, buckets)
		if b == nil {
			return fmt.Errorf("Bucket not found")
		}
		res = b.Stats().KeyN
		return nil
	})
	return res
}

func (d *DB) check(buckets []string) error {
	if !db.open {
		return fmt.Errorf("db is not opened")
	}
	if len(buckets) == 0 {
		return errors.New("No bucket provided")
	}
	return nil
}

func parseParams(p []Params) Params {
	r := Params{
		Offset: 0,
		Limit:  1000,
	}

	if len(p) > 0 {
		r = p[0]
	}
	return r
}
<commit_msg>var names cleanup<commit_after>package borm

import (
	"errors"
	"fmt"
	"reflect"
	"time"

	"github.com/boltdb/bolt"
	"github.com/vtg/pubsub"
)

// Events listener
var Events = *pubsub.Hub

// Params for List query
type Params struct {
	Offset  int
	Limit   int
	Reverse bool
}

func init() {
	Events.Start(4)
}

// DB
type DB struct {
	File string
	Log  bool

	db   *bolt.DB
	open bool
}

// Open opens database
func Open(dbfile string) (db DB, err error) {
	db.db, err = bolt.Open(dbfile, 0600, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		return
	}
	db.open = true
	db.File = dbfile
	return
}

// Close closing database
func (db *DB) Close() {
	db.open = false
	db.db.Close()
}

// Find returns model from database
// 		m := Model{}
// 		db.Find([]string{"bucket"}, &m)
func (db *DB) Find(path []string, id string, i interface{}) error {
	l := logit(db.Log, "FIND", path, id, nil)
	err := db.find(path, id, i)
	return l.done(err)
}

func (db *DB) find(path []string, id string, i interface{}) error {
	if err := db.check(path); err != nil {
		return err
	}

	return db.db.View(func(tx *bolt.Tx) error {
		var err error
		b := getBucket(tx, path)
		if b == nil {
			return errors.New("Bucket not found")
		}

		k := []byte(id)
		if err = unmarshal(b.Get(k), i); err != nil {
			return err
		}
		return nil
	})
}

// GET returns value by key
// 		val, err := db.FindValue([]string{"bucket"}, "1")
func (db *DB) Get(path []string, key string) ([]byte, error) {
	l := logit(db.Log, "GET", path, key, nil)
	v, err := db.get(path, key)
	return v, l.done(err)
}

func (db *DB) get(path []string, key string) ([]byte, error) {
	if err := db.check(path); err != nil {
		return nil, err
	}

	var v []byte
	err := db.db.View(func(tx *bolt.Tx) error {
		b := getBucket(tx, path)
		if b == nil {
			return errors.New("Bucket not found")
		}
		v = b.Get([]byte(key))
		return nil
	})
	return v, err
}

// Save saves model into database
// 		m := Model{Name: "Model Name"}
// 		db.Save([]string{"bucket"}, &m)
func (db *DB) Save(path []string, m mod) error {
	l := logit(db.Log, "SAVE", path, "", m)
	err := db.save(path, m)
	return l.done(err)
}

func (db *DB) save(path []string, m mod) error {
	if err := db.check(path); err != nil {
		return err
	}

	return db.db.Update(func(tx *bolt.Tx) error {
		b, err := createBucket(tx, path)
		if err != nil {
			return fmt.Errorf("create bucket: %s", err)
		}

		id, newItem := checkID(b, m)

		enc, err := marshal(m)
		if err != nil {
			return fmt.Errorf("could not encode %s: %s", id, err)
		}

		if err := b.Put([]byte(id), enc); err != nil {
			return err
		}

		if newItem {
			addEvent("Created", m)
		} else {
			addEvent("Updated", m)
		}
		return nil
	})
}

// SaveValue saves key/value pair into database
func (db *DB) SaveValue(path []string, id string, val []byte) error {
	l := logit(db.Log, "SAVE-VALUE", path, "", val)
	err := db.saveValue(path, id, val)
	return l.done(err)
}

func (db *DB) saveValue(path []string, id string, val []byte) error {
	if err := db.check(path); err != nil {
		return err
	}

	return db.db.Update(func(tx *bolt.Tx) error {
		b, err := createBucket(tx, path)
		if err != nil {
			return fmt.Errorf("create bucket: %s", err)
		}
		return b.Put([]byte(id), val)
	})
}

// Delete deletes model from database
// 		m := Model{}
// 		db.Find([]string{"bucket"}, &m)
// 		db.Delete([]string{"bucket"}, &m)
func (db *DB) Delete(path []string, m mod) error {
	err := db.DeleteKeys(path, []string{m.GetID()})
	if err == nil {
		addEvent("Deleted", m)
	}
	return err
}

// DeleteKeys deletes records from database by keys
// 		db.DeleteKeys([]string{"bucket"}, []string{"1","2","3"})
func (db *DB) DeleteKeys(path []string, keys []string) error {
	l := logit(db.Log, "Delete", path, "", keys)
	err := db.deleteKeys(path, keys)
	return l.done(err)
}

func (db *DB) deleteKeys(path []string, keys []string) error {
	if err := db.check(path); err != nil {
		return err
	}

	return db.db.Update(func(tx *bolt.Tx) error {
		b := getBucket(tx, path)
		if b == nil {
			return fmt.Errorf("Bucket not found")
		}
		for _, v := range keys {
			b.Delete([]byte(v))
		}
		return nil
	})
}

// DeleteBuckets deletes records from database by keys
// 		db.DeleteBuckets([]string{"bucket"}, []string{"1","2","3"})
func (db *DB) DeleteBuckets(path []string, keys []string) error {
	l := logit(db.Log, "DELETE BUCKET", path, "", keys)
	err := db.deleteBuckets(path, keys)
	return l.done(err)
}

func (db *DB) deleteBuckets(path []string, keys []string) error {
	if err := db.check(path); err != nil {
		return err
	}

	return db.db.Update(func(tx *bolt.Tx) error {
		b := getBucket(tx, path)
		if b == nil {
			return fmt.Errorf("Bucket not found")
		}
		for _, v := range keys {
			if err := b.DeleteBucket([]byte(v)); err != nil {
				return err
			}
		}
		return nil
	})
}

// List fills models slice with records from database
// 		m := []Model{}
// 		db.List([]string{"bucket"}, &m)
// load with params
// 		db.List([]string{"bucket"}, &m, Params{Offset: 10, Limit: 30})
func (db *DB) List(path []string, dest interface{}, params ...Params) error {
	l := logit(db.Log, "LIST", path, "", params)
	err := db.list(path, dest, params...)
	return l.done(err)
}

func (db *DB) list(path []string, dest interface{}, params ...Params) error {
	if err := db.check(path); err != nil {
		return err
	}

	opts := parseParams(params)

	return db.db.View(func(tx *bolt.Tx) error {
		b := getBucket(tx, path)
		if b == nil {
			return fmt.Errorf("Bucket not found")
		}

		v := reflect.ValueOf(dest)
		if v.Kind() != reflect.Ptr {
			return errors.New("expected pointer but value passed")
		}
		if v.IsNil() {
			return errors.New("nil pointer passed")
		}

		d := reflect.Indirect(v)
		slice, err := baseType(v.Type(), reflect.Slice)
		if err != nil {
			return err
		}

		ptr := slice.Elem().Kind() == reflect.Ptr
		tp := deref(slice.Elem())

		c := b.Cursor()
		i := 0
		for k, v := cursorStart(c, opts.Reverse); k != nil; k, v = cursorNext(c, opts.Reverse) {
			if v == nil || i < opts.Offset || i > opts.Offset+opts.Limit {
				continue
			}
			i++
			item := reflect.New(tp)
			if err := unmarshal(v, item.Interface()); err != nil && err.Error() != "unexpected end of JSON input" {
				return err
			}

			if ptr {
				d.Set(reflect.Append(d, item))
			} else {
				d.Set(reflect.Append(d, reflect.Indirect(item)))
			}
		}
		return nil
	})
}

// ListKeys fills models slice with records by keys provided
// 		m := []Model{}
// 		db.ListKeys([]string{"bucket"}, [][]byte{[]byte("1"),[]byte("2")}, &m)
func (db *DB) ListKeys(path []string, keys [][]byte, dest interface{}) error {
	l := logit(db.Log, "LISTKEYS", path, "", nil)
	err := db.listKeys(path, keys, dest)
	return l.done(err)
}

func (db *DB) listKeys(path []string, keys [][]byte, dest interface{}) error {
	if err := db.check(path); err != nil {
		return err
	}

	return db.db.View(func(tx *bolt.Tx) error {
		b := getBucket(tx, path)
		if b == nil {
			return fmt.Errorf("Bucket not found")
		}

		v := reflect.ValueOf(dest)
		if v.Kind() != reflect.Ptr {
			return errors.New("expected pointer but value passed")
		}
		if v.IsNil() {
			return errors.New("nil pointer passed")
		}

		d := reflect.Indirect(v)
		slice, err := baseType(v.Type(), reflect.Slice)
		if err != nil {
			return err
		}

		ptr := slice.Elem().Kind() == reflect.Ptr
		tp := deref(slice.Elem())

		for _, key := range keys {
			v := b.Get(key)
			if v == nil {
				continue
			}
			item := reflect.New(tp)
			if err := unmarshal(v, item.Interface()); err != nil && err.Error() != "unexpected end of JSON input" {
				return err
			}

			if ptr {
				d.Set(reflect.Append(d, item))
			} else {
				d.Set(reflect.Append(d, reflect.Indirect(item)))
			}
		}
		return nil
	})
}

// ListItems returns raw records from database
func (db *DB) ListItems(path []string, params ...Params) (map[string][]byte, error) {
	l := logit(db.Log, "LIST", path, "", params)
	res := make(map[string][]byte)
	err := db.listItems(path, res, params...)
	return res, l.done(err)
}

func (db *DB) listItems(path []string, res map[string][]byte, params ...Params) error {
	if err := db.check(path); err != nil {
		return err
	}

	opts := parseParams(params)

	err := db.db.View(func(tx *bolt.Tx) error {
		b := getBucket(tx, path)
		if b == nil {
			return fmt.Errorf("Bucket not found")
		}

		i := 0
		c := b.Cursor()
		for k, v := cursorStart(c, opts.Reverse); k != nil; k, v = cursorNext(c, opts.Reverse) {
			if i < opts.Offset || i > opts.Offset+opts.Limit {
				continue
			}
			i++
			res[string(k)] = v
		}

		return nil
	})
	return err
}

// Values returns values from bucket
func (db *DB) Values(path []string, params ...Params) ([][]byte, error) {
	l := logit(db.Log, "VALUES", path, "", params)
	res, err := db.values(path, params...)
	return res, l.done(err)
}

func (db *DB) values(path []string, params ...Params) (res [][]byte, err error) {
	if err = db.check(path); err != nil {
		return
	}

	opts := parseParams(params)

	err = db.db.View(func(tx *bolt.Tx) error {
		b := getBucket(tx, path)
		if b == nil {
			return fmt.Errorf("Bucket not found")
		}

		i := 0
		c := b.Cursor()
		for k, v := cursorStart(c, opts.Reverse); k != nil; k, v = cursorNext(c, opts.Reverse) {
			if i < opts.Offset || i > opts.Offset+opts.Limit {
				continue
			}
			i++
			res = append(res, v)
		}

		return nil
	})
	return
}

// Count returns number of records in bucket
func (db *DB) Count(path []string) int {
	res := 0

	if err := db.check(path); err != nil {
		return res
	}

	db.db.View(func(tx *bolt.Tx) error {
		b := getBucket(tx, path)
		if b == nil {
			return fmt.Errorf("Bucket not found")
		}
		res = b.Stats().KeyN
		return nil
	})
	return res
}

func (d *DB) check(path []string) error {
	if !db.open {
		return fmt.Errorf("db is not opened")
	}
	if len(path) == 0 {
		return errors.New("No bucket provided")
	}
	return nil
}

func parseParams(p []Params) Params {
	r := Params{
		Offset: 0,
		Limit:  1000,
	}

	if len(p) > 0 {
		r = p[0]
	}
	return r
}
<|endoftext|>


<commit_before>// pst is a command line tool for processing and combining columns across
// column oriented files
package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"runtime/debug"
	"strconv"
	"strings"
	"unicode"
)

// vars for the command line parser
var (
	inputSpec    string
	computeStats bool
)

// outData collects a row oriented list of column entries
type outData []string

// parseSpec describes for each input files which columns to parse
type parseSpec []int

func init() {
	flag.StringVar(&inputSpec, "e", "",
		`specify the input columns to extract.
     The spec format is "<column list file1>|<column list file2>|..."
     where each column specifier is of the form col_i,col_j,col_k-col_n, ....
     If the number of specifiers is less than the number of files, the last
     specifier i will be applied to files i through N, where N is the total
     number of files provided.`)
	flag.BoolVar(&computeStats, "s", false,
		`compute statistics across column values in each output row.
     Please note that each value in the output has to be convertible into a float
     for this to work. Currently the mean and standard deviation are computed`)
}

func main() {
	flag.Parse()
	if len(flag.Args()) < 1 || inputSpec == "" {
		usage()
		os.Exit(1)
	}

	colSpecs, err := parseInputSpec(inputSpec)
	if err != nil {
		log.Fatal(err)
	}

	if len(colSpecs) > len(flag.Args()) {
		log.Fatal("there are more per file column specifiers than supplied input files")
	}

	// read input files and assemble output
	output, err := readData(flag.Args(), colSpecs)
	if err != nil {
		log.Fatal(err)
	}

	// compute statistics or punch the data otherwise
	if computeStats == true {
		for _, row := range output {

			items, err := splitIntoFloats(row)
			if err != nil {
				log.Fatal(err)
			}
			fmt.Println(mean(items), variance(items))
		}
	} else {
		for _, row := range output {
			fmt.Println(row)
		}
	}
}

// parseFile reads the passed in file, extracts the columns requested per spec
// and the returns a slice with the requested column info.
func parseFile(fileName string, spec parseSpec) (outData, error) {

	file, err := os.Open(fileName)
	if err != nil {
		return nil, err
	}

	var out outData
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		items := strings.FieldsFunc(strings.TrimSpace(scanner.Text()), unicode.IsSpace)
		//row := make([]string, 0, len(spec))
		var row string
		for _, i := range spec {
			if i >= len(items) {
				return nil, fmt.Errorf("error parsing file %s: requested column %d "+
					"does not exist", fileName, i)
			}
			row += items[i]
			row += " "
		}
		out = append(out, row)
	}
	return out, nil
}

// parseInputSpec parses the inputSpec and turns it into a slice of parseSpecs,
// one for each input file
func parseInputSpec(input string) ([]parseSpec, error) {

	// split according to file specs
	fileSpecs := strings.Split(input, "|")

	spec := make([]parseSpec, len(fileSpecs))
	// split according to column specs
	for i, f := range fileSpecs {
		colSpecs := strings.Split(f, ",")
		if len(colSpecs) == 0 {
			return nil, fmt.Errorf("empty input specification for file entry #%d: %s",
				i, f)
		}

		var ps parseSpec
		for _, cr := range colSpecs {
			c := strings.TrimSpace(cr)

			// check for possible range
			colRange := strings.Split(c, "-")

			switch len(colRange) {
			case 1: // no range, simple columns
				cInt, err := strconv.Atoi(c)
				if err != nil {
					return nil, fmt.Errorf("could not convert %s into integer representation", c)
				}
				ps = append(ps, cInt)
			case 2: // range specified via begin and end
				aInt, err := strconv.Atoi(colRange[0])
				if err != nil {
					return nil, fmt.Errorf("could not convert %s into integer representation",
						colRange[0])
				}

				bInt, err := strconv.Atoi(colRange[1])
				if err != nil {
					return nil, fmt.Errorf("could not convert %s into integer representation",
						colRange[1])
				}

				for i := aInt; i < bInt; i++ {
					ps = append(ps, i)
				}
			default:
				return nil, fmt.Errorf("incorrect column range specification %s", c)
			}
		}
		spec[i] = ps
	}
	return spec, nil
}

// readData parses all the output files and populates and returns the output
// data set
func readData(files []string, colSpecs []parseSpec) (outData, error) {

	var output outData
	for i, file := range files {

		// pick the correct specification for parsing columns
		var spec parseSpec
		if i < len(colSpecs) {
			spec = colSpecs[i]
		} else {
			spec = colSpecs[len(colSpecs)-1]
		}

		parsedCols, err := parseFile(file, spec)
		if err != nil {
			log.Fatal(err)
		}

		// initialize output after parsing the first data file
		if i == 0 {
			output = make([]string, len(parsedCols))
		}

		// make sure input files have consistent length
		if len(parsedCols) != len(output) {
			return nil, fmt.Errorf("input file %s has %d rows which differs from %d "+
				"in previous files", file, len(parsedCols), len(output))
		}

		// append parsed data to output
		for i, row := range parsedCols {
			output[i] += row
		}

		// force a GC cycle
		parsedCols = nil
		debug.FreeOSMemory()
	}
	return output, nil
}

// splitIntoFloats splits a string consisting of whitespace separated floats
// into a list of floats.
func splitIntoFloats(floatString string) ([]float64, error) {

	items := strings.FieldsFunc(floatString, unicode.IsSpace)
	var floatList []float64
	for _, item := range items {
		val, err := strconv.ParseFloat(strings.TrimSpace(item), 64)
		if err != nil {
			return nil, err
		}
		floatList = append(floatList, val)
	}
	return floatList, nil
}

// mean computes the mean value of a list of float64 values
func mean(items []float64) float64 {
	var mean float64
	for _, x := range items {
		mean += x
	}
	return mean / float64(len(items))
}

// variance computes the variance of a list of float64 values
func variance(items []float64) float64 {
	var mk, qk float64 // helper values for one pass variance computation
	for i, d := range items {
		k := float64(i + 1)
		qk += (k - 1) * (d - mk) * (d - mk) / k
		mk += (d - mk) / k
	}

	var variance float64
	if len(items) > 1 {
		variance = qk / float64(len(items)-1)
	}
	return variance
}

// usage prints a simple usage/help message
func usage() {
	fmt.Println("pst (C) 2015 M. Dittrich")
	fmt.Println()
	fmt.Println("usage: pst <options> file1 file2 ...")
	fmt.Println()
	fmt.Println("options:")
	flag.PrintDefaults()
}
<commit_msg>Added ability to specify separators for parsing input files.<commit_after>// pst is a command line tool for processing and combining columns across
// column oriented files
package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"runtime/debug"
	"strconv"
	"strings"
	"unicode"
)

// command line switches
var (
	inputSpec    string
	inputSep     string
	computeStats bool
)

// outData collects a row oriented list of column entries
type outData []string

// parseSpec describes for each input files which columns to parse
type parseSpec []int

func init() {
	flag.StringVar(&inputSpec, "e", "",
		`specify the input columns to extract.
     The spec format is "<column list file1>|<column list file2>|..."
     where each column specifier is of the form col_i,col_j,col_k-col_n, ....
     If the number of specifiers is less than the number of files, the last
     specifier i will be applied to files i through N, where N is the total
     number of files provided.`)
	flag.BoolVar(&computeStats, "c", false,
		`compute statistics across column values in each output row.
     Please note that each value in the output has to be convertible into a float
     for this to work. Currently the mean and standard deviation are computed`)
	flag.StringVar(&inputSep, "s", "",
		`column separator for input files. The default separator is whitespace.`)
}

func main() {
	flag.Parse()
	if len(flag.Args()) < 1 || inputSpec == "" {
		usage()
		os.Exit(1)
	}

	inputSepFunc := getInputSepFunc(inputSep)

	colSpecs, err := parseInputSpec(inputSpec)
	if err != nil {
		log.Fatal(err)
	}

	if len(colSpecs) > len(flag.Args()) {
		log.Fatal("there are more per file column specifiers than supplied input files")
	}

	// read input files and assemble output
	output, err := readData(flag.Args(), colSpecs, inputSepFunc)
	if err != nil {
		log.Fatal(err)
	}

	// compute statistics or punch the data otherwise
	if computeStats == true {
		for _, row := range output {

			items, err := splitIntoFloats(row)
			if err != nil {
				log.Fatal(err)
			}
			fmt.Println(mean(items), variance(items))
		}
	} else {
		for _, row := range output {
			fmt.Println(row)
		}
	}
}

// parseFile reads the passed in file, extracts the columns requested per spec
// and the returns a slice with the requested column info.
func parseFile(fileName string, spec parseSpec, sepFun func(rune) bool) (outData, error) {

	file, err := os.Open(fileName)
	if err != nil {
		return nil, err
	}

	var out outData
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		items := strings.FieldsFunc(strings.TrimSpace(scanner.Text()), sepFun)
		var row string
		for _, i := range spec {
			if i >= len(items) {
				return nil, fmt.Errorf("error parsing file %s: requested column %d "+
					"does not exist", fileName, i)
			}
			row += items[i]
			row += " "
		}
		out = append(out, row)
	}
	return out, nil
}

// parseInputSpec parses the inputSpec and turns it into a slice of parseSpecs,
// one for each input file
func parseInputSpec(input string) ([]parseSpec, error) {

	// split according to file specs
	fileSpecs := strings.Split(input, "|")

	spec := make([]parseSpec, len(fileSpecs))
	// split according to column specs
	for i, f := range fileSpecs {
		colSpecs := strings.Split(f, ",")
		if len(colSpecs) == 0 {
			return nil, fmt.Errorf("empty input specification for file entry #%d: %s",
				i, f)
		}

		var ps parseSpec
		for _, cr := range colSpecs {
			c := strings.TrimSpace(cr)

			// check for possible range
			colRange := strings.Split(c, "-")

			switch len(colRange) {
			case 1: // no range, simple columns
				cInt, err := strconv.Atoi(c)
				if err != nil {
					return nil, fmt.Errorf("could not convert %s into integer representation", c)
				}
				ps = append(ps, cInt)
			case 2: // range specified via begin and end
				aInt, err := strconv.Atoi(colRange[0])
				if err != nil {
					return nil, fmt.Errorf("could not convert %s into integer representation",
						colRange[0])
				}

				bInt, err := strconv.Atoi(colRange[1])
				if err != nil {
					return nil, fmt.Errorf("could not convert %s into integer representation",
						colRange[1])
				}

				for i := aInt; i < bInt; i++ {
					ps = append(ps, i)
				}
			default:
				return nil, fmt.Errorf("incorrect column range specification %s", c)
			}
		}
		spec[i] = ps
	}
	return spec, nil
}

// readData parses all the output files and populates and returns the output
// data set
func readData(files []string, colSpecs []parseSpec, sepFun func(rune) bool) (outData, error) {

	var output outData
	for i, file := range files {

		// pick the correct specification for parsing columns
		var spec parseSpec
		if i < len(colSpecs) {
			spec = colSpecs[i]
		} else {
			spec = colSpecs[len(colSpecs)-1]
		}

		parsedCols, err := parseFile(file, spec, sepFun)
		if err != nil {
			log.Fatal(err)
		}

		// initialize output after parsing the first data file
		if i == 0 {
			output = make([]string, len(parsedCols))
		}

		// make sure input files have consistent length
		if len(parsedCols) != len(output) {
			return nil, fmt.Errorf("input file %s has %d rows which differs from %d "+
				"in previous files", file, len(parsedCols), len(output))
		}

		// append parsed data to output
		for i, row := range parsedCols {
			output[i] += row
		}

		// force a GC cycle
		parsedCols = nil
		debug.FreeOSMemory()
	}
	return output, nil
}

// splitIntoFloats splits a string consisting of whitespace separated floats
// into a list of floats.
func splitIntoFloats(floatString string) ([]float64, error) {

	items := strings.FieldsFunc(floatString, unicode.IsSpace)
	var floatList []float64
	for _, item := range items {
		val, err := strconv.ParseFloat(strings.TrimSpace(item), 64)
		if err != nil {
			return nil, err
		}
		floatList = append(floatList, val)
	}
	return floatList, nil
}

// mean computes the mean value of a list of float64 values
func mean(items []float64) float64 {
	var mean float64
	for _, x := range items {
		mean += x
	}
	return mean / float64(len(items))
}

// variance computes the variance of a list of float64 values
func variance(items []float64) float64 {
	var mk, qk float64 // helper values for one pass variance computation
	for i, d := range items {
		k := float64(i + 1)
		qk += (k - 1) * (d - mk) * (d - mk) / k
		mk += (d - mk) / k
	}

	var variance float64
	if len(items) > 1 {
		variance = qk / float64(len(items)-1)
	}
	return variance
}

// getInputSepFunc returns a closure used for separating the columns in the
// input files
func getInputSepFunc(inputSep string) func(rune) bool {
	inputSepFunc := unicode.IsSpace
	if len(inputSep) >= 1 {
		inputSepFunc = func(r rune) bool {
			for _, s := range inputSep {
				if s == r {
					return true
				}
			}
			return false
		}
	}
	return inputSepFunc
}

// usage prints a simple usage/help message
func usage() {
	fmt.Println("pst             (C) 2015 M. Dittrich")
	fmt.Println()
	fmt.Println("usage: pst <options> file1 file2 ...")
	fmt.Println()
	fmt.Println("options:")
	flag.PrintDefaults()
}
<|endoftext|>


<commit_before>package ddns

import (
	"log"
	"net"
	"strings"
	"github.com/stutiredboy/radix.v2/pool"
)

func (s *Server) logq2b(name string, addr net.Addr, pool *pool.Pool) error {
	conn, err := pool.Get()
	if err != nil {
		log.Printf("q2b get conn: %s", err)
		return err
	}
	defer pool.Put(conn)

	name = strings.TrimSuffix(name, ".")
	clientip, _, err := net.SplitHostPort(addr.String())
	if err != nil {
		return err
	}
	err = conn.Cmd("SETEX", name, 120, clientip).Err
	if err != nil {
		log.Printf("q2b setex: %s", err)
	}
	return err
}
<commit_msg>logger more<commit_after>package ddns

import (
	"log"
	"net"
	"strings"
	"github.com/stutiredboy/radix.v2/pool"
)

func (s *Server) logq2b(name string, addr net.Addr, pool *pool.Pool) error {
	conn, err := pool.Get()
	if err != nil {
		log.Printf("q2b get conn raise err: %s", err)
		return err
	}
	defer pool.Put(conn)

	name = strings.TrimSuffix(name, ".")
	clientip, _, err := net.SplitHostPort(addr.String())
	if err != nil {
		return err
	}
	err = conn.Cmd("SETEX", name, 120, clientip).Err
	if err != nil {
		log.Printf("q2b setex %s as %s raise err: %s", name, clientip, err)
	}
	return err
}
<|endoftext|>


<commit_before>package keyctl

import (
	"bytes"
	"errors"
	"os"
	"strconv"
)

var (
	// Error returned if the Get() method is called on a Reference that doesn't
	// represent a key or keychain.
	ErrUnsupportedKeyType = errors.New("unsupported keyctl key type")
	// Error returned if a reference is stale when Info() or Get() is called on
	// it.
	ErrInvalidReference = errors.New("invalid keyctl reference")
)

// Reference is a reference to an unloaded keyctl Key or Keychain. It can be
// dereferenced by calling the Get() method.
type Reference struct {
	// Id is the kernel key or keychain identifier referenced.
	Id int32

	info   *Info
	parent keyId
}

// Information about a keyctl reference as returned by ref.Info()
type Info struct {
	Type, Name string
	Uid, Gid   int
	Perm       KeyPerm

	valid bool
}

func getInfo(id keyId) (i Info, err error) {
	var desc []byte

	if desc, err = describeKeyId(id); err != nil {
		i.Name = err.Error()
		return
	}

	fields := bytes.Split(desc, []byte{';'})
	switch len(fields) {
	case 5:
		i.Name = string(fields[4])
		fallthrough
	case 4:
		p, _ := strconv.ParseUint(string(fields[3]), 16, 32)
		i.Perm = KeyPerm(p)
		fallthrough
	case 3:
		i.Gid, _ = strconv.Atoi(string(fields[2]))
		fallthrough
	case 2:
		i.Uid, _ = strconv.Atoi(string(fields[1]))
		fallthrough
	case 1:
		if i.Type = string(fields[0]); i.Type == "user" {
			i.Type = "key"
		}
		i.valid = true
	default:
		panic("invalid field count from kernel keyctl describe sysctl")
	}
	return
}

// Returns permissions in symbolic format.
func (i Info) Permissions() string {
	if i.Uid == os.Geteuid() {
		return encodePerms(uint8(i.Perm >> KeyPerm(16)))
	} else {
		fsgid, err := getfsgid()
		if (err == nil && i.Gid == int(fsgid)) || i.Gid == os.Getegid() {
			return encodePerms(uint8(i.Perm >> KeyPerm(8)))
		}
	}
	return encodePerms(uint8(i.Perm))
}

// Return Information about a keyctl reference.
func (r *Reference) Info() (i Info, err error) {
	if r.info == nil {
		i, err = getInfo(keyId(r.Id))
		r.info = &i
		return
	}

	return *r.info, err
}

// Returns true if the Info fetched by ref.Info() is valid.
func (i Info) Valid() bool {
	return i.valid
}

// Returns true if the keyctl reference is valid. Refererences can become
// invalid if they have expired since the reference was created.
func (r *Reference) Valid() bool {
	if r.info == nil {
		r.Info()
	}
	return r.info.valid
}

// Loads the referenced keyctl object, which must either be a key or a
// keyring otherwise ErrUnsupportedKeyType will be returned.
func (r *Reference) Get() (Id, error) {
	if r.info == nil {
		_, err := r.Info()
		if err != nil {
			return nil, err
		}
	}

	if !r.info.valid {
		return nil, ErrInvalidReference
	}

	switch r.info.Type {
	case "key":
		return &Key{Name: r.info.Name, id: keyId(r.Id), ring: r.parent}, nil
	case "keyring":
		ring := &keyring{id: keyId(r.Id)}
		if r.Id > 0 && r.info.Name != "" {
			return &namedKeyring{
				keyring: ring,
				parent:  r.parent,
				name:    r.info.Name,
			}, nil
		}
		return ring, nil
	default:
		return nil, ErrUnsupportedKeyType
	}
}

// List the contents of a keyring. Each contained object is represented by a
// Reference struct. Addl information is available by calling ref.Info(), and
// contained objects which are keys or subordinate keyrings can be fetched by
// calling ref.Get()
func ListKeyring(kr Keyring) ([]Reference, error) {
	id := keyId(kr.Id())
	keys, err := listKeys(id)
	if err != nil {
		return nil, err
	}

	refs := make([]Reference, len(keys))

	for i, k := range keys {
		refs[i].Id, refs[i].parent = int32(k), id
	}

	return refs, nil
}
<commit_msg>Add big_key support<commit_after>package keyctl

import (
	"bytes"
	"errors"
	"os"
	"strconv"
)

var (
	// Error returned if the Get() method is called on a Reference that doesn't
	// represent a key or keychain.
	ErrUnsupportedKeyType = errors.New("unsupported keyctl key type")
	// Error returned if a reference is stale when Info() or Get() is called on
	// it.
	ErrInvalidReference = errors.New("invalid keyctl reference")
)

// Reference is a reference to an unloaded keyctl Key or Keychain. It can be
// dereferenced by calling the Get() method.
type Reference struct {
	// Id is the kernel key or keychain identifier referenced.
	Id int32

	info   *Info
	parent keyId
}

// Information about a keyctl reference as returned by ref.Info()
type Info struct {
	Type, Name string
	Uid, Gid   int
	Perm       KeyPerm

	valid bool
}

func getInfo(id keyId) (i Info, err error) {
	var desc []byte

	if desc, err = describeKeyId(id); err != nil {
		i.Name = err.Error()
		return
	}

	fields := bytes.Split(desc, []byte{';'})
	switch len(fields) {
	case 5:
		i.Name = string(fields[4])
		fallthrough
	case 4:
		p, _ := strconv.ParseUint(string(fields[3]), 16, 32)
		i.Perm = KeyPerm(p)
		fallthrough
	case 3:
		i.Gid, _ = strconv.Atoi(string(fields[2]))
		fallthrough
	case 2:
		i.Uid, _ = strconv.Atoi(string(fields[1]))
		fallthrough
	case 1:
		if i.Type = string(fields[0]); i.Type == "user" {
			i.Type = "key"
		}
		i.valid = true
	default:
		panic("invalid field count from kernel keyctl describe sysctl")
	}
	return
}

// Returns permissions in symbolic format.
func (i Info) Permissions() string {
	if i.Uid == os.Geteuid() {
		return encodePerms(uint8(i.Perm >> KeyPerm(16)))
	} else {
		fsgid, err := getfsgid()
		if (err == nil && i.Gid == int(fsgid)) || i.Gid == os.Getegid() {
			return encodePerms(uint8(i.Perm >> KeyPerm(8)))
		}
	}
	return encodePerms(uint8(i.Perm))
}

// Return Information about a keyctl reference.
func (r *Reference) Info() (i Info, err error) {
	if r.info == nil {
		i, err = getInfo(keyId(r.Id))
		r.info = &i
		return
	}

	return *r.info, err
}

// Returns true if the Info fetched by ref.Info() is valid.
func (i Info) Valid() bool {
	return i.valid
}

// Returns true if the keyctl reference is valid. Refererences can become
// invalid if they have expired since the reference was created.
func (r *Reference) Valid() bool {
	if r.info == nil {
		r.Info()
	}
	return r.info.valid
}

// Loads the referenced keyctl object, which must either be a key or a
// keyring otherwise ErrUnsupportedKeyType will be returned.
func (r *Reference) Get() (Id, error) {
	if r.info == nil {
		_, err := r.Info()
		if err != nil {
			return nil, err
		}
	}

	if !r.info.valid {
		return nil, ErrInvalidReference
	}

	switch r.info.Type {
	case "key", "big_key":
		return &Key{Name: r.info.Name, id: keyId(r.Id), ring: r.parent}, nil
	case "keyring":
		ring := &keyring{id: keyId(r.Id)}
		if r.Id > 0 && r.info.Name != "" {
			return &namedKeyring{
				keyring: ring,
				parent:  r.parent,
				name:    r.info.Name,
			}, nil
		}
		return ring, nil
	default:
		return nil, ErrUnsupportedKeyType
	}
}

// List the contents of a keyring. Each contained object is represented by a
// Reference struct. Addl information is available by calling ref.Info(), and
// contained objects which are keys or subordinate keyrings can be fetched by
// calling ref.Get()
func ListKeyring(kr Keyring) ([]Reference, error) {
	id := keyId(kr.Id())
	keys, err := listKeys(id)
	if err != nil {
		return nil, err
	}

	refs := make([]Reference, len(keys))

	for i, k := range keys {
		refs[i].Id, refs[i].parent = int32(k), id
	}

	return refs, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Alex Fluter

package lotsawa

import (
	"net"
	"net/rpc"
	"time"
)

type CompileArgs struct {
	// The code to compile
	Code string

	// The language of the code
	Lang string
}

type CompileReply struct {
	// The command line used to compile this piece of code
	Cmd string
	// Whether the code has main function and can be executed
	Main bool
	// Errors during the compiling
	Error string
	// Time took to compile and run the program
	Time time.Duration
	// Compiler's standard output
	C_Output string
	// Compiler's standard error
	C_Error string
	// The program's standard error, if compiled successfully and had main
	P_Output string
	// The program's standard output, if compiled successfully and had main
	P_Error string
}

// RPC service
type CompileService struct {
	server *CompilerServer
}

func NewCompileService(server *CompilerServer) *CompileService {
	return &CompileService{
		server: server,
	}
}

func (c *CompileService) Compile(args *CompileArgs, reply *CompileReply) error {
	req := &Request{
		received: time.Now(),
		args:     args,
		chRes:    make(chan *Result),
	}

	c.server.Submit(req)

	res := <-req.chRes
	res.done = time.Now()

	reply.Time = res.done.Sub(req.received)

	reply.Error, reply.Cmd, reply.Main = res.err, res.cmd, res.main
	reply.C_Output, reply.C_Error = res.c_out, res.c_err
	if res.main {
		reply.P_Output, reply.P_Error = res.p_out, res.p_err
	}

	return nil
}

// RPC stub for the client
type CompileServiceStub struct {
	client *rpc.Client
}

func NewCompileServiceStub(net, addr string) (*CompileServiceStub, error) {
	var err error
	s := new(CompileServiceStub)
	s.client, err = rpc.Dial(net, addr)
	if err != nil {
		return nil, err
	}
	return s, nil
}

func (c *CompileServiceStub) Compile(args *CompileArgs, reply *CompileReply) error {
	var err error

	err = c.client.Call("CompileService.Compile", args, reply)
	return err
}

// RPC server
type RpcServer struct {
	svr    *rpc.Server
	l      *net.TCPListener
	addr   *net.TCPAddr
	server *Server
}

func NewRpcServer(server *Server, addr string) (*RpcServer, error) {
	var err error
	s := new(RpcServer)
	s.svr = rpc.NewServer()
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}
	s.addr = tcpAddr
	s.server = server

	return s, nil
}

func (s *RpcServer) Init() error {
	l, err := net.ListenTCP("tcp", s.addr)
	if err != nil {
		return err
	}
	s.l = l
	s.svr.Register(NewCompileService(s.server.compSvr))
	return nil
}

func (s *RpcServer) Run() {
	go s.svr.Accept(s.l)
}

func (s *RpcServer) Wait() {
	s.svr.Accept(s.l)
}
<commit_msg>close channel when done<commit_after>// Copyright 2016 Alex Fluter

package lotsawa

import (
	"net"
	"net/rpc"
	"time"
)

type CompileArgs struct {
	// The code to compile
	Code string

	// The language of the code
	Lang string
}

type CompileReply struct {
	// The command line used to compile this piece of code
	Cmd string
	// Whether the code has main function and can be executed
	Main bool
	// Errors during the compiling
	Error string
	// Time took to compile and run the program
	Time time.Duration
	// Compiler's standard output
	C_Output string
	// Compiler's standard error
	C_Error string
	// The program's standard error, if compiled successfully and had main
	P_Output string
	// The program's standard output, if compiled successfully and had main
	P_Error string
}

// RPC service
type CompileService struct {
	server *CompilerServer
}

func NewCompileService(server *CompilerServer) *CompileService {
	return &CompileService{
		server: server,
	}
}

func (c *CompileService) Compile(args *CompileArgs, reply *CompileReply) error {
	req := &Request{
		received: time.Now(),
		args:     args,
		chRes:    make(chan *Result),
	}

	c.server.Submit(req)

	res := <-req.chRes
	res.done = time.Now()

	reply.Time = res.done.Sub(req.received)

	reply.Error, reply.Cmd, reply.Main = res.err, res.cmd, res.main
	reply.C_Output, reply.C_Error = res.c_out, res.c_err
	if res.main {
		reply.P_Output, reply.P_Error = res.p_out, res.p_err
	}

	close(req.chRes)
	return nil
}

// RPC stub for the client
type CompileServiceStub struct {
	client *rpc.Client
}

func NewCompileServiceStub(net, addr string) (*CompileServiceStub, error) {
	var err error
	s := new(CompileServiceStub)
	s.client, err = rpc.Dial(net, addr)
	if err != nil {
		return nil, err
	}
	return s, nil
}

func (c *CompileServiceStub) Compile(args *CompileArgs, reply *CompileReply) error {
	var err error

	err = c.client.Call("CompileService.Compile", args, reply)
	return err
}

// RPC server
type RpcServer struct {
	svr    *rpc.Server
	l      *net.TCPListener
	addr   *net.TCPAddr
	server *Server
}

func NewRpcServer(server *Server, addr string) (*RpcServer, error) {
	var err error
	s := new(RpcServer)
	s.svr = rpc.NewServer()
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}
	s.addr = tcpAddr
	s.server = server

	return s, nil
}

func (s *RpcServer) Init() error {
	l, err := net.ListenTCP("tcp", s.addr)
	if err != nil {
		return err
	}
	s.l = l
	s.svr.Register(NewCompileService(s.server.compSvr))
	return nil
}

func (s *RpcServer) Run() {
	go s.svr.Accept(s.l)
}

func (s *RpcServer) Wait() {
	s.svr.Accept(s.l)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"

	"github.com/dim13/gold/articles"
)

type Rss struct {
	Url      string
	Title    string
	Subtitle string
	Articles articles.Articles
}

func rssHandler(w http.ResponseWriter, r *http.Request) {
	app := conf.Blog.ArticlesPerPage
	a := data.Articles.Enabled().Limit(app)

	rss := Rss{
		Url:      "http://" + r.Host,
		Title:    conf.Blog.Title,
		Subtitle: conf.Blog.Subtitle,
		Articles: a,
	}
	err := tmpl.ExecuteTemplate(w, "rss.tmpl", rss)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>fatal -> println<commit_after>package main

import (
	"log"
	"net/http"

	"github.com/dim13/gold/articles"
)

type Rss struct {
	Url      string
	Title    string
	Subtitle string
	Articles articles.Articles
}

func rssHandler(w http.ResponseWriter, r *http.Request) {
	app := conf.Blog.ArticlesPerPage
	a := data.Articles.Enabled().Limit(app)

	rss := Rss{
		Url:      "http://" + r.Host,
		Title:    conf.Blog.Title,
		Subtitle: conf.Blog.Subtitle,
		Articles: a,
	}
	err := tmpl.ExecuteTemplate(w, "rss.tmpl", rss)
	if err != nil {
		log.Println(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"
	path "path/filepath"
	"runtime"
)

var cmdRun = &Command{
	UsageLine: "run [appname]",
	Short:     "run the app which can hot compile",
	Long: `
start the appname throw exec.Command

then start a inotify watch for current dir

when the file has changed bee will auto go build and restart the app

	file changed
	     |
  check if it's go file
	     |
     yes     no
      |       |
 go build    do nothing
     |
 restart app
`,
}

var defaultJson = `
{
	"go_install": false,
	"dir_structure":{
		"controllers": "",
		"models": "",
		"others": []
	},
	"main_files":{
		"main.go": "",
		"others": []
	}
}
`

func init() {
	cmdRun.Run = runApp
}

var appname string
var conf struct {
	// Indicates whether execute "go install" before "go build".
	GoInstall bool `json:"go_install"`

	DirStruct struct {
		Controllers string
		Models      string
		Others      []string // Other directories.
	} `json:"dir_structure"`

	MainFiles struct {
		Main   string   `json:"main.go"`
		Others []string // Others files of package main.
	} `json:"main_files"`
}

func runApp(cmd *Command, args []string) {
	if len(args) != 1 {
		fmt.Println("[ERRO] Argument [appname] is missing")
		os.Exit(2)
	}
	crupath, _ := os.Getwd()
	Debugf("current path:%s\n", crupath)

	err := loadConfig()
	if err != nil {
		fmt.Println("[ERRO] Fail to parse bee.json:", err)
	}
	var paths []string
	paths = append(paths,
		path.Join(crupath, conf.DirStruct.Controllers),
		path.Join(crupath, conf.DirStruct.Models),
		path.Join(crupath, "./")) // Current path.
	// Because monitor files has some issues, we watch current directory
	// and ignore non-go files.
	paths = append(paths, conf.DirStruct.Others...)
	paths = append(paths, conf.MainFiles.Others...)

	NewWatcher(paths)
	appname = args[0]
	Autobuild()
	for {
		runtime.Gosched()
	}
}

// loadConfig loads customized configuration.
func loadConfig() error {
	f, err := os.Open("bee.json")
	if err != nil {
		// Use default.
		err = json.Unmarshal([]byte(defaultJson), &conf)
		if err != nil {
			return err
		}
	} else {
		defer f.Close()
		fmt.Println("[INFO] Detected bee.json")
		d := json.NewDecoder(f)
		err = d.Decode(&conf)
		if err != nil {
			return err
		}
	}
	// Set variables.
	if len(conf.DirStruct.Controllers) == 0 {
		conf.DirStruct.Controllers = "controllers"
	}
	if len(conf.DirStruct.Models) == 0 {
		conf.DirStruct.Models = "models"
	}
	if len(conf.MainFiles.Main) == 0 {
		conf.MainFiles.Main = "main.go"
	}
	return nil
}
<commit_msg>fix linux cpu 100%<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"
	path "path/filepath"
	"runtime"
)

var cmdRun = &Command{
	UsageLine: "run [appname]",
	Short:     "run the app which can hot compile",
	Long: `
start the appname throw exec.Command

then start a inotify watch for current dir

when the file has changed bee will auto go build and restart the app

	file changed
	     |
  check if it's go file
	     |
     yes     no
      |       |
 go build    do nothing
     |
 restart app
`,
}

var defaultJson = `
{
	"go_install": false,
	"dir_structure":{
		"controllers": "",
		"models": "",
		"others": []
	},
	"main_files":{
		"main.go": "",
		"others": []
	}
}
`

func init() {
	cmdRun.Run = runApp
}

var appname string
var conf struct {
	// Indicates whether execute "go install" before "go build".
	GoInstall bool `json:"go_install"`

	DirStruct struct {
		Controllers string
		Models      string
		Others      []string // Other directories.
	} `json:"dir_structure"`

	MainFiles struct {
		Main   string   `json:"main.go"`
		Others []string // Others files of package main.
	} `json:"main_files"`
}

func runApp(cmd *Command, args []string) {
	exit := make(chan bool)
	if len(args) != 1 {
		fmt.Println("[ERRO] Argument [appname] is missing")
		os.Exit(2)
	}
	crupath, _ := os.Getwd()
	Debugf("current path:%s\n", crupath)

	err := loadConfig()
	if err != nil {
		fmt.Println("[ERRO] Fail to parse bee.json:", err)
	}
	var paths []string
	paths = append(paths,
		path.Join(crupath, conf.DirStruct.Controllers),
		path.Join(crupath, conf.DirStruct.Models),
		path.Join(crupath, "./")) // Current path.
	// Because monitor files has some issues, we watch current directory
	// and ignore non-go files.
	paths = append(paths, conf.DirStruct.Others...)
	paths = append(paths, conf.MainFiles.Others...)

	NewWatcher(paths)
	appname = args[0]
	Autobuild()
	for {
		select {
		case <-exit:
			runtime.Goexit()
		}
	}
}

// loadConfig loads customized configuration.
func loadConfig() error {
	f, err := os.Open("bee.json")
	if err != nil {
		// Use default.
		err = json.Unmarshal([]byte(defaultJson), &conf)
		if err != nil {
			return err
		}
	} else {
		defer f.Close()
		fmt.Println("[INFO] Detected bee.json")
		d := json.NewDecoder(f)
		err = d.Decode(&conf)
		if err != nil {
			return err
		}
	}
	// Set variables.
	if len(conf.DirStruct.Controllers) == 0 {
		conf.DirStruct.Controllers = "controllers"
	}
	if len(conf.DirStruct.Models) == 0 {
		conf.DirStruct.Models = "models"
	}
	if len(conf.MainFiles.Main) == 0 {
		conf.MainFiles.Main = "main.go"
	}
	return nil
}
<|endoftext|>


<commit_before>package dynago

import (
	"encoding/json"
)

const (
	SELECT_ALL        = "ALL_ATTRIBUTES"
	SELECT_PROJECTED  = "ALL_PROJECTED_ATTRIBUTES"
	SELECT_ATTRIBUTES = "SPECIFIC_ATTRIBUTES"
	SELECT_COUNT      = "COUNT"
)

var (
	RETURN_CONSUMED = map[bool]string{true: "TOTAL", false: "NONE"}
)

type ConsumedCapacityDescription struct {
	CapacityUnits float32
	TableName     string
}

type KeyValue struct {
	Key   AttributeDefinition
	Value interface{}
}

// Items are maps of name/value pairs
type Item map[string]interface{}

func (pi *Item) UnmarshalJSON(data []byte) error {
	var dbitem DBItem

	if err := json.Unmarshal(data, &dbitem); err != nil {
		return err
	}

	item := make(Item)

	for k, v := range dbitem {
		item[k] = DecodeValue(v)
	}

	*pi = item
	return nil
}

//////////////////////////////////////////////////////////////////////////////
//
// GetItem
//

type GetItemRequest struct {
	TableName              string
	Key                    map[string]AttributeValue
	AttributesToGet        []string
	ConsistentRead         bool
	ReturnConsumedCapacity string
}

type GetItemResult struct {
	ConsumedCapacity ConsumedCapacityDescription

	Item DBItem
}

func (db *DBClient) GetItem(tableName string, hashKey *KeyValue, rangeKey *KeyValue, attributes []string, consistent bool, consumed bool) (map[string]interface{}, float32, error) {

	getReq := GetItemRequest{TableName: tableName, AttributesToGet: attributes, ConsistentRead: consistent, ReturnConsumedCapacity: RETURN_CONSUMED[consumed]}
	getReq.Key = EncodeAttribute(hashKey.Key, hashKey.Value)
	if rangeKey != nil {
		getReq.Key[rangeKey.Key.AttributeName] = EncodeAttributeValue(rangeKey.Key, rangeKey.Value)
	}

	var getRes GetItemResult

	if err := db.Query("GetItem", getReq).Decode(&getRes); err != nil {
		return nil, 0.0, err
	}

	if len(getRes.Item) == 0 {
		return nil, getRes.ConsumedCapacity.CapacityUnits, nil
	}

	return DecodeItem(getRes.Item), getRes.ConsumedCapacity.CapacityUnits, nil
}

//////////////////////////////////////////////////////////////////////////////
//
// Query
//

type QueryRequest struct {
	TableName              string
	AttributesToGet        []string `json:",omitempty"`
	ScanIndexForward       bool
	ExclusiveStartKey      AttributeNameValue   `json:",omitempty"`
	KeyConditions          map[string]Condition `json:",omitempty"`
	IndexName              string               `json:",omitempty"`
	Limit                  *int                 `json:",omitempty"`
	Select                 string               `json:",omitempty"`
	ReturnConsumedCapacity string               `json:",omitempty"`

	table *TableInstance
}

type QueryResult struct {
	Items            []Item
	ConsumedCapacity ConsumedCapacityDescription
	LastEvaluatedKey AttributeNameValue
	Count            int
	ScannedCount     int
}

func QueryTable(table *TableInstance) *QueryRequest {
	return &QueryRequest{TableName: table.Name, ScanIndexForward: true, KeyConditions: make(map[string]Condition), table: table}
}

func Query(tableName string) *QueryRequest {
	return &QueryRequest{TableName: tableName, ScanIndexForward: true, KeyConditions: make(map[string]Condition)}
}

func (queryReq *QueryRequest) WithAttributes(attributes []string) *QueryRequest {
	queryReq.AttributesToGet = attributes
	return queryReq
}

func (queryReq *QueryRequest) WithStartKey(startKey AttributeNameValue) *QueryRequest {
	queryReq.ExclusiveStartKey = startKey
	return queryReq
}

func (queryReq *QueryRequest) WithIndex(indexName string) *QueryRequest {
	queryReq.IndexName = indexName
	return queryReq
}

func (queryReq *QueryRequest) WithCondition(attrName string, condition Condition) *QueryRequest {
	queryReq.KeyConditions[attrName] = condition
	return queryReq
}

func (queryReq *QueryRequest) WithAttrCondition(cond AttrCondition) *QueryRequest {
	for k, v := range cond {
		queryReq.KeyConditions[k] = v
	}

	return queryReq
}

func (queryReq *QueryRequest) WithLimit(limit int) *QueryRequest {
	queryReq.Limit = &limit
	return queryReq
}

func (queryReq *QueryRequest) WithSelect(selectValue string) *QueryRequest {
	queryReq.Select = selectValue
	return queryReq
}

func (queryReq *QueryRequest) WithConsumed(consumed bool) *QueryRequest {
	queryReq.ReturnConsumedCapacity = RETURN_CONSUMED[consumed]
	return queryReq
}

func (queryReq *QueryRequest) Exec(db *DBClient) ([]Item, AttributeNameValue, float32, error) {
	if db == nil && queryReq.table != nil {
		db = queryReq.table.DB
	}

	var queryRes QueryResult

	if err := db.Query("Query", queryReq).Decode(&queryRes); err != nil {
		return nil, nil, 0.0, err
	}

	return queryRes.Items, queryRes.LastEvaluatedKey, queryRes.ConsumedCapacity.CapacityUnits, nil
}

//////////////////////////////////////////////////////////////////////////////
//
// Scan
//

type ScanRequest struct {
	TableName              string
	AttributesToGet        []string
	ExclusiveStartKey      AttributeNameValue
	ScanFilter             map[string]Condition
	Limit                  *int   `json:",omitempty"`
	Segment                *int   `json:",omitempty"`
	TotalSegments          *int   `json:",omitempty"`
	Select                 string `json:",omitempty"`
	ReturnConsumedCapacity string `json:",omitempty"`

	table *TableInstance
}

func ScanTable(table *TableInstance) *ScanRequest {
	return &ScanRequest{TableName: table.Name, table: table}
}

func Scan(tableName string) *ScanRequest {
	return &ScanRequest{TableName: tableName}
}

func (scanReq *ScanRequest) WithAttributes(attributes []string) *ScanRequest {
	scanReq.AttributesToGet = attributes
	return scanReq
}

func (scanReq *ScanRequest) WithStartKey(startKey AttributeNameValue) *ScanRequest {
	scanReq.ExclusiveStartKey = startKey
	return scanReq
}

func (scanReq *ScanRequest) WithFilter(attrName string, condition Condition) *ScanRequest {
	if scanReq.ScanFilter == nil {
		scanReq.ScanFilter = map[string]Condition{attrName: condition}
	} else {
		scanReq.ScanFilter[attrName] = condition
	}
	return scanReq
}

func (scanReq *ScanRequest) WithFilters(filters AttrCondition) *ScanRequest {
	scanReq.ScanFilter = filters
	return scanReq
}

func (scanReq *ScanRequest) WithLimit(limit int) *ScanRequest {
	scanReq.Limit = &limit
	return scanReq
}

func (scanReq *ScanRequest) WithSegment(segment, totalSegments int) *ScanRequest {
	scanReq.Segment = &segment
	scanReq.TotalSegments = &totalSegments
	return scanReq
}

func (scanReq *ScanRequest) WithSelect(selectValue string) *ScanRequest {
	scanReq.Select = selectValue
	return scanReq
}

func (scanReq *ScanRequest) WithConsumed(consumed bool) *ScanRequest {
	scanReq.ReturnConsumedCapacity = RETURN_CONSUMED[consumed]
	return scanReq
}

func (scanReq *ScanRequest) Exec(db *DBClient) ([]Item, AttributeNameValue, float32, error) {
	var scanRes QueryResult

	if err := db.Query("Scan", scanReq).Decode(&scanRes); err != nil {
		return nil, nil, 0.0, err
	}

	return scanRes.Items, scanRes.LastEvaluatedKey, scanRes.ConsumedCapacity.CapacityUnits, nil
}

func (scanReq *ScanRequest) Count(db *DBClient) (count int, scount int, consumed float32, err error) {
	var scanRes QueryResult

	req := *scanReq
	req.Select = SELECT_COUNT

	for {
		scanRes.LastEvaluatedKey = nil

		if err = db.Query("Scan", &req).Decode(&scanRes); err != nil {
			break
		}

		count += scanRes.Count
		scount += scanRes.ScannedCount
		consumed += scanRes.ConsumedCapacity.CapacityUnits

		if scanRes.LastEvaluatedKey == nil {
			break
		}

		req.ExclusiveStartKey = make(AttributeNameValue)
		for k, v := range scanRes.LastEvaluatedKey {
			req.ExclusiveStartKey[k] = v
		}
	}

	return
}
<commit_msg>Don't need to retrieve a DBItem and decode to Item. I can just say that Item is an Item.<commit_after>package dynago

import (
	"encoding/json"
)

const (
	SELECT_ALL        = "ALL_ATTRIBUTES"
	SELECT_PROJECTED  = "ALL_PROJECTED_ATTRIBUTES"
	SELECT_ATTRIBUTES = "SPECIFIC_ATTRIBUTES"
	SELECT_COUNT      = "COUNT"
)

var (
	RETURN_CONSUMED = map[bool]string{true: "TOTAL", false: "NONE"}
)

type ConsumedCapacityDescription struct {
	CapacityUnits float32
	TableName     string
}

type KeyValue struct {
	Key   AttributeDefinition
	Value interface{}
}

// Items are maps of name/value pairs
type Item map[string]interface{}

func (pi *Item) UnmarshalJSON(data []byte) error {
	var dbitem DBItem

	if err := json.Unmarshal(data, &dbitem); err != nil {
		return err
	}

	item := make(Item)

	for k, v := range dbitem {
		item[k] = DecodeValue(v)
	}

	*pi = item
	return nil
}

//////////////////////////////////////////////////////////////////////////////
//
// GetItem
//

type GetItemRequest struct {
	TableName              string
	Key                    map[string]AttributeValue
	AttributesToGet        []string
	ConsistentRead         bool
	ReturnConsumedCapacity string
}

type GetItemResult struct {
	ConsumedCapacity ConsumedCapacityDescription

	Item Item
}

func (db *DBClient) GetItem(tableName string, hashKey *KeyValue, rangeKey *KeyValue, attributes []string, consistent bool, consumed bool) (map[string]interface{}, float32, error) {

	getReq := GetItemRequest{TableName: tableName, AttributesToGet: attributes, ConsistentRead: consistent, ReturnConsumedCapacity: RETURN_CONSUMED[consumed]}
	getReq.Key = EncodeAttribute(hashKey.Key, hashKey.Value)
	if rangeKey != nil {
		getReq.Key[rangeKey.Key.AttributeName] = EncodeAttributeValue(rangeKey.Key, rangeKey.Value)
	}

	var getRes GetItemResult

	if err := db.Query("GetItem", getReq).Decode(&getRes); err != nil {
		return nil, 0.0, err
	}

	if len(getRes.Item) == 0 {
		return nil, getRes.ConsumedCapacity.CapacityUnits, nil
	}

	return getRes.Item, getRes.ConsumedCapacity.CapacityUnits, nil
}

//////////////////////////////////////////////////////////////////////////////
//
// Query
//

type QueryRequest struct {
	TableName              string
	AttributesToGet        []string `json:",omitempty"`
	ScanIndexForward       bool
	ExclusiveStartKey      AttributeNameValue   `json:",omitempty"`
	KeyConditions          map[string]Condition `json:",omitempty"`
	IndexName              string               `json:",omitempty"`
	Limit                  *int                 `json:",omitempty"`
	Select                 string               `json:",omitempty"`
	ReturnConsumedCapacity string               `json:",omitempty"`

	table *TableInstance
}

type QueryResult struct {
	Items            []Item
	ConsumedCapacity ConsumedCapacityDescription
	LastEvaluatedKey AttributeNameValue
	Count            int
	ScannedCount     int
}

func QueryTable(table *TableInstance) *QueryRequest {
	return &QueryRequest{TableName: table.Name, ScanIndexForward: true, KeyConditions: make(map[string]Condition), table: table}
}

func Query(tableName string) *QueryRequest {
	return &QueryRequest{TableName: tableName, ScanIndexForward: true, KeyConditions: make(map[string]Condition)}
}

func (queryReq *QueryRequest) WithAttributes(attributes []string) *QueryRequest {
	queryReq.AttributesToGet = attributes
	return queryReq
}

func (queryReq *QueryRequest) WithStartKey(startKey AttributeNameValue) *QueryRequest {
	queryReq.ExclusiveStartKey = startKey
	return queryReq
}

func (queryReq *QueryRequest) WithIndex(indexName string) *QueryRequest {
	queryReq.IndexName = indexName
	return queryReq
}

func (queryReq *QueryRequest) WithCondition(attrName string, condition Condition) *QueryRequest {
	queryReq.KeyConditions[attrName] = condition
	return queryReq
}

func (queryReq *QueryRequest) WithAttrCondition(cond AttrCondition) *QueryRequest {
	for k, v := range cond {
		queryReq.KeyConditions[k] = v
	}

	return queryReq
}

func (queryReq *QueryRequest) WithLimit(limit int) *QueryRequest {
	queryReq.Limit = &limit
	return queryReq
}

func (queryReq *QueryRequest) WithSelect(selectValue string) *QueryRequest {
	queryReq.Select = selectValue
	return queryReq
}

func (queryReq *QueryRequest) WithConsumed(consumed bool) *QueryRequest {
	queryReq.ReturnConsumedCapacity = RETURN_CONSUMED[consumed]
	return queryReq
}

func (queryReq *QueryRequest) Exec(db *DBClient) ([]Item, AttributeNameValue, float32, error) {
	if db == nil && queryReq.table != nil {
		db = queryReq.table.DB
	}

	var queryRes QueryResult

	if err := db.Query("Query", queryReq).Decode(&queryRes); err != nil {
		return nil, nil, 0.0, err
	}

	return queryRes.Items, queryRes.LastEvaluatedKey, queryRes.ConsumedCapacity.CapacityUnits, nil
}

//////////////////////////////////////////////////////////////////////////////
//
// Scan
//

type ScanRequest struct {
	TableName              string
	AttributesToGet        []string
	ExclusiveStartKey      AttributeNameValue
	ScanFilter             map[string]Condition
	Limit                  *int   `json:",omitempty"`
	Segment                *int   `json:",omitempty"`
	TotalSegments          *int   `json:",omitempty"`
	Select                 string `json:",omitempty"`
	ReturnConsumedCapacity string `json:",omitempty"`

	table *TableInstance
}

func ScanTable(table *TableInstance) *ScanRequest {
	return &ScanRequest{TableName: table.Name, table: table}
}

func Scan(tableName string) *ScanRequest {
	return &ScanRequest{TableName: tableName}
}

func (scanReq *ScanRequest) WithAttributes(attributes []string) *ScanRequest {
	scanReq.AttributesToGet = attributes
	return scanReq
}

func (scanReq *ScanRequest) WithStartKey(startKey AttributeNameValue) *ScanRequest {
	scanReq.ExclusiveStartKey = startKey
	return scanReq
}

func (scanReq *ScanRequest) WithFilter(attrName string, condition Condition) *ScanRequest {
	if scanReq.ScanFilter == nil {
		scanReq.ScanFilter = map[string]Condition{attrName: condition}
	} else {
		scanReq.ScanFilter[attrName] = condition
	}
	return scanReq
}

func (scanReq *ScanRequest) WithFilters(filters AttrCondition) *ScanRequest {
	scanReq.ScanFilter = filters
	return scanReq
}

func (scanReq *ScanRequest) WithLimit(limit int) *ScanRequest {
	scanReq.Limit = &limit
	return scanReq
}

func (scanReq *ScanRequest) WithSegment(segment, totalSegments int) *ScanRequest {
	scanReq.Segment = &segment
	scanReq.TotalSegments = &totalSegments
	return scanReq
}

func (scanReq *ScanRequest) WithSelect(selectValue string) *ScanRequest {
	scanReq.Select = selectValue
	return scanReq
}

func (scanReq *ScanRequest) WithConsumed(consumed bool) *ScanRequest {
	scanReq.ReturnConsumedCapacity = RETURN_CONSUMED[consumed]
	return scanReq
}

func (scanReq *ScanRequest) Exec(db *DBClient) ([]Item, AttributeNameValue, float32, error) {
	var scanRes QueryResult

	if err := db.Query("Scan", scanReq).Decode(&scanRes); err != nil {
		return nil, nil, 0.0, err
	}

	return scanRes.Items, scanRes.LastEvaluatedKey, scanRes.ConsumedCapacity.CapacityUnits, nil
}

func (scanReq *ScanRequest) Count(db *DBClient) (count int, scount int, consumed float32, err error) {
	var scanRes QueryResult

	req := *scanReq
	req.Select = SELECT_COUNT

	for {
		scanRes.LastEvaluatedKey = nil

		if err = db.Query("Scan", &req).Decode(&scanRes); err != nil {
			break
		}

		count += scanRes.Count
		scount += scanRes.ScannedCount
		consumed += scanRes.ConsumedCapacity.CapacityUnits

		if scanRes.LastEvaluatedKey == nil {
			break
		}

		req.ExclusiveStartKey = make(AttributeNameValue)
		for k, v := range scanRes.LastEvaluatedKey {
			req.ExclusiveStartKey[k] = v
		}
	}

	return
}
<|endoftext|>


<commit_before>package ephemerald

import (
	"context"
	"sync"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types"
)

type poolItemEvent string

const (
	eventPoolItemStart poolItemEvent = "start"
	eventPoolItemReset poolItemEvent = "reset"
	eventPoolItemKill  poolItemEvent = "kill"

	eventPoolItemLiveError  poolItemEvent = "live-error"
	eventPoolItemLive       poolItemEvent = "live"
	eventPoolItemResetError poolItemEvent = "reset-error"
	eventPoolItemReady      poolItemEvent = "ready"
	eventPoolItemReadyError poolItemEvent = "ready-error"
)

type poolItemState string

const (
	poolItemStateRunning poolItemState = "running"
	poolItemStateExited  poolItemState = "exited"
)

type poolItem struct {
	state       poolItemState
	provisioner Provisioner
	adapter     Adapter
	container   PoolContainer

	events chan poolItemEvent
	joinch chan (chan<- poolEvent)

	// closed when exited
	exited chan bool

	ctx    context.Context
	cancel context.CancelFunc

	wg  sync.WaitGroup
	log logrus.FieldLogger
}

func createPoolItem(log logrus.FieldLogger, adapter Adapter, provisioner Provisioner) (PoolItem, error) {
	log = log.WithField("component", "pool-item")

	container, err := createPoolContainer(log, adapter)
	if err != nil {
		log.WithError(err).
			Error("can't create container")
		return nil, err
	}

	log = lcid(log, container.ID())

	ctx, cancel := context.WithCancel(context.Background())

	item := &poolItem{
		state:       poolItemStateRunning,
		provisioner: provisioner,
		adapter:     adapter,
		container:   container,
		events:      make(chan poolItemEvent),
		joinch:      make(chan (chan<- poolEvent)),
		exited:      make(chan bool),
		ctx:         ctx,
		cancel:      cancel,
		log:         log,
	}

	go item.run()

	return item, nil
}

func (i *poolItem) ID() string {
	return i.container.ID()
}

func (i *poolItem) Status() types.ContainerJSON {
	return i.container.Status()
}

func (i *poolItem) Join(ch chan<- poolEvent) {
	i.joinch <- ch
}

func (i *poolItem) Start() {
	select {
	case <-i.exited:
	case i.events <- eventPoolItemStart:
	}
}

func (i *poolItem) Reset() {
	go func() {
		select {
		case <-i.exited:
		case i.events <- eventPoolItemReset:
		}
	}()
}

func (i *poolItem) Kill() {
	go func() {
		select {
		case <-i.exited:
		case i.events <- eventPoolItemKill:
		}
	}()
}

func (i *poolItem) run() {
	ch := i.runWaitJoin()
	i.runMainLoop(ch)

	i.drain()
}

func (i *poolItem) runWaitJoin() chan<- poolEvent {
	defer close(i.joinch)

	log := i.log.WithField("method", "runWaitJoin")

	for {
		select {
		case ch := <-i.joinch:
			log.Debug("joined")
			return ch
		case e := <-i.events:
			log.WithField("event", e).Debug("item-event")
			switch e {
			case eventPoolItemKill:
				i.container.Stop()
			}
		}
	}
}

func (i *poolItem) runMainLoop(ch chan<- poolEvent) {
	defer close(i.exited)
	defer i.cancel()

	log := i.log.WithField("method", "runMainLoop")

	for {
		select {
		case e := <-i.container.Events():
			log.WithField("event", e).Debug("container-event")

			switch e {
			case containerEventExitSuccess:
				fallthrough
			case containerEventExitError:
				fallthrough
			case containerEventStartFailed:
				ch <- poolEvent{eventItemExit, i}
				return
			case containerEventStarted:
				i.do(i.onChildStarted)
			}

		case e := <-i.events:
			log.WithField("event", e).Debug("item-event")

			switch e {
			case eventPoolItemKill:
				i.container.Stop()
			case eventPoolItemStart:
				i.container.Start()
			case eventPoolItemLive:
				i.do(i.onChildLive)
			case eventPoolItemLiveError:
				i.container.Stop()
			case eventPoolItemReady:
				ch <- poolEvent{eventItemReady, i}
			case eventPoolItemReadyError:
				i.container.Stop()
			case eventPoolItemReset:
				i.do(i.onChildReset)
			}

		}
	}
}

func (i *poolItem) drain() {
	log := i.log.WithField("method", "drain")

	defer close(i.events)

	ch := make(chan bool)
	go func() {
		i.wg.Wait()
		close(ch)
	}()

	for {
		select {
		case <-ch:
			log.Debug("done")
			return
		case e := <-i.events:
			log.WithField("event", e).Debug("stale item-event")
		}
	}
}

func (i *poolItem) onChildStarted() {
	if prov, ok := isLiveCheckProvisioner(i.provisioner); ok {
		if err := prov.LiveCheck(i.ctx, i.container); err != nil {
			i.log.WithError(err).Error("error checking liveliness")
			i.events <- eventPoolItemLiveError
			return
		}
	}
	i.events <- eventPoolItemLive
}

func (i *poolItem) onChildLive() {
	if prov, ok := isInitializeProvisioner(i.provisioner); ok {
		if err := prov.Initialize(i.ctx, i.container); err != nil {
			i.log.WithError(err).Error("error initializing")
			i.events <- eventPoolItemReadyError
			return
		}
	}
	i.events <- eventPoolItemReady
}

func (i *poolItem) onChildReset() {
	if prov, ok := isResetProvisioner(i.provisioner); ok {
		if err := prov.Reset(i.ctx, i.container); err != nil {
			i.log.WithError(err).Error("error provisioning")
			i.events <- eventPoolItemResetError
		}
		i.events <- eventPoolItemReady
		return
	}
	i.container.Stop()
}

func (i *poolItem) do(fn func()) {
	i.wg.Add(1)
	go func() {
		defer i.wg.Done()
		fn()
	}()
}
<commit_msg>cleanup<commit_after>package ephemerald

import (
	"context"
	"sync"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types"
)

type poolItemEvent string

const (
	eventPoolItemStart poolItemEvent = "start"
	eventPoolItemReset poolItemEvent = "reset"
	eventPoolItemKill  poolItemEvent = "kill"

	eventPoolItemLiveError  poolItemEvent = "live-error"
	eventPoolItemLive       poolItemEvent = "live"
	eventPoolItemResetError poolItemEvent = "reset-error"
	eventPoolItemReady      poolItemEvent = "ready"
	eventPoolItemReadyError poolItemEvent = "ready-error"
)

type poolItemState string

const (
	poolItemStateRunning poolItemState = "running"
	poolItemStateExited  poolItemState = "exited"
)

type poolItem struct {
	state       poolItemState
	provisioner Provisioner
	adapter     Adapter
	container   PoolContainer

	events chan poolItemEvent
	joinch chan (chan<- poolEvent)

	// closed when exited
	exited chan bool

	ctx    context.Context
	cancel context.CancelFunc

	wg  sync.WaitGroup
	log logrus.FieldLogger
}

func createPoolItem(log logrus.FieldLogger, adapter Adapter, provisioner Provisioner) (PoolItem, error) {
	log = log.WithField("component", "pool-item")

	container, err := createPoolContainer(log, adapter)
	if err != nil {
		log.WithError(err).
			Error("can't create container")
		return nil, err
	}

	log = lcid(log, container.ID())

	ctx, cancel := context.WithCancel(context.Background())

	item := &poolItem{
		state:       poolItemStateRunning,
		provisioner: provisioner,
		adapter:     adapter,
		container:   container,
		events:      make(chan poolItemEvent),
		joinch:      make(chan (chan<- poolEvent)),
		exited:      make(chan bool),
		ctx:         ctx,
		cancel:      cancel,
		log:         log,
	}

	go item.run()

	return item, nil
}

func (i *poolItem) ID() string {
	return i.container.ID()
}

func (i *poolItem) Status() types.ContainerJSON {
	return i.container.Status()
}

func (i *poolItem) Join(ch chan<- poolEvent) {
	i.joinch <- ch
}

func (i *poolItem) Start() {
	go i.sendEvent(eventPoolItemStart)
}

func (i *poolItem) Reset() {
	go i.sendEvent(eventPoolItemReset)
}

func (i *poolItem) Kill() {
	go i.sendEvent(eventPoolItemKill)
}

func (i *poolItem) sendEvent(e poolItemEvent) {
	select {
	case <-i.exited:
	case i.events <- e:
	}
}

func (i *poolItem) run() {
	ch := i.runWaitJoin()
	i.runMainLoop(ch)

	i.drain()
}

func (i *poolItem) runWaitJoin() chan<- poolEvent {
	defer close(i.joinch)

	log := i.log.WithField("method", "runWaitJoin")

	for {
		select {
		case ch := <-i.joinch:
			log.Debug("joined")
			return ch
		case e := <-i.events:
			log.WithField("event", e).Debug("item-event")
			switch e {
			case eventPoolItemKill:
				i.container.Stop()
			}
		}
	}
}

func (i *poolItem) runMainLoop(ch chan<- poolEvent) {
	defer close(i.exited)
	defer i.cancel()

	log := i.log.WithField("method", "runMainLoop")

	for {
		select {
		case e := <-i.container.Events():
			log.WithField("event", e).Debug("container-event")

			switch e {
			case containerEventExitSuccess:
				fallthrough
			case containerEventExitError:
				fallthrough
			case containerEventStartFailed:
				ch <- poolEvent{eventItemExit, i}
				return
			case containerEventStarted:
				i.do(i.onChildStarted)
			}

		case e := <-i.events:
			log.WithField("event", e).Debug("item-event")

			switch e {
			case eventPoolItemKill:
				i.container.Stop()
			case eventPoolItemStart:
				i.container.Start()
			case eventPoolItemLive:
				i.do(i.onChildLive)
			case eventPoolItemLiveError:
				i.container.Stop()
			case eventPoolItemReady:
				ch <- poolEvent{eventItemReady, i}
			case eventPoolItemReadyError:
				i.container.Stop()
			case eventPoolItemReset:
				i.do(i.onChildReset)
			}

		}
	}
}

func (i *poolItem) drain() {
	log := i.log.WithField("method", "drain")

	defer close(i.events)

	ch := make(chan bool)
	go func() {
		i.wg.Wait()
		close(ch)
	}()

	for {
		select {
		case <-ch:
			log.Debug("done")
			return
		case e := <-i.events:
			log.WithField("event", e).Debug("stale item-event")
		}
	}
}

func (i *poolItem) onChildStarted() {
	if prov, ok := isLiveCheckProvisioner(i.provisioner); ok {
		if err := prov.LiveCheck(i.ctx, i.container); err != nil {
			i.log.WithError(err).Error("error checking liveliness")
			i.events <- eventPoolItemLiveError
			return
		}
	}
	i.events <- eventPoolItemLive
}

func (i *poolItem) onChildLive() {
	if prov, ok := isInitializeProvisioner(i.provisioner); ok {
		if err := prov.Initialize(i.ctx, i.container); err != nil {
			i.log.WithError(err).Error("error initializing")
			i.events <- eventPoolItemReadyError
			return
		}
	}
	i.events <- eventPoolItemReady
}

func (i *poolItem) onChildReset() {
	if prov, ok := isResetProvisioner(i.provisioner); ok {
		if err := prov.Reset(i.ctx, i.container); err != nil {
			i.log.WithError(err).Error("error provisioning")
			i.events <- eventPoolItemResetError
		}
		i.events <- eventPoolItemReady
		return
	}
	i.container.Stop()
}

func (i *poolItem) do(fn func()) {
	i.wg.Add(1)
	go func() {
		defer i.wg.Done()
		fn()
	}()
}
<|endoftext|>


<commit_before>package unleash

import (
    "os"
    log "github.com/Sirupsen/logrus"
    "path/filepath"
    "strconv"
    "strings"
    "time"
)

// Jobs to build a Docker image from a Dockerfile coming from a repository and push
// it to the registry
func BuildAndPushFromRepository(repositoryUrl string, repositoryFullName string, branch string, commit string) error {
    contextLogger := log.WithFields(log.Fields{
            "repository_url": repositoryUrl,
            "repository": repositoryFullName,
            "branch": branch,
            "commit": commit,
      })
    contextLogger.Info("Build started")

    // Generate a temporary and unique directory that will hold the build
    uniqueWorkingDirectory := filepath.Join(Config.WorkingDirectory, strconv.FormatInt(time.Now().UnixNano(), 10) + "-" + commit)
    defer os.RemoveAll(uniqueWorkingDirectory)

    // Generate the full working directory path for the sake of explicitness
    fullWorkingDirectory := filepath.Join(uniqueWorkingDirectory, repositoryFullName, branch, commit)

    // Clone the repository
    if _, err := Clone(repositoryUrl, fullWorkingDirectory, branch); err != nil {
        contextLogger.Error("Error cloning, cause: ", err)
        return err
    }

    // Construct the tag to use when building the image
    imageRepository := strings.ToLower(repositoryFullName)
    if branch != "master" {
        imageRepository = imageRepository + ":" + branch
    } else {
        imageRepository = imageRepository + ":latest"
    }

    // Add a registry enpoint if specified in the configuration
    if Config.RegistryURL != "" {
        imageRepository = filepath.Join(Config.RegistryURL, imageRepository)
    }

    // The repository used to tag the image must be in lower case
    if _, err := BuildFromDirectory(fullWorkingDirectory, imageRepository, contextLogger); err != nil {
        contextLogger.Error("Error building Dockerfile, cause: ", err)
        return err
    }

    // Push the Docker image to the registry
    if err := PushImage(imageRepository); err != nil {
        contextLogger.Error("Error pushing Docker image, cause: ", err)
        return err
    }

    contextLogger.Info("Build finished")

    return nil
}
<commit_msg>Adding debug step message on building job<commit_after>package unleash

import (
    "os"
    log "github.com/Sirupsen/logrus"
    "path/filepath"
    "strconv"
    "strings"
    "time"
)

// Jobs to build a Docker image from a Dockerfile coming from a repository and push
// it to the registry
func BuildAndPushFromRepository(repositoryUrl string, repositoryFullName string, branch string, commit string) error {
    contextLogger := log.WithFields(log.Fields{
            "repository_url": repositoryUrl,
            "repository": repositoryFullName,
            "branch": branch,
            "commit": commit,
      })
    contextLogger.Info("Build started")

    // Generate a temporary and unique directory that will hold the build
    uniqueWorkingDirectory := filepath.Join(Config.WorkingDirectory, strconv.FormatInt(time.Now().UnixNano(), 10) + "-" + commit)
    defer os.RemoveAll(uniqueWorkingDirectory)

    // Generate the full working directory path for the sake of explicitness
    fullWorkingDirectory := filepath.Join(uniqueWorkingDirectory, repositoryFullName, branch, commit)

    // Clone the repository
    contextLogger.Debug("Cloning")
    if _, err := Clone(repositoryUrl, fullWorkingDirectory, branch); err != nil {
        contextLogger.Error("Error cloning, cause: ", err)
        return err
    }

    // Construct the tag to use when building the image
    imageRepository := strings.ToLower(repositoryFullName)
    if branch != "master" {
        imageRepository = imageRepository + ":" + branch
    } else {
        imageRepository = imageRepository + ":latest"
    }

    // Add a registry enpoint if specified in the configuration
    if Config.RegistryURL != "" {
        imageRepository = filepath.Join(Config.RegistryURL, imageRepository)
    }

    // The repository used to tag the image must be in lower case
    contextLogger.Debug("Building ", imageRepository)
    if _, err := BuildFromDirectory(fullWorkingDirectory, imageRepository, contextLogger); err != nil {
        contextLogger.Error("Error building Dockerfile, cause: ", err)
        return err
    }

    // Push the Docker image to the registry
    contextLogger.Debug("Pushing ", imageRepository)
    if err := PushImage(imageRepository); err != nil {
        contextLogger.Error("Error pushing Docker image, cause: ", err)
        return err
    }

    contextLogger.Info("Build finished")

    return nil
}
<|endoftext|>