<commit_before>// Copyright 2017 Corey Scott http://www.sage42.org/
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cache

import (
	"context"
	"encoding"
	"sync/atomic"
	"time"
)

// Client defines a cache instance.
//
// This can represent the cache for the entire system or for a particular use-case/type.
//
// If a cache is used for multiple purposes, then care must be taken to ensure uniqueness of cache keys.
//
// It is not recommended to change this struct's member data after creation as a data race will likely ensue.
type Client struct {
	// Storage is the cache storage scheme. (Required)
	Storage Storage

	// Logger defines a logger to used for errors during async cache writes (optional)
	Logger Logger

	// Metrics allow for tracking cache events (hit/miss/etc) (optional)
	Metrics Metrics

	// WriteTimeout is the max time spent waiting for cache writes to complete (optional - default 3 seconds)
	WriteTimeout time.Duration

	// track pending cache writes
	pendingWrites int64
}

// Get attempts to retrieve the value from cache and when it misses will run the builder func to create the value.
//
// It will asynchronously update/save the value in the cache on after a successful builder run
func (c *Client) Get(ctx context.Context, key string, dest BinaryEncoder, builder Builder) error {
	bytes, err := c.Storage.Get(ctx, key)
	if err != nil {
		if err == ErrCacheMiss {
			c.getMetrics().Track(CacheMiss)
			return c.onCacheMiss(ctx, key, dest, builder)
		}

		c.getLogger().Log("cache get error. key: '%s' error: %s", key, err)
		c.getMetrics().Track(CacheError)
		return err
	}

	return c.onCacheHit(ctx, key, dest, bytes)
}

func (c *Client) onCacheMiss(ctx context.Context, key string, dest BinaryEncoder, builder Builder) error {
	err := builder.Build(ctx, key, dest)
	if err != nil {
		c.getLogger().Log("cache miss build error. key: '%s' error: %s", key, err)
		c.getMetrics().Track(CacheLambdaError)
		return err
	}

	atomic.AddInt64(&c.pendingWrites, 1)
	go c.updateCache(key, dest)

	return err
}

func (c *Client) onCacheHit(ctx context.Context, key string, dest encoding.BinaryUnmarshaler, bytes []byte) error {
	err := dest.UnmarshalBinary(bytes)
	if err != nil {
		c.getLogger().Log("cache hit unmarshal error. error: %s", err)
		c.getMetrics().Track(CacheUnmarshalError)

		// invalidate to remove "bad" data
		_ = c.Invalidate(ctx, key)

		return err
	}

	c.getMetrics().Track(CacheHit)
	return nil
}

// update the cache with the supplied key/value pair
func (c *Client) updateCache(key string, val encoding.BinaryMarshaler) {
	defer func() {
		// update tracking
		atomic.AddInt64(&c.pendingWrites, -1)
	}()

	// use independent context so we don't miss cache updated
	ctx, cancelFn := context.WithTimeout(context.Background(), c.getWriteTimeout())
	defer cancelFn()

	bytes, err := val.MarshalBinary()
	if err != nil {
		c.getLogger().Log("cache update marshal error. key: '%s' error: %s", key, err)
		c.getMetrics().Track(CacheMarshalError)
		return
	}

	err = c.Storage.Set(ctx, key, bytes)
	if err != nil {
		c.getLogger().Log("cache update set error. key: '%s' error: %s", key, err)
		c.getMetrics().Track(CacheSetError)
	}
}

// Invalidate will force invalidate any matching key in the cache
func (c *Client) Invalidate(ctx context.Context, key string) error {
	err := c.Storage.Invalidate(ctx, key)
	if err != nil {
		c.getLogger().Log("cache invalidate error. key: '%s' error: %s", key, err)
		c.getMetrics().Track(CacheInvalidateError)
		return err
	}

	return nil
}

// return the supplied logger or a no-op implementation
func (c *Client) getLogger() Logger {
	if c.Logger != nil {
		return c.Logger
	}

	return noopLogger
}

// return the supplied metric tracker or a no-op implementation
func (c *Client) getMetrics() Metrics {
	if c.Metrics != nil {
		return c.Metrics
	}

	return noopMetrics
}

// return the timeout on cache writes
func (c *Client) getWriteTimeout() time.Duration {
	if int64(c.WriteTimeout) > 0 {
		return c.WriteTimeout
	}

	return 3 * time.Second
}

// Builder builds the data for a key
type Builder interface {
	// Build returns the data for the supplied key by populating dest
	Build(ctx context.Context, key string, dest BinaryEncoder) error
}

// BuilderFunc implements Builder as a function
type BuilderFunc func(ctx context.Context, key string, dest BinaryEncoder) error

// Build implements Builder
func (b BuilderFunc) Build(ctx context.Context, key string, dest BinaryEncoder) error {
	return b(ctx, key, dest)
}

// BinaryEncoder encodes/decodes the receiver to and from binary form
type BinaryEncoder interface {
	encoding.BinaryMarshaler
	encoding.BinaryUnmarshaler
}
<commit_msg>Revert earlier change so that cache will return an error during 'cache error'.  Leaving the user responsible for any degrade/retry<commit_after>// Copyright 2017 Corey Scott http://www.sage42.org/
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cache

import (
	"context"
	"encoding"
	"sync/atomic"
	"time"
)

// Client defines a cache instance.
//
// This can represent the cache for the entire system or for a particular use-case/type.
//
// If a cache is used for multiple purposes, then care must be taken to ensure uniqueness of cache keys.
//
// It is not recommended to change this struct's member data after creation as a data race will likely ensue.
type Client struct {
	// Storage is the cache storage scheme. (Required)
	Storage Storage

	// Logger defines a logger to used for errors during async cache writes (optional)
	Logger Logger

	// Metrics allow for tracking cache events (hit/miss/etc) (optional)
	Metrics Metrics

	// WriteTimeout is the max time spent waiting for cache writes to complete (optional - default 3 seconds)
	WriteTimeout time.Duration

	// track pending cache writes
	pendingWrites int64
}

// Get attempts to retrieve the value from cache and when it misses will run the builder func to create the value.
//
// It will asynchronously update/save the value in the cache on after a successful builder run
func (c *Client) Get(ctx context.Context, key string, dest BinaryEncoder, builder Builder) error {
	bytes, err := c.Storage.Get(ctx, key)
	if err != nil {
		if err == ErrCacheMiss {
			c.getMetrics().Track(CacheMiss)
			return c.onCacheMiss(ctx, key, dest, builder)
		}

		c.getLogger().Log("cache get error. key: '%s' error: %s", key, err)
		c.getMetrics().Track(CacheGetError)
		return err
	}

	return c.onCacheHit(ctx, key, dest, bytes)
}

func (c *Client) onCacheMiss(ctx context.Context, key string, dest BinaryEncoder, builder Builder) error {
	err := builder.Build(ctx, key, dest)
	if err != nil {
		c.getLogger().Log("cache miss build error. key: '%s' error: %s", key, err)
		c.getMetrics().Track(CacheLambdaError)
		return err
	}

	atomic.AddInt64(&c.pendingWrites, 1)
	go c.updateCache(key, dest)

	return err
}

func (c *Client) onCacheHit(ctx context.Context, key string, dest encoding.BinaryUnmarshaler, bytes []byte) error {
	err := dest.UnmarshalBinary(bytes)
	if err != nil {
		c.getLogger().Log("cache hit unmarshal error. error: %s", err)
		c.getMetrics().Track(CacheUnmarshalError)

		// invalidate to remove "bad" data
		_ = c.Invalidate(ctx, key)

		return err
	}

	c.getMetrics().Track(CacheHit)
	return nil
}

// update the cache with the supplied key/value pair
func (c *Client) updateCache(key string, val encoding.BinaryMarshaler) {
	defer func() {
		// update tracking
		atomic.AddInt64(&c.pendingWrites, -1)
	}()

	// use independent context so we don't miss cache updated
	ctx, cancelFn := context.WithTimeout(context.Background(), c.getWriteTimeout())
	defer cancelFn()

	bytes, err := val.MarshalBinary()
	if err != nil {
		c.getLogger().Log("cache update marshal error. key: '%s' error: %s", key, err)
		c.getMetrics().Track(CacheMarshalError)
		return
	}

	err = c.Storage.Set(ctx, key, bytes)
	if err != nil {
		c.getLogger().Log("cache update set error. key: '%s' error: %s", key, err)
		c.getMetrics().Track(CacheSetError)
	}
}

// Invalidate will force invalidate any matching key in the cache
func (c *Client) Invalidate(ctx context.Context, key string) error {
	err := c.Storage.Invalidate(ctx, key)
	if err != nil {
		c.getLogger().Log("cache invalidate error. key: '%s' error: %s", key, err)
		c.getMetrics().Track(CacheInvalidateError)
		return err
	}

	return nil
}

// return the supplied logger or a no-op implementation
func (c *Client) getLogger() Logger {
	if c.Logger != nil {
		return c.Logger
	}

	return noopLogger
}

// return the supplied metric tracker or a no-op implementation
func (c *Client) getMetrics() Metrics {
	if c.Metrics != nil {
		return c.Metrics
	}

	return noopMetrics
}

// return the timeout on cache writes
func (c *Client) getWriteTimeout() time.Duration {
	if int64(c.WriteTimeout) > 0 {
		return c.WriteTimeout
	}

	return 3 * time.Second
}

// Builder builds the data for a key
type Builder interface {
	// Build returns the data for the supplied key by populating dest
	Build(ctx context.Context, key string, dest BinaryEncoder) error
}

// BuilderFunc implements Builder as a function
type BuilderFunc func(ctx context.Context, key string, dest BinaryEncoder) error

// Build implements Builder
func (b BuilderFunc) Build(ctx context.Context, key string, dest BinaryEncoder) error {
	return b(ctx, key, dest)
}

// BinaryEncoder encodes/decodes the receiver to and from binary form
type BinaryEncoder interface {
	encoding.BinaryMarshaler
	encoding.BinaryUnmarshaler
}
<|endoftext|>


<commit_before>package cache

/*
Based on https://github.com/orcaman/concurrent-map
*/

import (
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"

	"github.com/lomik/go-carbon/helper"
	"github.com/lomik/go-carbon/points"
)

type WriteStrategy int

const (
	MaximumLength WriteStrategy = iota
	TimestampOrder
	Noop
)

const shardCount = 1024

// A "thread" safe map of type string:Anything.
// To avoid lock bottlenecks this map is dived to several (shardCount) map shards.
type Cache struct {
	sync.Mutex

	queueLastBuild time.Time

	data []*Shard

	maxSize       int32
	writeStrategy WriteStrategy

	writeoutQueue *WriteoutQueue

	xlog atomic.Value // io.Writer

	stat struct {
		size              int32  // changing via atomic
		queueBuildCnt     uint32 // number of times writeout queue was built
		queueBuildTimeMs  uint32 // time spent building writeout queue in milliseconds
		queueWriteoutTime uint32 // in milliseconds
		overflowCnt       uint32 // drop packages if cache full
		queryCnt          uint32 // number of queries
	}
}

// A "thread" safe string to anything map.
type Shard struct {
	sync.RWMutex     // Read Write mutex, guards access to internal map.
	items            map[string]*points.Points
	notConfirmed     []*points.Points // linear search for value/slot
	notConfirmedUsed int              // search value in notConfirmed[:notConfirmedUsed]
}

// Creates a new cache instance
func New() *Cache {
	c := &Cache{
		data:          make([]*Shard, shardCount),
		writeStrategy: Noop,
		maxSize:       1000000,
	}

	for i := 0; i < shardCount; i++ {
		c.data[i] = &Shard{
			items:        make(map[string]*points.Points),
			notConfirmed: make([]*points.Points, 4),
		}
	}

	c.writeoutQueue = NewWriteoutQueue(c)
	return c
}

// SetWriteStrategy ...
func (c *Cache) SetWriteStrategy(s string) (err error) {
	c.Lock()
	defer c.Unlock()

	switch s {
	case "max":
		c.writeStrategy = MaximumLength
	case "sort":
		c.writeStrategy = TimestampOrder
	case "noop":
		c.writeStrategy = Noop
	default:
		return fmt.Errorf("Unknown write strategy '%s', should be one of: max, sort, noop", s)
	}
	return nil
}

// SetMaxSize of cache
func (c *Cache) SetMaxSize(maxSize uint32) {
	c.maxSize = int32(maxSize)
}

func (c *Cache) Stop() {}

// Collect cache metrics
func (c *Cache) Stat(send helper.StatCallback) {
	send("size", float64(c.Size()))
	send("metrics", float64(c.Len()))
	send("maxSize", float64(c.maxSize))

	helper.SendAndSubstractUint32("queries", &c.stat.queryCnt, send)
	helper.SendAndSubstractUint32("overflow", &c.stat.overflowCnt, send)

	helper.SendAndSubstractUint32("queueBuildCount", &c.stat.queueBuildCnt, send)
	helper.SendAndSubstractUint32("queueBuildTimeMs", &c.stat.queueBuildTimeMs, send)
	helper.SendUint32("queueWriteoutTime", &c.stat.queueWriteoutTime, send)

	// confirm tracker stats
	notConfirmedUsedTotal := 0
	notConfirmedLenTotal := 0
	notConfirmedShardUsedMax := 0
	notConfirmedShardLenMax := 0

	for i := 0; i < shardCount; i++ {
		shard := c.data[i]
		shard.Lock()

		l := len(shard.notConfirmed)

		notConfirmedUsedTotal += shard.notConfirmedUsed
		notConfirmedLenTotal += l
		if shard.notConfirmedUsed > notConfirmedShardUsedMax {
			notConfirmedShardUsedMax = shard.notConfirmedUsed
		}
		if l > notConfirmedShardLenMax {
			notConfirmedShardLenMax = l
		}
		shard.Unlock()
	}

	send("notConfirmedUsedTotal", float64(notConfirmedUsedTotal))
	send("notConfirmedLenTotal", float64(notConfirmedLenTotal))
	send("notConfirmedShardUsedMax", float64(notConfirmedShardUsedMax))
	send("notConfirmedShardLenMax", float64(notConfirmedShardLenMax))
}

// hash function
// @TODO: try crc32 or something else?
func fnv32(key string) uint32 {
	hash := uint32(2166136261)
	const prime32 = uint32(16777619)
	for i := 0; i < len(key); i++ {
		hash *= prime32
		hash ^= uint32(key[i])
	}
	return hash
}

// Returns shard under given key
func (c *Cache) GetShard(key string) *Shard {
	// @TODO: remove type casts?
	return c.data[uint(fnv32(key))%uint(shardCount)]
}

func (c *Cache) Get(key string) []points.Point {
	atomic.AddUint32(&c.stat.queryCnt, 1)

	shard := c.GetShard(key)

	var data []points.Point
	shard.Lock()
	for _, p := range shard.notConfirmed[:shard.notConfirmedUsed] {
		if p.Metric == key {
			if p != nil {
				if data == nil {
					data = p.Data
				} else {
					data = append(data, p.Data...)
				}
			}
		}
	}

	if p, exists := shard.items[key]; exists {
		if data == nil {
			data = p.Data
		} else {
			data = append(data, p.Data...)
		}
	}
	shard.Unlock()
	return data
}

func (c *Cache) Confirm(p *points.Points) {
	var i, j int
	shard := c.GetShard(p.Metric)

	shard.Lock()
	for i = 0; i < shard.notConfirmedUsed; i++ {
		if shard.notConfirmed[i] == p {
			shard.notConfirmed[i] = nil

			for j = i + 1; j < shard.notConfirmedUsed; j++ {
				shard.notConfirmed[j-1] = shard.notConfirmed[j]
			}

			shard.notConfirmedUsed--
		}
	}
	shard.Unlock()
}

func (c *Cache) Len() int32 {
	l := 0
	for i := 0; i < shardCount; i++ {
		shard := c.data[i]
		shard.Lock()
		l += len(shard.items)
		shard.Unlock()
	}
	return int32(l)
}

func (c *Cache) Size() int32 {
	return atomic.LoadInt32(&c.stat.size)
}

func (c *Cache) DivertToXlog(w io.Writer) {
	c.xlog.Store(w)
}

func (c *Cache) Dump(w io.Writer) error {
	for i := 0; i < shardCount; i++ {
		shard := c.data[i]
		shard.Lock()

		for _, p := range shard.notConfirmed[:shard.notConfirmedUsed] {
			if p == nil {
				continue
			}
			if _, err := p.WriteTo(w); err != nil {
				return err
			}
		}

		for _, p := range shard.items {
			if _, err := p.WriteTo(w); err != nil {
				return err
			}
		}

		shard.Unlock()
	}

	return nil
}

// Sets the given value under the specified key.
func (c *Cache) Add(p *points.Points) {
	xlog := c.xlog.Load()

	if xlog != nil {
		p.WriteTo(xlog.(io.Writer))
	}

	// Get map shard.
	count := len(p.Data)

	if c.maxSize > 0 && c.Size() > c.maxSize {
		atomic.AddUint32(&c.stat.overflowCnt, uint32(count))
		return
	}

	shard := c.GetShard(p.Metric)

	shard.Lock()
	if values, exists := shard.items[p.Metric]; exists {
		values.Data = append(values.Data, p.Data...)
	} else {
		shard.items[p.Metric] = p
	}
	shard.Unlock()

	atomic.AddInt32(&c.stat.size, int32(count))
}

// Removes an element from the map and returns it
func (c *Cache) Pop(key string) (p *points.Points, exists bool) {
	// Try to get shard.
	shard := c.GetShard(key)
	shard.Lock()
	p, exists = shard.items[key]
	delete(shard.items, key)
	shard.Unlock()

	if exists {
		atomic.AddInt32(&c.stat.size, -int32(len(p.Data)))
	}

	return p, exists
}

func (c *Cache) PopNotConfirmed(key string) (p *points.Points, exists bool) {
	// Try to get shard.
	shard := c.GetShard(key)
	shard.Lock()
	p, exists = shard.items[key]
	delete(shard.items, key)

	if exists {
		if shard.notConfirmedUsed < len(shard.notConfirmed) {
			shard.notConfirmed[shard.notConfirmedUsed] = p
		} else {
			shard.notConfirmed = append(shard.notConfirmed, p)
		}
		shard.notConfirmedUsed++
	}
	shard.Unlock()

	if exists {
		atomic.AddInt32(&c.stat.size, -int32(len(p.Data)))
	}

	return p, exists
}

func (c *Cache) WriteoutQueue() *WriteoutQueue {
	return c.writeoutQueue
}
<commit_msg>fix nil pointer validation<commit_after>package cache

/*
Based on https://github.com/orcaman/concurrent-map
*/

import (
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"

	"github.com/lomik/go-carbon/helper"
	"github.com/lomik/go-carbon/points"
)

type WriteStrategy int

const (
	MaximumLength WriteStrategy = iota
	TimestampOrder
	Noop
)

const shardCount = 1024

// A "thread" safe map of type string:Anything.
// To avoid lock bottlenecks this map is dived to several (shardCount) map shards.
type Cache struct {
	sync.Mutex

	queueLastBuild time.Time

	data []*Shard

	maxSize       int32
	writeStrategy WriteStrategy

	writeoutQueue *WriteoutQueue

	xlog atomic.Value // io.Writer

	stat struct {
		size              int32  // changing via atomic
		queueBuildCnt     uint32 // number of times writeout queue was built
		queueBuildTimeMs  uint32 // time spent building writeout queue in milliseconds
		queueWriteoutTime uint32 // in milliseconds
		overflowCnt       uint32 // drop packages if cache full
		queryCnt          uint32 // number of queries
	}
}

// A "thread" safe string to anything map.
type Shard struct {
	sync.RWMutex     // Read Write mutex, guards access to internal map.
	items            map[string]*points.Points
	notConfirmed     []*points.Points // linear search for value/slot
	notConfirmedUsed int              // search value in notConfirmed[:notConfirmedUsed]
}

// Creates a new cache instance
func New() *Cache {
	c := &Cache{
		data:          make([]*Shard, shardCount),
		writeStrategy: Noop,
		maxSize:       1000000,
	}

	for i := 0; i < shardCount; i++ {
		c.data[i] = &Shard{
			items:        make(map[string]*points.Points),
			notConfirmed: make([]*points.Points, 4),
		}
	}

	c.writeoutQueue = NewWriteoutQueue(c)
	return c
}

// SetWriteStrategy ...
func (c *Cache) SetWriteStrategy(s string) (err error) {
	c.Lock()
	defer c.Unlock()

	switch s {
	case "max":
		c.writeStrategy = MaximumLength
	case "sort":
		c.writeStrategy = TimestampOrder
	case "noop":
		c.writeStrategy = Noop
	default:
		return fmt.Errorf("Unknown write strategy '%s', should be one of: max, sort, noop", s)
	}
	return nil
}

// SetMaxSize of cache
func (c *Cache) SetMaxSize(maxSize uint32) {
	c.maxSize = int32(maxSize)
}

func (c *Cache) Stop() {}

// Collect cache metrics
func (c *Cache) Stat(send helper.StatCallback) {
	send("size", float64(c.Size()))
	send("metrics", float64(c.Len()))
	send("maxSize", float64(c.maxSize))

	helper.SendAndSubstractUint32("queries", &c.stat.queryCnt, send)
	helper.SendAndSubstractUint32("overflow", &c.stat.overflowCnt, send)

	helper.SendAndSubstractUint32("queueBuildCount", &c.stat.queueBuildCnt, send)
	helper.SendAndSubstractUint32("queueBuildTimeMs", &c.stat.queueBuildTimeMs, send)
	helper.SendUint32("queueWriteoutTime", &c.stat.queueWriteoutTime, send)

	// confirm tracker stats
	notConfirmedUsedTotal := 0
	notConfirmedLenTotal := 0
	notConfirmedShardUsedMax := 0
	notConfirmedShardLenMax := 0

	for i := 0; i < shardCount; i++ {
		shard := c.data[i]
		shard.Lock()

		l := len(shard.notConfirmed)

		notConfirmedUsedTotal += shard.notConfirmedUsed
		notConfirmedLenTotal += l
		if shard.notConfirmedUsed > notConfirmedShardUsedMax {
			notConfirmedShardUsedMax = shard.notConfirmedUsed
		}
		if l > notConfirmedShardLenMax {
			notConfirmedShardLenMax = l
		}
		shard.Unlock()
	}

	send("notConfirmedUsedTotal", float64(notConfirmedUsedTotal))
	send("notConfirmedLenTotal", float64(notConfirmedLenTotal))
	send("notConfirmedShardUsedMax", float64(notConfirmedShardUsedMax))
	send("notConfirmedShardLenMax", float64(notConfirmedShardLenMax))
}

// hash function
// @TODO: try crc32 or something else?
func fnv32(key string) uint32 {
	hash := uint32(2166136261)
	const prime32 = uint32(16777619)
	for i := 0; i < len(key); i++ {
		hash *= prime32
		hash ^= uint32(key[i])
	}
	return hash
}

// Returns shard under given key
func (c *Cache) GetShard(key string) *Shard {
	// @TODO: remove type casts?
	return c.data[uint(fnv32(key))%uint(shardCount)]
}

func (c *Cache) Get(key string) []points.Point {
	atomic.AddUint32(&c.stat.queryCnt, 1)

	shard := c.GetShard(key)

	var data []points.Point
	shard.Lock()
	for _, p := range shard.notConfirmed[:shard.notConfirmedUsed] {
		if p != nil && p.Metric == key {
			if data == nil {
				data = p.Data
			} else {
				data = append(data, p.Data...)
			}
		}
	}

	if p, exists := shard.items[key]; exists {
		if data == nil {
			data = p.Data
		} else {
			data = append(data, p.Data...)
		}
	}
	shard.Unlock()
	return data
}

func (c *Cache) Confirm(p *points.Points) {
	var i, j int
	shard := c.GetShard(p.Metric)

	shard.Lock()
	for i = 0; i < shard.notConfirmedUsed; i++ {
		if shard.notConfirmed[i] == p {
			shard.notConfirmed[i] = nil

			for j = i + 1; j < shard.notConfirmedUsed; j++ {
				shard.notConfirmed[j-1] = shard.notConfirmed[j]
			}

			shard.notConfirmedUsed--
		}
	}
	shard.Unlock()
}

func (c *Cache) Len() int32 {
	l := 0
	for i := 0; i < shardCount; i++ {
		shard := c.data[i]
		shard.Lock()
		l += len(shard.items)
		shard.Unlock()
	}
	return int32(l)
}

func (c *Cache) Size() int32 {
	return atomic.LoadInt32(&c.stat.size)
}

func (c *Cache) DivertToXlog(w io.Writer) {
	c.xlog.Store(w)
}

func (c *Cache) Dump(w io.Writer) error {
	for i := 0; i < shardCount; i++ {
		shard := c.data[i]
		shard.Lock()

		for _, p := range shard.notConfirmed[:shard.notConfirmedUsed] {
			if p == nil {
				continue
			}
			if _, err := p.WriteTo(w); err != nil {
				return err
			}
		}

		for _, p := range shard.items {
			if _, err := p.WriteTo(w); err != nil {
				return err
			}
		}

		shard.Unlock()
	}

	return nil
}

// Sets the given value under the specified key.
func (c *Cache) Add(p *points.Points) {
	xlog := c.xlog.Load()

	if xlog != nil {
		p.WriteTo(xlog.(io.Writer))
	}

	// Get map shard.
	count := len(p.Data)

	if c.maxSize > 0 && c.Size() > c.maxSize {
		atomic.AddUint32(&c.stat.overflowCnt, uint32(count))
		return
	}

	shard := c.GetShard(p.Metric)

	shard.Lock()
	if values, exists := shard.items[p.Metric]; exists {
		values.Data = append(values.Data, p.Data...)
	} else {
		shard.items[p.Metric] = p
	}
	shard.Unlock()

	atomic.AddInt32(&c.stat.size, int32(count))
}

// Removes an element from the map and returns it
func (c *Cache) Pop(key string) (p *points.Points, exists bool) {
	// Try to get shard.
	shard := c.GetShard(key)
	shard.Lock()
	p, exists = shard.items[key]
	delete(shard.items, key)
	shard.Unlock()

	if exists {
		atomic.AddInt32(&c.stat.size, -int32(len(p.Data)))
	}

	return p, exists
}

func (c *Cache) PopNotConfirmed(key string) (p *points.Points, exists bool) {
	// Try to get shard.
	shard := c.GetShard(key)
	shard.Lock()
	p, exists = shard.items[key]
	delete(shard.items, key)

	if exists {
		if shard.notConfirmedUsed < len(shard.notConfirmed) {
			shard.notConfirmed[shard.notConfirmedUsed] = p
		} else {
			shard.notConfirmed = append(shard.notConfirmed, p)
		}
		shard.notConfirmedUsed++
	}
	shard.Unlock()

	if exists {
		atomic.AddInt32(&c.stat.size, -int32(len(p.Data)))
	}

	return p, exists
}

func (c *Cache) WriteoutQueue() *WriteoutQueue {
	return c.writeoutQueue
}
<|endoftext|>


<commit_before>package captainslog

import "errors"

var (
	// ErrTransform is returned by a Transformer when it cannot
	// perform its function
	ErrTransform = errors.New("transform error")
)

// ChannelerCmd represents a command that can
// be sent to a Channeler
type ChannelerCmd int

const (
	// CmdStop tells a Channeler to stop
	CmdStop ChannelerCmd = iota
)

// MatchType represents types of matches that can
// be made against a SyslogMsg
type MatchType int

const (
	// Program match is an exact match against syslog program name
	Program = iota

	// Contains match checks if the syslog content contains a string
	Contains
)

// Transformer accept a SyslogMsg, and return a modified SyslogMsg
type Transformer interface {
	Transform(SyslogMsg) (SyslogMsg, error)
}

// Mutator accepts a pointer to a SyslogMsg and modifies it in place
type Mutator interface {
	Mutate(*SyslogMsg) error
}

// Matcher accepts a SyslogMsg and returns true if it matches
type Matcher interface {
	Match(msg *SyslogMsg) bool
}

// Outputter is an interface that provides
// specific functionality to OutputChannelers. They
// are transport adapters - for instance, TCPOutputter
// converts *Syslog messages received off a channeler
// to RFC3164 []byte encoded syslog messages sent over TCP.
type Outputter interface {
	Output(s *SyslogMsg) (int, error)
	Connect() error
	RetryInterval() int
	Close()
}

// Inputter is an interface that provides
// specific functionality to InputChannelers. They
// are transport adapters - for instance, TCPInputter
// converts syslog byte's received over TCP to a
// *SyslogMsg and sends it over a channel.
type Inputter interface {
	Listen() <-chan *SyslogMsg
	Close()
}
<commit_msg>Problem: MatchType was not properly typed<commit_after>package captainslog

import "errors"

var (
	// ErrTransform is returned by a Transformer when it cannot
	// perform its function
	ErrTransform = errors.New("transform error")
)

// ChannelerCmd represents a command that can
// be sent to a Channeler
type ChannelerCmd int

const (
	// CmdStop tells a Channeler to stop
	CmdStop ChannelerCmd = iota
)

// MatchType represents types of matches that can
// be made against a SyslogMsg
type MatchType int

const (
	// Program match is an exact match against syslog program name
	Program MatchType = iota

	// Contains match checks if the syslog content contains a string
	Contains
)

// Transformer accept a SyslogMsg, and return a modified SyslogMsg
type Transformer interface {
	Transform(SyslogMsg) (SyslogMsg, error)
}

// Mutator accepts a pointer to a SyslogMsg and modifies it in place
type Mutator interface {
	Mutate(*SyslogMsg) error
}

// Matcher accepts a SyslogMsg and returns true if it matches
type Matcher interface {
	Match(msg *SyslogMsg) bool
}

// Outputter is an interface that provides
// specific functionality to OutputChannelers. They
// are transport adapters - for instance, TCPOutputter
// converts *Syslog messages received off a channeler
// to RFC3164 []byte encoded syslog messages sent over TCP.
type Outputter interface {
	Output(s *SyslogMsg) (int, error)
	Connect() error
	RetryInterval() int
	Close()
}

// Inputter is an interface that provides
// specific functionality to InputChannelers. They
// are transport adapters - for instance, TCPInputter
// converts syslog byte's received over TCP to a
// *SyslogMsg and sends it over a channel.
type Inputter interface {
	Listen() <-chan *SyslogMsg
	Close()
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package cases provides general and language-specific case mappers.
package cases // import "golang.org/x/text/cases"

import (
	"golang.org/x/text/language"
	"golang.org/x/text/transform"
)

// References:
// - Unicode Reference Manual Chapter 3.13, 4.2, and 5.18.
// - http://www.unicode.org/reports/tr29/
// - http://www.unicode.org/Public/6.3.0/ucd/CaseFolding.txt
// - http://www.unicode.org/Public/6.3.0/ucd/SpecialCasing.txt
// - http://www.unicode.org/Public/6.3.0/ucd/DerivedCoreProperties.txt
// - http://www.unicode.org/Public/6.3.0/ucd/auxiliary/WordBreakProperty.txt
// - http://www.unicode.org/Public/6.3.0/ucd/auxiliary/WordBreakTest.txt
// - http://userguide.icu-project.org/transforms/casemappings

// TODO:
// - Case folding
// - Wide and Narrow?
// - Segmenter option for title casing.
// - ASCII fast paths
// - Encode Soft-Dotted property within trie somehow.

// A Caser transforms given input to a certain case. It implements
// transform.Transformer.
//
// A Caser may be stateful and should therefore not be shared between
// goroutines.
type Caser struct {
	t transform.Transformer
}

// Bytes returns a new byte slice with the result of converting b to the case
// form implemented by c.
func (c Caser) Bytes(b []byte) []byte {
	b, _, _ = transform.Bytes(c.t, b)
	return b
}

// String returns a string with the result of transforming s to the case form
// implemented by c.
func (c Caser) String(s string) string {
	s, _, _ = transform.String(c.t, s)
	return s
}

// Reset resets the Caser to be reused for new input after a previous call to
// Transform.
func (c Caser) Reset() { c.t.Reset() }

// Transform implements the Transformer interface and transforms the given input
// to the case form implemented by c.
func (c Caser) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
	return c.t.Transform(dst, src, atEOF)
}

// Upper returns a Caser for language-specific uppercasing.
func Upper(t language.Tag, opts ...Option) Caser {
	return Caser{makeUpper(t, getOpts(opts...))}
}

// Lower returns a Caser for language-specific lowercasing.
func Lower(t language.Tag, opts ...Option) Caser {
	return Caser{makeLower(t, getOpts(opts...))}
}

// Title returns a Caser for language-specific title casing. It uses an
// approximation of the default Unicode Word Break algorithm.
func Title(t language.Tag, opts ...Option) Caser {
	return Caser{makeTitle(t, getOpts(opts...))}
}

// An Option is used to modify the behavior of a Caser.
type Option func(o *options)

var (
	// NoLower disables the lowercasing of non-leading letters for a title
	// caser.
	NoLower Option = noLower
)

type options struct {
	noLower bool

	// TODO: segmenter, max ignorable, alternative versions, etc.

	noFinalSigma bool // Only used for testing.
}

func getOpts(o ...Option) (res options) {
	for _, f := range o {
		f(&res)
	}
	return
}

func noLower(o *options) {
	o.noLower = true
}
<commit_msg>text/cases: API proposal for case folding.<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package cases provides general and language-specific case mappers.
package cases // import "golang.org/x/text/cases"

import (
	"golang.org/x/text/language"
	"golang.org/x/text/transform"
)

// References:
// - Unicode Reference Manual Chapter 3.13, 4.2, and 5.18.
// - http://www.unicode.org/reports/tr29/
// - http://www.unicode.org/Public/6.3.0/ucd/CaseFolding.txt
// - http://www.unicode.org/Public/6.3.0/ucd/SpecialCasing.txt
// - http://www.unicode.org/Public/6.3.0/ucd/DerivedCoreProperties.txt
// - http://www.unicode.org/Public/6.3.0/ucd/auxiliary/WordBreakProperty.txt
// - http://www.unicode.org/Public/6.3.0/ucd/auxiliary/WordBreakTest.txt
// - http://userguide.icu-project.org/transforms/casemappings

// TODO:
// - Case folding
// - Wide and Narrow?
// - Segmenter option for title casing.
// - ASCII fast paths
// - Encode Soft-Dotted property within trie somehow.

// A Caser transforms given input to a certain case. It implements
// transform.Transformer.
//
// A Caser may be stateful and should therefore not be shared between
// goroutines.
type Caser struct {
	t transform.Transformer
}

// Bytes returns a new byte slice with the result of converting b to the case
// form implemented by c.
func (c Caser) Bytes(b []byte) []byte {
	b, _, _ = transform.Bytes(c.t, b)
	return b
}

// String returns a string with the result of transforming s to the case form
// implemented by c.
func (c Caser) String(s string) string {
	s, _, _ = transform.String(c.t, s)
	return s
}

// Reset resets the Caser to be reused for new input after a previous call to
// Transform.
func (c Caser) Reset() { c.t.Reset() }

// Transform implements the Transformer interface and transforms the given input
// to the case form implemented by c.
func (c Caser) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
	return c.t.Transform(dst, src, atEOF)
}

// Upper returns a Caser for language-specific uppercasing.
func Upper(t language.Tag, opts ...Option) Caser {
	return Caser{makeUpper(t, getOpts(opts...))}
}

// Lower returns a Caser for language-specific lowercasing.
func Lower(t language.Tag, opts ...Option) Caser {
	return Caser{makeLower(t, getOpts(opts...))}
}

// Title returns a Caser for language-specific title casing. It uses an
// approximation of the default Unicode Word Break algorithm.
func Title(t language.Tag, opts ...Option) Caser {
	return Caser{makeTitle(t, getOpts(opts...))}
}

// Fold returns a Caser that implements Unicode case folding.
//
// Folding is like mapping to lowercase without context-dependent mappings. Its
// primary use is for caseless matching. Note that case folding does not
// normalize the input and may not preserve a normal form.
func Fold(t language.Tag, opts ...Option) Caser {
	panic("TODO: implement")
}

// An Option is used to modify the behavior of a Caser.
type Option func(o *options)

var (
	// NoLower disables the lowercasing of non-leading letters for a title
	// caser.
	NoLower Option = noLower

	// Compact omits mappings in case folding for characters that would grow the
	// input.
	Compact Option = compact
)

// TODO: option to preserve a normal form, if applicable?

type options struct {
	noLower bool
	simple  bool

	// TODO: segmenter, max ignorable, alternative versions, etc.

	noFinalSigma bool // Only used for testing.
}

func getOpts(o ...Option) (res options) {
	for _, f := range o {
		f(&res)
	}
	return
}

func noLower(o *options) {
	o.noLower = true
}

func compact(o *options) {
	o.simple = true
}
<|endoftext|>


<commit_before>package gopdf

//Left left
const Left = 8 //001000
//Top top
const Top = 4 //000100
//Right right
const Right = 2 //000010
//Bottom bottom
const Bottom = 1 //000001
//Center center
const Center = 16 //010000
//Middle middle
const Middle = 32 //100000

//CellOption cell option
type CellOption struct {
	Align  int //Allows to align the text. Possible values are: Left,Center,Right,Top,Bottom,Middle
	Border int //Indicates if borders must be drawn around the cell. Possible values are: Left, Top, Right, Bottom, ALL
	Float  int //Indicates where the current position should go after the call. Possible values are: Right, Bottom
}
<commit_msg>add AllBorders Parameter<commit_after>package gopdf

//Left left
const Left = 8 //001000
//Top top
const Top = 4 //000100
//Right right
const Right = 2 //000010
//Bottom bottom
const Bottom = 1 //000001
//Center center
const Center = 16 //010000
//Middle middle
const Middle = 32 //100000
//AllBorders allborders
const AllBorders = 15 //001111

//CellOption cell option
type CellOption struct {
	Align  int //Allows to align the text. Possible values are: Left,Center,Right,Top,Bottom,Middle
	Border int //Indicates if borders must be drawn around the cell. Possible values are: Left, Top, Right, Bottom, AllBorders
	Float  int //Indicates where the current position should go after the call. Possible values are: Right, Bottom
}
<|endoftext|>


<commit_before>// Challenge 33 - Implement Diffie-Hellman
// http://cryptopals.com/sets/5/challenges/33

package cryptopals

import (
	"crypto/rand"
	"math/big"
)

type challenge33 struct {
}

type network struct {
	in  <-chan *big.Int
	out chan<- *big.Int
}

var p = new(big.Int)
var g = new(big.Int)

func init() {
	p.SetString("ffffffffffffffffc90fdaa22168c234c4c6628b80dc1cd129024"+
		"e088a67cc74020bbea63b139b22514a08798e3404ddef9519b3cd"+
		"3a431b302b0a6df25f14374fe1356d6d51c245e485b576625e7ec"+
		"6f44c42e9a637ed6b0bff5cb6f406b7edee386bfb5a899fa5ae9f"+
		"24117c4b1fe649286651ece45b3dc2007cb8a163bf0598da48361"+
		"c55d39a69163fa8fd24cf5f83655d23dca3ad961c62f356208552"+
		"bb9ed529077096966d670c354e4abc9804f1746c08ca237327fff"+
		"fffffffffffff", 16)

	g.SetInt64(2)
}

func (challenge33) NewNetwork() *network {
	return &network{
		in:  make(chan *big.Int, 1),
		out: make(chan *big.Int, 1),
	}
}

func (net *network) Read() *big.Int {
	return <-net.in
}

func (net *network) Write(n *big.Int) {
	net.out <- n
}

func (challenge33) DiffieHellman(net *network) *big.Int {
	a, _ := rand.Int(rand.Reader, p)

	A := new(big.Int)
	A.Exp(g, a, p)

	net.Write(A)
	B := net.Read()

	s := new(big.Int)
	s.Exp(B, a, p)

	return s
}
<commit_msg>Add simple Diffie-Hellman key exchange test<commit_after>// Challenge 33 - Implement Diffie-Hellman
// http://cryptopals.com/sets/5/challenges/33

package cryptopals

import (
	"crypto/rand"
	"math/big"
)

type challenge33 struct {
}

type network struct {
	in  <-chan *big.Int
	out chan<- *big.Int
}

var p = new(big.Int)
var g = new(big.Int)

func init() {
	p.SetString("ffffffffffffffffc90fdaa22168c234c4c6628b80dc1cd129024"+
		"e088a67cc74020bbea63b139b22514a08798e3404ddef9519b3cd"+
		"3a431b302b0a6df25f14374fe1356d6d51c245e485b576625e7ec"+
		"6f44c42e9a637ed6b0bff5cb6f406b7edee386bfb5a899fa5ae9f"+
		"24117c4b1fe649286651ece45b3dc2007cb8a163bf0598da48361"+
		"c55d39a69163fa8fd24cf5f83655d23dca3ad961c62f356208552"+
		"bb9ed529077096966d670c354e4abc9804f1746c08ca237327fff"+
		"fffffffffffff", 16)

	g.SetInt64(2)
}

func (net *network) Read() *big.Int {
	return <-net.in
}

func (net *network) Write(n *big.Int) {
	net.out <- n
}

func (challenge33) DiffieHellman(net *network) *big.Int {
	a, _ := rand.Int(rand.Reader, p)

	A := new(big.Int)
	A.Exp(g, a, p)

	net.Write(A)
	B := net.Read()

	s := new(big.Int)
	s.Exp(B, a, p)

	return s
}
<|endoftext|>


<commit_before>package sdl

// #include "sdl_wrapper.h"
import "C"
import "unsafe"

// Point (https://wiki.libsdl.org/SDL_Point)
type Point struct {
	X int32
	Y int32
}

// Rect (https://wiki.libsdl.org/SDL_Rect)
type Rect struct {
	X int32
	Y int32
	W int32
	H int32
}

func (p *Point) cptr() *C.SDL_Point {
	return (*C.SDL_Point)(unsafe.Pointer(p))
}

func (r *Rect) cptr() *C.SDL_Rect {
	return (*C.SDL_Rect)(unsafe.Pointer(r))
}

// Rect (https://wiki.libsdl.org/SDL_RectEmpty)
func (r *Rect) Empty() bool {
	return r == nil || r.W <= 0 || r.H <= 0
}

// Rect (https://wiki.libsdl.org/SDL_RectEquals)
func (a *Rect) Equals(b *Rect) bool {
	if (a != nil) && (b != nil) &&
		(a.X == b.X) && (a.Y == a.Y) &&
		(a.W == b.W) && (a.H == b.H) {
		return true
	}
	return false
}

// Rect (https://wiki.libsdl.org/SDL_HasIntersection)
func (a *Rect) HasIntersection(b *Rect) bool {
	return C.SDL_HasIntersection(a.cptr(), b.cptr()) > 0
}

// Rect (https://wiki.libsdl.org/SDL_IntersectRect)
func (a *Rect) Intersect(b *Rect) (result Rect, ok bool) {
	ok = C.SDL_IntersectRect(a.cptr(), b.cptr(), result.cptr()) > 0
    return
}

// Rect (https://wiki.libsdl.org/SDL_UnionRect)
func (a *Rect) Union(b *Rect) (result Rect) {
	C.SDL_UnionRect(a.cptr(), b.cptr(), result.cptr())
    return
}

// EnclosePoints (https://wiki.libsdl.org/SDL_EnclosePoints)
func EnclosePoints(points []Point, clip *Rect) (result Rect, ok bool) {
	ok = C.SDL_EnclosePoints(points[0].cptr(), C.int(len(points)), clip.cptr(), result.cptr()) > 0
    return
}

// IntersectLine
func (rect *Rect) IntersectLine(X1, Y1, X2, Y2 *int) bool {
	_X1 := (*C.int)(unsafe.Pointer(X1))
	_Y1 := (*C.int)(unsafe.Pointer(Y1))
	_X2 := (*C.int)(unsafe.Pointer(X2))
	_Y2 := (*C.int)(unsafe.Pointer(Y2))
	return C.SDL_IntersectRectAndLine(rect.cptr(), _X1, _Y1, _X2, _Y2) > 0
}
<commit_msg>gofmt fixes<commit_after>package sdl

// #include "sdl_wrapper.h"
import "C"
import "unsafe"

// Point (https://wiki.libsdl.org/SDL_Point)
type Point struct {
	X int32
	Y int32
}

// Rect (https://wiki.libsdl.org/SDL_Rect)
type Rect struct {
	X int32
	Y int32
	W int32
	H int32
}

func (p *Point) cptr() *C.SDL_Point {
	return (*C.SDL_Point)(unsafe.Pointer(p))
}

func (r *Rect) cptr() *C.SDL_Rect {
	return (*C.SDL_Rect)(unsafe.Pointer(r))
}

// Rect (https://wiki.libsdl.org/SDL_RectEmpty)
func (r *Rect) Empty() bool {
	return r == nil || r.W <= 0 || r.H <= 0
}

// Rect (https://wiki.libsdl.org/SDL_RectEquals)
func (a *Rect) Equals(b *Rect) bool {
	if (a != nil) && (b != nil) &&
		(a.X == b.X) && (a.Y == a.Y) &&
		(a.W == b.W) && (a.H == b.H) {
		return true
	}
	return false
}

// Rect (https://wiki.libsdl.org/SDL_HasIntersection)
func (a *Rect) HasIntersection(b *Rect) bool {
	return C.SDL_HasIntersection(a.cptr(), b.cptr()) > 0
}

// Rect (https://wiki.libsdl.org/SDL_IntersectRect)
func (a *Rect) Intersect(b *Rect) (result Rect, ok bool) {
	ok = C.SDL_IntersectRect(a.cptr(), b.cptr(), result.cptr()) > 0
	return
}

// Rect (https://wiki.libsdl.org/SDL_UnionRect)
func (a *Rect) Union(b *Rect) (result Rect) {
	C.SDL_UnionRect(a.cptr(), b.cptr(), result.cptr())
	return
}

// EnclosePoints (https://wiki.libsdl.org/SDL_EnclosePoints)
func EnclosePoints(points []Point, clip *Rect) (result Rect, ok bool) {
	ok = C.SDL_EnclosePoints(points[0].cptr(), C.int(len(points)), clip.cptr(), result.cptr()) > 0
	return
}

// IntersectLine
func (rect *Rect) IntersectLine(X1, Y1, X2, Y2 *int) bool {
	_X1 := (*C.int)(unsafe.Pointer(X1))
	_Y1 := (*C.int)(unsafe.Pointer(Y1))
	_X2 := (*C.int)(unsafe.Pointer(X2))
	_Y2 := (*C.int)(unsafe.Pointer(Y2))
	return C.SDL_IntersectRectAndLine(rect.cptr(), _X1, _Y1, _X2, _Y2) > 0
}
<|endoftext|>


<commit_before>package selenium

import (
	"testing"
)

/* Element finding options */
const (
	ById              = "id"
	ByXPATH           = "xpath"
	ByLinkText        = "link text"
	ByPartialLinkText = "partial link text"
	ByName            = "name"
	ByTagName         = "tag name"
	ByClassName       = "class name"
	ByCSSSelector     = "css selector"
)

/* Mouse buttons */
const (
	LeftButton = iota
	MiddleButton
	RightButton
)

/* Keys */
const (
	NullKey       = string('\ue000')
	CancelKey     = string('\ue001')
	HelpKey       = string('\ue002')
	BackspaceKey  = string('\ue003')
	TabKey        = string('\ue004')
	ClearKey      = string('\ue005')
	ReturnKey     = string('\ue006')
	EnterKey      = string('\ue007')
	ShiftKey      = string('\ue008')
	ControlKey    = string('\ue009')
	AltKey        = string('\ue00a')
	PauseKey      = string('\ue00b')
	EscapeKey     = string('\ue00c')
	SpaceKey      = string('\ue00d')
	PageUpKey     = string('\ue00e')
	PageDownKey   = string('\ue00f')
	EndKey        = string('\ue010')
	HomeKey       = string('\ue011')
	LeftArrowKey  = string('\ue012')
	UpArrowKey    = string('\ue013')
	RightArrowKey = string('\ue014')
	DownArrowKey  = string('\ue015')
	InsertKey     = string('\ue016')
	DeleteKey     = string('\ue017')
	SemicolonKey  = string('\ue018')
	EqualsKey     = string('\ue019')
	Numpad0Key    = string('\ue01a')
	Numpad1Key    = string('\ue01b')
	Numpad2Key    = string('\ue01c')
	Numpad3Key    = string('\ue01d')
	Numpad4Key    = string('\ue01e')
	Numpad5Key    = string('\ue01f')
	Numpad6Key    = string('\ue020')
	Numpad7Key    = string('\ue021')
	Numpad8Key    = string('\ue022')
	Numpad9Key    = string('\ue023')
	MultiplyKey   = string('\ue024')
	AddKey        = string('\ue025')
	SeparatorKey  = string('\ue026')
	SubstractKey  = string('\ue027')
	DecimalKey    = string('\ue028')
	DivideKey     = string('\ue029')
	F1Key         = string('\ue031')
	F2Key         = string('\ue032')
	F3Key         = string('\ue033')
	F4Key         = string('\ue034')
	F5Key         = string('\ue035')
	F6Key         = string('\ue036')
	F7Key         = string('\ue037')
	F8Key         = string('\ue038')
	F9Key         = string('\ue039')
	F10Key        = string('\ue03a')
	F11Key        = string('\ue03b')
	F12Key        = string('\ue03c')
	MetaKey       = string('\ue03d')
)

/* Browser capabilities, see
http://code.google.com/p/selenium/wiki/JsonWireProtocol#Capabilities_JSON_Object
*/
type Capabilities map[string]interface{}

/* Build object, part of Status return. */
type Build struct {
	Version, Revision, Time string
}

/* OS object, part of Status return. */
type OS struct {
	Arch, Name, Version string
}

/* Information retured by Status method. */
type Status struct {
	Build `json:"build"`
	OS    `json:"os"`
}

/* Point */
type Point struct {
	X, Y int
}

/* Size */
type Size struct {
	Width, Height int
}

/* Cookie */
type Cookie struct {
	Name   string `json:"name"`
	Value  string `json:"value"`
	Path   string `json:"path"`
	Domain string `json:"domain"`
	Secure bool   `json:"secure"`
	Expiry uint   `json:"expiry"`
}

type WebDriver interface {
	/* Status (info) on server */
	Status() (*Status, error)

	/* Start a new session, return session id */
	NewSession() (string, error)

	/* Current session capabilities */
	Capabilities() (Capabilities, error)
	/* Set the amount of time, in milliseconds, that asynchronous scripts are permitted to run before they are aborted. */
	SetAsyncScriptTimeout(ms uint) error
	/* Set the amount of time, in milliseconds, the driver should wait when searching for elements. */
	SetImplicitWaitTimeout(ms uint) error

	// IME
	/* List all available engines on the machine. */
	AvailableEngines() ([]string, error)
	/* Get the name of the active IME engine. */
	ActiveEngine() (string, error)
	/* Indicates whether IME input is active at the moment. */
	IsEngineActivated() (bool, error)
	/* De-activates the currently-active IME engine. */
	DeactivateEngine() error
	/* Make an engines active */
	ActivateEngine(engine string) error

	/* Quit (end) current session */
	Quit() error

	// Page information and manipulation
	/* Return id of current window handle. */
	CurrentWindowHandle() (string, error)
	/* Return ids of current open windows. */
	WindowHandles() ([]string, error)
	/* Current url. */
	CurrentURL() (string, error)
	/* Page title. */
	Title() (string, error)
	/* Get page source. */
	PageSource() (string, error)
	/* Close current window. */
	Close() error
	/* Switch to frame, frame parameter can be name or id. */
	SwitchFrame(frame string) error
	/* Swtich to window. */
	SwitchWindow(name string) error
	/* Close window. */
	CloseWindow(name string) error

	// Navigation
	/* Open url. */
	Get(url string) error
	/* Move forward in history. */
	Forward() error
	/* Move backward in history. */
	Back() error
	/* Refresh page. */
	Refresh() error

	// Finding element(s)
	/* Find, return one element. */
	FindElement(by, value string) (WebElement, error)
	/* Find, return list of elements. */
	FindElements(by, value string) ([]WebElement, error)
	/* Current active element. */
	ActiveElement() (WebElement, error)

	// Cookies
	/* Get all cookies */
	GetCookies() ([]Cookie, error)
	/* Add a cookies */
	AddCookie(cookie *Cookie) error
	/* Delete all cookies */
	DeleteAllCookies() error
	/* Delete a cookie */
	DeleteCookie(name string) error

	// Mouse
	/* Click mouse button, button should be on of RightButton, MiddleButton or
	LeftButton.
	*/
	Click(button int) error
	/* Dobule click */
	DoubleClick() error
	/* Mouse button down */
	ButtonDown() error
	/* Mouse button up */
	ButtonUp() error

	// Misc
	/* Send modifier key to active element.
	modifier can be one of ShiftKey, ControlKey, AltKey, MetaKey.
	*/
	SendModifier(modifier string, isDown bool) error
	Screenshot() ([]byte, error)

	// Alerts
	/* Dismiss current alert. */
	DismissAlert() error
	/* Accept current alert. */
	AcceptAlert() error
	/* Current alert text. */
	AlertText() (string, error)
	/* Set current alert text. */
	SetAlertText(text string) error

	// Scripts
	/* Execute a script. */
	ExecuteScript(script string, args []interface{}) (interface{}, error)
	/* Execute a script async. */
	ExecuteScriptAsync(script string, args []interface{}) (interface{}, error)

	// Get a WebDriverT of this element that has methods that call t.Fatalf upon encountering
	// errors instead of using multiple returns to indicate errors.
	T(t *testing.T) WebDriverT
}

type WebElement interface {
	// Manipulation

	/* Click on element */
	Click() error
	/* Send keys (type) into element */
	SendKeys(keys string) error
	/* Submit */
	Submit() error
	/* Clear */
	Clear() error
	/* Move mouse to relative coordinates */
	MoveTo(xOffset, yOffset int) error

	// Finding

	/* Find children, return one element. */
	FindElement(by, value string) (WebElement, error)
	/* Find children, return list of elements. */
	FindElements(by, value string) ([]WebElement, error)

	// Porperties

	/* Element name */
	TagName() (string, error)
	/* Text of element */
	Text() (string, error)
	/* Check if element is selected. */
	IsSelected() (bool, error)
	/* Check if element is enabled. */
	IsEnabled() (bool, error)
	/* Check if element is displayed. */
	IsDisplayed() (bool, error)
	/* Get element attribute. */
	GetAttribute(name string) (string, error)
	/* Element location. */
	Location() (*Point, error)
	/* Element location once it has been scrolled into view. */
	LocationInView() (*Point, error)
	/* Element size */
	Size() (*Size, error)
	/* Get element CSS property value. */
	CSSProperty(name string) (string, error)

	// Get a WebElementT of this element that has methods that call t.Fatalf upon encountering
	// errors instead of using multiple returns to indicate errors.
	T(t *testing.T) WebElementT
}
<commit_msg>Use float64 for JSON numbers (required for Chrome)<commit_after>package selenium

import (
	"testing"
)

/* Element finding options */
const (
	ById              = "id"
	ByXPATH           = "xpath"
	ByLinkText        = "link text"
	ByPartialLinkText = "partial link text"
	ByName            = "name"
	ByTagName         = "tag name"
	ByClassName       = "class name"
	ByCSSSelector     = "css selector"
)

/* Mouse buttons */
const (
	LeftButton = iota
	MiddleButton
	RightButton
)

/* Keys */
const (
	NullKey       = string('\ue000')
	CancelKey     = string('\ue001')
	HelpKey       = string('\ue002')
	BackspaceKey  = string('\ue003')
	TabKey        = string('\ue004')
	ClearKey      = string('\ue005')
	ReturnKey     = string('\ue006')
	EnterKey      = string('\ue007')
	ShiftKey      = string('\ue008')
	ControlKey    = string('\ue009')
	AltKey        = string('\ue00a')
	PauseKey      = string('\ue00b')
	EscapeKey     = string('\ue00c')
	SpaceKey      = string('\ue00d')
	PageUpKey     = string('\ue00e')
	PageDownKey   = string('\ue00f')
	EndKey        = string('\ue010')
	HomeKey       = string('\ue011')
	LeftArrowKey  = string('\ue012')
	UpArrowKey    = string('\ue013')
	RightArrowKey = string('\ue014')
	DownArrowKey  = string('\ue015')
	InsertKey     = string('\ue016')
	DeleteKey     = string('\ue017')
	SemicolonKey  = string('\ue018')
	EqualsKey     = string('\ue019')
	Numpad0Key    = string('\ue01a')
	Numpad1Key    = string('\ue01b')
	Numpad2Key    = string('\ue01c')
	Numpad3Key    = string('\ue01d')
	Numpad4Key    = string('\ue01e')
	Numpad5Key    = string('\ue01f')
	Numpad6Key    = string('\ue020')
	Numpad7Key    = string('\ue021')
	Numpad8Key    = string('\ue022')
	Numpad9Key    = string('\ue023')
	MultiplyKey   = string('\ue024')
	AddKey        = string('\ue025')
	SeparatorKey  = string('\ue026')
	SubstractKey  = string('\ue027')
	DecimalKey    = string('\ue028')
	DivideKey     = string('\ue029')
	F1Key         = string('\ue031')
	F2Key         = string('\ue032')
	F3Key         = string('\ue033')
	F4Key         = string('\ue034')
	F5Key         = string('\ue035')
	F6Key         = string('\ue036')
	F7Key         = string('\ue037')
	F8Key         = string('\ue038')
	F9Key         = string('\ue039')
	F10Key        = string('\ue03a')
	F11Key        = string('\ue03b')
	F12Key        = string('\ue03c')
	MetaKey       = string('\ue03d')
)

/* Browser capabilities, see
http://code.google.com/p/selenium/wiki/JsonWireProtocol#Capabilities_JSON_Object
*/
type Capabilities map[string]interface{}

/* Build object, part of Status return. */
type Build struct {
	Version, Revision, Time string
}

/* OS object, part of Status return. */
type OS struct {
	Arch, Name, Version string
}

/* Information retured by Status method. */
type Status struct {
	Build `json:"build"`
	OS    `json:"os"`
}

/* Point */
type Point struct {
	X, Y float64
}

/* Size */
type Size struct {
	Width, Height float64
}

/* Cookie */
type Cookie struct {
	Name   string `json:"name"`
	Value  string `json:"value"`
	Path   string `json:"path"`
	Domain string `json:"domain"`
	Secure bool   `json:"secure"`
	Expiry uint   `json:"expiry"`
}

type WebDriver interface {
	/* Status (info) on server */
	Status() (*Status, error)

	/* Start a new session, return session id */
	NewSession() (string, error)

	/* Current session capabilities */
	Capabilities() (Capabilities, error)
	/* Set the amount of time, in milliseconds, that asynchronous scripts are permitted to run before they are aborted. */
	SetAsyncScriptTimeout(ms uint) error
	/* Set the amount of time, in milliseconds, the driver should wait when searching for elements. */
	SetImplicitWaitTimeout(ms uint) error

	// IME
	/* List all available engines on the machine. */
	AvailableEngines() ([]string, error)
	/* Get the name of the active IME engine. */
	ActiveEngine() (string, error)
	/* Indicates whether IME input is active at the moment. */
	IsEngineActivated() (bool, error)
	/* De-activates the currently-active IME engine. */
	DeactivateEngine() error
	/* Make an engines active */
	ActivateEngine(engine string) error

	/* Quit (end) current session */
	Quit() error

	// Page information and manipulation
	/* Return id of current window handle. */
	CurrentWindowHandle() (string, error)
	/* Return ids of current open windows. */
	WindowHandles() ([]string, error)
	/* Current url. */
	CurrentURL() (string, error)
	/* Page title. */
	Title() (string, error)
	/* Get page source. */
	PageSource() (string, error)
	/* Close current window. */
	Close() error
	/* Switch to frame, frame parameter can be name or id. */
	SwitchFrame(frame string) error
	/* Swtich to window. */
	SwitchWindow(name string) error
	/* Close window. */
	CloseWindow(name string) error

	// Navigation
	/* Open url. */
	Get(url string) error
	/* Move forward in history. */
	Forward() error
	/* Move backward in history. */
	Back() error
	/* Refresh page. */
	Refresh() error

	// Finding element(s)
	/* Find, return one element. */
	FindElement(by, value string) (WebElement, error)
	/* Find, return list of elements. */
	FindElements(by, value string) ([]WebElement, error)
	/* Current active element. */
	ActiveElement() (WebElement, error)

	// Cookies
	/* Get all cookies */
	GetCookies() ([]Cookie, error)
	/* Add a cookies */
	AddCookie(cookie *Cookie) error
	/* Delete all cookies */
	DeleteAllCookies() error
	/* Delete a cookie */
	DeleteCookie(name string) error

	// Mouse
	/* Click mouse button, button should be on of RightButton, MiddleButton or
	LeftButton.
	*/
	Click(button int) error
	/* Dobule click */
	DoubleClick() error
	/* Mouse button down */
	ButtonDown() error
	/* Mouse button up */
	ButtonUp() error

	// Misc
	/* Send modifier key to active element.
	modifier can be one of ShiftKey, ControlKey, AltKey, MetaKey.
	*/
	SendModifier(modifier string, isDown bool) error
	Screenshot() ([]byte, error)

	// Alerts
	/* Dismiss current alert. */
	DismissAlert() error
	/* Accept current alert. */
	AcceptAlert() error
	/* Current alert text. */
	AlertText() (string, error)
	/* Set current alert text. */
	SetAlertText(text string) error

	// Scripts
	/* Execute a script. */
	ExecuteScript(script string, args []interface{}) (interface{}, error)
	/* Execute a script async. */
	ExecuteScriptAsync(script string, args []interface{}) (interface{}, error)

	// Get a WebDriverT of this element that has methods that call t.Fatalf upon encountering
	// errors instead of using multiple returns to indicate errors.
	T(t *testing.T) WebDriverT
}

type WebElement interface {
	// Manipulation

	/* Click on element */
	Click() error
	/* Send keys (type) into element */
	SendKeys(keys string) error
	/* Submit */
	Submit() error
	/* Clear */
	Clear() error
	/* Move mouse to relative coordinates */
	MoveTo(xOffset, yOffset int) error

	// Finding

	/* Find children, return one element. */
	FindElement(by, value string) (WebElement, error)
	/* Find children, return list of elements. */
	FindElements(by, value string) ([]WebElement, error)

	// Porperties

	/* Element name */
	TagName() (string, error)
	/* Text of element */
	Text() (string, error)
	/* Check if element is selected. */
	IsSelected() (bool, error)
	/* Check if element is enabled. */
	IsEnabled() (bool, error)
	/* Check if element is displayed. */
	IsDisplayed() (bool, error)
	/* Get element attribute. */
	GetAttribute(name string) (string, error)
	/* Element location. */
	Location() (*Point, error)
	/* Element location once it has been scrolled into view. */
	LocationInView() (*Point, error)
	/* Element size */
	Size() (*Size, error)
	/* Get element CSS property value. */
	CSSProperty(name string) (string, error)

	// Get a WebElementT of this element that has methods that call t.Fatalf upon encountering
	// errors instead of using multiple returns to indicate errors.
	T(t *testing.T) WebElementT
}
<|endoftext|>


<commit_before>package check

import (
	"bitbucket.org/kardianos/osext"
	"bytes"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"github.com/inconshreveable/go-update"
	"io/ioutil"
	"net/http"
	"runtime"
)

type Initiative string

const (
	INITIATIVE_NEVER  = "never"
	INITIATIVE_AUTO   = "auto"
	INITIATIVE_MANUAL = "manual"
)

var NoUpdateAvailable error = fmt.Errorf("No update available")

type Params struct {
	// protocol version
	Version int `json:"version"`
	// identifier of the application to update
	AppId string `json:"app_id"`
	// version of the application updating itself
	AppVersion string `json:"app_version"`
	// operating system of target platform
	OS string `json:"-"`
	// hardware architecture of target platform
	Arch string `json:"-"`
	// application-level user identifier
	UserId string `json:"user_id"`
	// checksum of the binary to replace (used for returning diff patches)
	Checksum string `json:"checksum"`
	// tags for custom update channels
	Tags map[string]string `json:"tags"`
}

type Result struct {
	Initiative Initiative
	Url        string           // url where to download the updated application
	PatchUrl   string           // a URL to a patch to apply
	PatchType  update.PatchType // the patch format (only bsdiff supported at the moment)
	Version    string           // version of the new application
	Checksum   string           // expected checksum of the new application
	Signature  string           // signature for verifying update authenticity
}

// CheckForUpdate makes an HTTP post to a URL with the JSON serialized
// representation of Params. It returns the deserialized result object
// returned by the remote endpoint or an error. If you do not set
// OS/Arch, CheckForUpdate will populate them for you. Similarly, if
// Version is 0, it will be set to 1. Lastly, if Checksum is the empty
// string, it will be automatically be computed for the running program's
// executable file.
func (p *Params) CheckForUpdate(url string) (*Result, error) {
	if p.OS == "" {
		p.OS = runtime.GOOS
	}

	if p.Arch == "" {
		p.Arch = runtime.GOARCH
	}

	if p.Version == 0 {
		p.Version = 1
	}

	// ignore errors auto-populating the checksum
	// if it fails, you just won't be able to patch
	if p.Checksum == "" {
		p.Checksum = defaultChecksum()
	}

	p.Tags["os"] = p.OS
	p.Tags["arch"] = p.Arch

	body, err := json.Marshal(p)
	if err != nil {
		return nil, err
	}

	resp, err := http.Post(url, "application/json", bytes.NewReader(body))
	if err != nil {
		return nil, err
	}

	// no content means no available update
	if resp.StatusCode == 204 {
		return nil, NoUpdateAvailable
	}

	defer resp.Body.Close()
	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	result := new(Result)
	if err := json.Unmarshal(respBytes, result); err != nil {
		return nil, err
	}

	return result, nil
}

func (p *Params) CheckAndApplyUpdate(url string, u *update.Update) (result *Result, err error, errRecover error) {
	// check for an update
	result, err = p.CheckForUpdate(url)
	if err != nil {
		return
	}

	// run the update if one is available and the server says it's auto
	if result.Update != nil && result.Initiative == INITIATIVE_AUTO {
		err, errRecover = result.Update(u)
	}

	return
}

func (r *Result) Update(u *update.Update) (err error, errRecover error) {
	if r.Checksum != "" {
		u.Checksum, err = hex.DecodeString(r.Checksum)
		if err != nil {
			return
		}
	}

	if r.Signature != "" {
		u.Signature, err = hex.DecodeString(r.Signature)
		if err != nil {
			return
		}
	}

	if r.PatchType != "" {
		u.PatchType = r.PatchType
	}

	if r.Url == "" && r.PatchUrl == "" {
		err = fmt.Errorf("Result does not contain an update url or patch update url")
		return
	}

	if r.PatchUrl != "" {
		err, errRecover = u.FromUrl(r.PatchUrl)
		if err == nil {
			// success!
			return
		} else {
			// failed to update from patch URL, try with the whole thing
			if r.Url == "" || errRecover != nil {
				// we can't try updating from a URL with the full contents
				// in these cases, so fail
				return
			} else {
				u.PatchType = update.PATCHTYPE_NONE
			}
		}
	}

	// try updating from a URL with the full contents
	return u.FromUrl(r.Url)
}

func defaultChecksum() string {
	path, err := osext.Executable()
	if err != nil {
		return ""
	}

	checksum, err := update.ChecksumForFile(path)
	if err != nil {
		return ""
	}

	return hex.EncodeToString(checksum)
}
<commit_msg>fix for assignment into nil map<commit_after>package check

import (
	"bitbucket.org/kardianos/osext"
	"bytes"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"github.com/inconshreveable/go-update"
	"io/ioutil"
	"net/http"
	"runtime"
)

type Initiative string

const (
	INITIATIVE_NEVER  = "never"
	INITIATIVE_AUTO   = "auto"
	INITIATIVE_MANUAL = "manual"
)

var NoUpdateAvailable error = fmt.Errorf("No update available")

type Params struct {
	// protocol version
	Version int `json:"version"`
	// identifier of the application to update
	AppId string `json:"app_id"`
	// version of the application updating itself
	AppVersion string `json:"app_version"`
	// operating system of target platform
	OS string `json:"-"`
	// hardware architecture of target platform
	Arch string `json:"-"`
	// application-level user identifier
	UserId string `json:"user_id"`
	// checksum of the binary to replace (used for returning diff patches)
	Checksum string `json:"checksum"`
	// tags for custom update channels
	Tags map[string]string `json:"tags"`
}

type Result struct {
	Initiative Initiative
	Url        string           // url where to download the updated application
	PatchUrl   string           // a URL to a patch to apply
	PatchType  update.PatchType // the patch format (only bsdiff supported at the moment)
	Version    string           // version of the new application
	Checksum   string           // expected checksum of the new application
	Signature  string           // signature for verifying update authenticity
}

// CheckForUpdate makes an HTTP post to a URL with the JSON serialized
// representation of Params. It returns the deserialized result object
// returned by the remote endpoint or an error. If you do not set
// OS/Arch, CheckForUpdate will populate them for you. Similarly, if
// Version is 0, it will be set to 1. Lastly, if Checksum is the empty
// string, it will be automatically be computed for the running program's
// executable file.
func (p *Params) CheckForUpdate(url string) (*Result, error) {
    if p.Tags == nil {
        p.Tags = make(map[string]string)
    }

	if p.OS == "" {
		p.OS = runtime.GOOS
	}

	if p.Arch == "" {
		p.Arch = runtime.GOARCH
	}

	if p.Version == 0 {
		p.Version = 1
	}

	// ignore errors auto-populating the checksum
	// if it fails, you just won't be able to patch
	if p.Checksum == "" {
		p.Checksum = defaultChecksum()
	}

	p.Tags["os"] = p.OS
	p.Tags["arch"] = p.Arch

	body, err := json.Marshal(p)
	if err != nil {
		return nil, err
	}

	resp, err := http.Post(url, "application/json", bytes.NewReader(body))
	if err != nil {
		return nil, err
	}

	// no content means no available update
	if resp.StatusCode == 204 {
		return nil, NoUpdateAvailable
	}

	defer resp.Body.Close()
	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	result := new(Result)
	if err := json.Unmarshal(respBytes, result); err != nil {
		return nil, err
	}

	return result, nil
}

func (p *Params) CheckAndApplyUpdate(url string, u *update.Update) (result *Result, err error, errRecover error) {
	// check for an update
	result, err = p.CheckForUpdate(url)
	if err != nil {
		return
	}

	// run the update if one is available and the server says it's auto
	if result.Update != nil && result.Initiative == INITIATIVE_AUTO {
		err, errRecover = result.Update(u)
	}

	return
}

func (r *Result) Update(u *update.Update) (err error, errRecover error) {
	if r.Checksum != "" {
		u.Checksum, err = hex.DecodeString(r.Checksum)
		if err != nil {
			return
		}
	}

	if r.Signature != "" {
		u.Signature, err = hex.DecodeString(r.Signature)
		if err != nil {
			return
		}
	}

	if r.PatchType != "" {
		u.PatchType = r.PatchType
	}

	if r.Url == "" && r.PatchUrl == "" {
		err = fmt.Errorf("Result does not contain an update url or patch update url")
		return
	}

	if r.PatchUrl != "" {
		err, errRecover = u.FromUrl(r.PatchUrl)
		if err == nil {
			// success!
			return
		} else {
			// failed to update from patch URL, try with the whole thing
			if r.Url == "" || errRecover != nil {
				// we can't try updating from a URL with the full contents
				// in these cases, so fail
				return
			} else {
				u.PatchType = update.PATCHTYPE_NONE
			}
		}
	}

	// try updating from a URL with the full contents
	return u.FromUrl(r.Url)
}

func defaultChecksum() string {
	path, err := osext.Executable()
	if err != nil {
		return ""
	}

	checksum, err := update.ChecksumForFile(path)
	if err != nil {
		return ""
	}

	return hex.EncodeToString(checksum)
}
<|endoftext|>


<commit_before>// package merkledag implements the ipfs Merkle DAG datastructures.
package merkledag

import (
	"fmt"
	"strings"
	"sync"

	blocks "github.com/ipfs/go-ipfs/blocks"
	key "github.com/ipfs/go-ipfs/blocks/key"
	bserv "github.com/ipfs/go-ipfs/blockservice"
	logging "gx/ipfs/QmYtB7Qge8cJpXc4irsEp8zRqfnZMBeB7aTrMEkPk67DRv/go-log"
	"gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
)

var log = logging.Logger("merkledag")
var ErrNotFound = fmt.Errorf("merkledag: not found")

// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
	Add(*Node) (key.Key, error)
	Get(context.Context, key.Key) (*Node, error)
	Remove(*Node) error

	// GetDAG returns, in order, all the single leve child
	// nodes of the passed in node.
	GetMany(context.Context, []key.Key) <-chan *NodeOption

	Batch() *Batch
}

func NewDAGService(bs *bserv.BlockService) DAGService {
	return &dagService{bs}
}

// dagService is an IPFS Merkle DAG service.
// - the root is virtual (like a forest)
// - stores nodes' data in a BlockService
// TODO: should cache Nodes that are in memory, and be
//       able to free some of them when vm pressure is high
type dagService struct {
	Blocks *bserv.BlockService
}

// Add adds a node to the dagService, storing the block in the BlockService
func (n *dagService) Add(nd *Node) (key.Key, error) {
	if n == nil { // FIXME remove this assertion. protect with constructor invariant
		return "", fmt.Errorf("dagService is nil")
	}

	d, err := nd.EncodeProtobuf(false)
	if err != nil {
		return "", err
	}

	mh, err := nd.Multihash()
	if err != nil {
		return "", err
	}

	b, _ := blocks.NewBlockWithHash(d, mh)

	return n.Blocks.AddBlock(b)
}

func (n *dagService) Batch() *Batch {
	return &Batch{ds: n, MaxSize: 8 * 1024 * 1024}
}

// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(ctx context.Context, k key.Key) (*Node, error) {
	if k == "" {
		return nil, ErrNotFound
	}
	if n == nil {
		return nil, fmt.Errorf("dagService is nil")
	}
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	b, err := n.Blocks.GetBlock(ctx, k)
	if err != nil {
		if err == bserv.ErrNotFound {
			return nil, ErrNotFound
		}
		return nil, fmt.Errorf("Failed to get block for %s: %v", k.B58String(), err)
	}

	res, err := DecodeProtobuf(b.Data())
	if err != nil {
		if strings.Contains(err.Error(), "Unmarshal failed") {
			return nil, fmt.Errorf("%s was not a valid merkledag node", k)
		}
		return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err)
	}
	return res, nil
}

func (n *dagService) Remove(nd *Node) error {
	k, err := nd.Key()
	if err != nil {
		return err
	}
	return n.Blocks.DeleteBlock(k)
}

// FetchGraph fetches all nodes that are children of the given node
func FetchGraph(ctx context.Context, root *Node, serv DAGService) error {
	return EnumerateChildrenAsync(ctx, serv, root, key.NewKeySet())
}

// FindLinks searches this nodes links for the given key,
// returns the indexes of any links pointing to it
func FindLinks(links []key.Key, k key.Key, start int) []int {
	var out []int
	for i, lnk_k := range links[start:] {
		if k == lnk_k {
			out = append(out, i+start)
		}
	}
	return out
}

type NodeOption struct {
	Node *Node
	Err  error
}

func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) <-chan *NodeOption {
	out := make(chan *NodeOption, len(keys))
	blocks := ds.Blocks.GetBlocks(ctx, keys)
	var count int

	go func() {
		defer close(out)
		for {
			select {
			case b, ok := <-blocks:
				if !ok {
					if count != len(keys) {
						out <- &NodeOption{Err: fmt.Errorf("failed to fetch all nodes")}
					}
					return
				}
				nd, err := DecodeProtobuf(b.Data())
				if err != nil {
					out <- &NodeOption{Err: err}
					return
				}

				// buffered, no need to select
				out <- &NodeOption{Node: nd}
				count++

			case <-ctx.Done():
				out <- &NodeOption{Err: ctx.Err()}
				return
			}
		}
	}()
	return out
}

// GetDAG will fill out all of the links of the given Node.
// It returns a channel of nodes, which the caller can receive
// all the child nodes of 'root' on, in proper order.
func GetDAG(ctx context.Context, ds DAGService, root *Node) []NodeGetter {
	var keys []key.Key
	for _, lnk := range root.Links {
		keys = append(keys, key.Key(lnk.Hash))
	}

	return GetNodes(ctx, ds, keys)
}

// GetNodes returns an array of 'NodeGetter' promises, with each corresponding
// to the key with the same index as the passed in keys
func GetNodes(ctx context.Context, ds DAGService, keys []key.Key) []NodeGetter {

	// Early out if no work to do
	if len(keys) == 0 {
		return nil
	}

	promises := make([]NodeGetter, len(keys))
	for i := range keys {
		promises[i] = newNodePromise(ctx)
	}

	dedupedKeys := dedupeKeys(keys)
	go func() {
		ctx, cancel := context.WithCancel(ctx)
		defer cancel()

		nodechan := ds.GetMany(ctx, dedupedKeys)

		for count := 0; count < len(keys); {
			select {
			case opt, ok := <-nodechan:
				if !ok {
					for _, p := range promises {
						p.Fail(ErrNotFound)
					}
					return
				}

				if opt.Err != nil {
					for _, p := range promises {
						p.Fail(opt.Err)
					}
					return
				}

				nd := opt.Node

				k, err := nd.Key()
				if err != nil {
					log.Error("Failed to get node key: ", err)
					continue
				}

				is := FindLinks(keys, k, 0)
				for _, i := range is {
					count++
					promises[i].Send(nd)
				}
			case <-ctx.Done():
				return
			}
		}
	}()
	return promises
}

// Remove duplicates from a list of keys
func dedupeKeys(ks []key.Key) []key.Key {
	kmap := make(map[key.Key]struct{})
	var out []key.Key
	for _, k := range ks {
		if _, ok := kmap[k]; !ok {
			kmap[k] = struct{}{}
			out = append(out, k)
		}
	}
	return out
}

func newNodePromise(ctx context.Context) NodeGetter {
	return &nodePromise{
		recv: make(chan *Node, 1),
		ctx:  ctx,
		err:  make(chan error, 1),
	}
}

type nodePromise struct {
	cache *Node
	clk   sync.Mutex
	recv  chan *Node
	ctx   context.Context
	err   chan error
}

// NodeGetter provides a promise like interface for a dag Node
// the first call to Get will block until the Node is received
// from its internal channels, subsequent calls will return the
// cached node.
type NodeGetter interface {
	Get(context.Context) (*Node, error)
	Fail(err error)
	Send(*Node)
}

func (np *nodePromise) Fail(err error) {
	np.clk.Lock()
	v := np.cache
	np.clk.Unlock()

	// if promise has a value, don't fail it
	if v != nil {
		return
	}

	np.err <- err
}

func (np *nodePromise) Send(nd *Node) {
	var already bool
	np.clk.Lock()
	if np.cache != nil {
		already = true
	}
	np.cache = nd
	np.clk.Unlock()

	if already {
		panic("sending twice to the same promise is an error!")
	}

	np.recv <- nd
}

func (np *nodePromise) Get(ctx context.Context) (*Node, error) {
	np.clk.Lock()
	c := np.cache
	np.clk.Unlock()
	if c != nil {
		return c, nil
	}

	select {
	case nd := <-np.recv:
		return nd, nil
	case <-np.ctx.Done():
		return nil, np.ctx.Err()
	case <-ctx.Done():
		return nil, ctx.Err()
	case err := <-np.err:
		return nil, err
	}
}

type Batch struct {
	ds *dagService

	blocks  []blocks.Block
	size    int
	MaxSize int
}

func (t *Batch) Add(nd *Node) (key.Key, error) {
	d, err := nd.EncodeProtobuf(false)
	if err != nil {
		return "", err
	}

	mh, err := nd.Multihash()
	if err != nil {
		return "", err
	}

	b, _ := blocks.NewBlockWithHash(d, mh)

	k := key.Key(mh)

	t.blocks = append(t.blocks, b)
	t.size += len(b.Data())
	if t.size > t.MaxSize {
		return k, t.Commit()
	}
	return k, nil
}

func (t *Batch) Commit() error {
	_, err := t.ds.Blocks.AddBlocks(t.blocks)
	t.blocks = nil
	t.size = 0
	return err
}

// EnumerateChildren will walk the dag below the given root node and add all
// unseen children to the passed in set.
// TODO: parallelize to avoid disk latency perf hits?
func EnumerateChildren(ctx context.Context, ds DAGService, root *Node, set key.KeySet) error {
	for _, lnk := range root.Links {
		k := key.Key(lnk.Hash)
		if !set.Has(k) {
			set.Add(k)
			child, err := ds.Get(ctx, k)
			if err != nil {
				return err
			}
			err = EnumerateChildren(ctx, ds, child, set)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func EnumerateChildrenAsync(ctx context.Context, ds DAGService, root *Node, set key.KeySet) error {
	toprocess := make(chan []key.Key, 8)
	nodes := make(chan *NodeOption, 8)

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	defer close(toprocess)

	go fetchNodes(ctx, ds, toprocess, nodes)

	nodes <- &NodeOption{Node: root}
	live := 1

	for {
		select {
		case opt, ok := <-nodes:
			if !ok {
				return nil
			}

			if opt.Err != nil {
				return opt.Err
			}

			nd := opt.Node

			// a node has been fetched
			live--

			var keys []key.Key
			for _, lnk := range nd.Links {
				k := key.Key(lnk.Hash)
				if !set.Has(k) {
					set.Add(k)
					live++
					keys = append(keys, k)
				}
			}

			if live == 0 {
				return nil
			}

			if len(keys) > 0 {
				select {
				case toprocess <- keys:
				case <-ctx.Done():
					return ctx.Err()
				}
			}
		case <-ctx.Done():
			return ctx.Err()
		}
	}
}

func fetchNodes(ctx context.Context, ds DAGService, in <-chan []key.Key, out chan<- *NodeOption) {
	var wg sync.WaitGroup
	defer func() {
		// wait for all 'get' calls to complete so we don't accidentally send
		// on a closed channel
		wg.Wait()
		close(out)
	}()

	get := func(ks []key.Key) {
		defer wg.Done()
		nodes := ds.GetMany(ctx, ks)
		for opt := range nodes {
			select {
			case out <- opt:
			case <-ctx.Done():
				return
			}
		}
	}

	for ks := range in {
		wg.Add(1)
		go get(ks)
	}
}
<commit_msg>add a little bit more verbosity to the error<commit_after>// package merkledag implements the ipfs Merkle DAG datastructures.
package merkledag

import (
	"fmt"
	"strings"
	"sync"

	blocks "github.com/ipfs/go-ipfs/blocks"
	key "github.com/ipfs/go-ipfs/blocks/key"
	bserv "github.com/ipfs/go-ipfs/blockservice"
	logging "gx/ipfs/QmYtB7Qge8cJpXc4irsEp8zRqfnZMBeB7aTrMEkPk67DRv/go-log"
	"gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
)

var log = logging.Logger("merkledag")
var ErrNotFound = fmt.Errorf("merkledag: not found")

// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
	Add(*Node) (key.Key, error)
	Get(context.Context, key.Key) (*Node, error)
	Remove(*Node) error

	// GetDAG returns, in order, all the single leve child
	// nodes of the passed in node.
	GetMany(context.Context, []key.Key) <-chan *NodeOption

	Batch() *Batch
}

func NewDAGService(bs *bserv.BlockService) DAGService {
	return &dagService{bs}
}

// dagService is an IPFS Merkle DAG service.
// - the root is virtual (like a forest)
// - stores nodes' data in a BlockService
// TODO: should cache Nodes that are in memory, and be
//       able to free some of them when vm pressure is high
type dagService struct {
	Blocks *bserv.BlockService
}

// Add adds a node to the dagService, storing the block in the BlockService
func (n *dagService) Add(nd *Node) (key.Key, error) {
	if n == nil { // FIXME remove this assertion. protect with constructor invariant
		return "", fmt.Errorf("dagService is nil")
	}

	d, err := nd.EncodeProtobuf(false)
	if err != nil {
		return "", err
	}

	mh, err := nd.Multihash()
	if err != nil {
		return "", err
	}

	b, _ := blocks.NewBlockWithHash(d, mh)

	return n.Blocks.AddBlock(b)
}

func (n *dagService) Batch() *Batch {
	return &Batch{ds: n, MaxSize: 8 * 1024 * 1024}
}

// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(ctx context.Context, k key.Key) (*Node, error) {
	if k == "" {
		return nil, ErrNotFound
	}
	if n == nil {
		return nil, fmt.Errorf("dagService is nil")
	}
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	b, err := n.Blocks.GetBlock(ctx, k)
	if err != nil {
		if err == bserv.ErrNotFound {
			return nil, ErrNotFound
		}
		return nil, fmt.Errorf("Failed to get block for %s: %v", k.B58String(), err)
	}

	res, err := DecodeProtobuf(b.Data())
	if err != nil {
		if strings.Contains(err.Error(), "Unmarshal failed") {
			return nil, fmt.Errorf("The block referred to by '%s' was not a valid merkledag node", k)
		}
		return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err)
	}
	return res, nil
}

func (n *dagService) Remove(nd *Node) error {
	k, err := nd.Key()
	if err != nil {
		return err
	}
	return n.Blocks.DeleteBlock(k)
}

// FetchGraph fetches all nodes that are children of the given node
func FetchGraph(ctx context.Context, root *Node, serv DAGService) error {
	return EnumerateChildrenAsync(ctx, serv, root, key.NewKeySet())
}

// FindLinks searches this nodes links for the given key,
// returns the indexes of any links pointing to it
func FindLinks(links []key.Key, k key.Key, start int) []int {
	var out []int
	for i, lnk_k := range links[start:] {
		if k == lnk_k {
			out = append(out, i+start)
		}
	}
	return out
}

type NodeOption struct {
	Node *Node
	Err  error
}

func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) <-chan *NodeOption {
	out := make(chan *NodeOption, len(keys))
	blocks := ds.Blocks.GetBlocks(ctx, keys)
	var count int

	go func() {
		defer close(out)
		for {
			select {
			case b, ok := <-blocks:
				if !ok {
					if count != len(keys) {
						out <- &NodeOption{Err: fmt.Errorf("failed to fetch all nodes")}
					}
					return
				}
				nd, err := DecodeProtobuf(b.Data())
				if err != nil {
					out <- &NodeOption{Err: err}
					return
				}

				// buffered, no need to select
				out <- &NodeOption{Node: nd}
				count++

			case <-ctx.Done():
				out <- &NodeOption{Err: ctx.Err()}
				return
			}
		}
	}()
	return out
}

// GetDAG will fill out all of the links of the given Node.
// It returns a channel of nodes, which the caller can receive
// all the child nodes of 'root' on, in proper order.
func GetDAG(ctx context.Context, ds DAGService, root *Node) []NodeGetter {
	var keys []key.Key
	for _, lnk := range root.Links {
		keys = append(keys, key.Key(lnk.Hash))
	}

	return GetNodes(ctx, ds, keys)
}

// GetNodes returns an array of 'NodeGetter' promises, with each corresponding
// to the key with the same index as the passed in keys
func GetNodes(ctx context.Context, ds DAGService, keys []key.Key) []NodeGetter {

	// Early out if no work to do
	if len(keys) == 0 {
		return nil
	}

	promises := make([]NodeGetter, len(keys))
	for i := range keys {
		promises[i] = newNodePromise(ctx)
	}

	dedupedKeys := dedupeKeys(keys)
	go func() {
		ctx, cancel := context.WithCancel(ctx)
		defer cancel()

		nodechan := ds.GetMany(ctx, dedupedKeys)

		for count := 0; count < len(keys); {
			select {
			case opt, ok := <-nodechan:
				if !ok {
					for _, p := range promises {
						p.Fail(ErrNotFound)
					}
					return
				}

				if opt.Err != nil {
					for _, p := range promises {
						p.Fail(opt.Err)
					}
					return
				}

				nd := opt.Node

				k, err := nd.Key()
				if err != nil {
					log.Error("Failed to get node key: ", err)
					continue
				}

				is := FindLinks(keys, k, 0)
				for _, i := range is {
					count++
					promises[i].Send(nd)
				}
			case <-ctx.Done():
				return
			}
		}
	}()
	return promises
}

// Remove duplicates from a list of keys
func dedupeKeys(ks []key.Key) []key.Key {
	kmap := make(map[key.Key]struct{})
	var out []key.Key
	for _, k := range ks {
		if _, ok := kmap[k]; !ok {
			kmap[k] = struct{}{}
			out = append(out, k)
		}
	}
	return out
}

func newNodePromise(ctx context.Context) NodeGetter {
	return &nodePromise{
		recv: make(chan *Node, 1),
		ctx:  ctx,
		err:  make(chan error, 1),
	}
}

type nodePromise struct {
	cache *Node
	clk   sync.Mutex
	recv  chan *Node
	ctx   context.Context
	err   chan error
}

// NodeGetter provides a promise like interface for a dag Node
// the first call to Get will block until the Node is received
// from its internal channels, subsequent calls will return the
// cached node.
type NodeGetter interface {
	Get(context.Context) (*Node, error)
	Fail(err error)
	Send(*Node)
}

func (np *nodePromise) Fail(err error) {
	np.clk.Lock()
	v := np.cache
	np.clk.Unlock()

	// if promise has a value, don't fail it
	if v != nil {
		return
	}

	np.err <- err
}

func (np *nodePromise) Send(nd *Node) {
	var already bool
	np.clk.Lock()
	if np.cache != nil {
		already = true
	}
	np.cache = nd
	np.clk.Unlock()

	if already {
		panic("sending twice to the same promise is an error!")
	}

	np.recv <- nd
}

func (np *nodePromise) Get(ctx context.Context) (*Node, error) {
	np.clk.Lock()
	c := np.cache
	np.clk.Unlock()
	if c != nil {
		return c, nil
	}

	select {
	case nd := <-np.recv:
		return nd, nil
	case <-np.ctx.Done():
		return nil, np.ctx.Err()
	case <-ctx.Done():
		return nil, ctx.Err()
	case err := <-np.err:
		return nil, err
	}
}

type Batch struct {
	ds *dagService

	blocks  []blocks.Block
	size    int
	MaxSize int
}

func (t *Batch) Add(nd *Node) (key.Key, error) {
	d, err := nd.EncodeProtobuf(false)
	if err != nil {
		return "", err
	}

	mh, err := nd.Multihash()
	if err != nil {
		return "", err
	}

	b, _ := blocks.NewBlockWithHash(d, mh)

	k := key.Key(mh)

	t.blocks = append(t.blocks, b)
	t.size += len(b.Data())
	if t.size > t.MaxSize {
		return k, t.Commit()
	}
	return k, nil
}

func (t *Batch) Commit() error {
	_, err := t.ds.Blocks.AddBlocks(t.blocks)
	t.blocks = nil
	t.size = 0
	return err
}

// EnumerateChildren will walk the dag below the given root node and add all
// unseen children to the passed in set.
// TODO: parallelize to avoid disk latency perf hits?
func EnumerateChildren(ctx context.Context, ds DAGService, root *Node, set key.KeySet) error {
	for _, lnk := range root.Links {
		k := key.Key(lnk.Hash)
		if !set.Has(k) {
			set.Add(k)
			child, err := ds.Get(ctx, k)
			if err != nil {
				return err
			}
			err = EnumerateChildren(ctx, ds, child, set)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func EnumerateChildrenAsync(ctx context.Context, ds DAGService, root *Node, set key.KeySet) error {
	toprocess := make(chan []key.Key, 8)
	nodes := make(chan *NodeOption, 8)

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	defer close(toprocess)

	go fetchNodes(ctx, ds, toprocess, nodes)

	nodes <- &NodeOption{Node: root}
	live := 1

	for {
		select {
		case opt, ok := <-nodes:
			if !ok {
				return nil
			}

			if opt.Err != nil {
				return opt.Err
			}

			nd := opt.Node

			// a node has been fetched
			live--

			var keys []key.Key
			for _, lnk := range nd.Links {
				k := key.Key(lnk.Hash)
				if !set.Has(k) {
					set.Add(k)
					live++
					keys = append(keys, k)
				}
			}

			if live == 0 {
				return nil
			}

			if len(keys) > 0 {
				select {
				case toprocess <- keys:
				case <-ctx.Done():
					return ctx.Err()
				}
			}
		case <-ctx.Done():
			return ctx.Err()
		}
	}
}

func fetchNodes(ctx context.Context, ds DAGService, in <-chan []key.Key, out chan<- *NodeOption) {
	var wg sync.WaitGroup
	defer func() {
		// wait for all 'get' calls to complete so we don't accidentally send
		// on a closed channel
		wg.Wait()
		close(out)
	}()

	get := func(ks []key.Key) {
		defer wg.Done()
		nodes := ds.GetMany(ctx, ks)
		for opt := range nodes {
			select {
			case out <- opt:
			case <-ctx.Done():
				return
			}
		}
	}

	for ks := range in {
		wg.Add(1)
		go get(ks)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"strings"

	"github.com/jawher/mow.cli"

	docker "github.com/bywan/go-dockercommand"

	dockerclient "github.com/fsouza/go-dockerclient"
)

var allContainers []dockerclient.APIContainers

func run(cmd *cli.Cmd) {
	cmd.Spec = "[--home|--scm-key|--registry]... [--restart|--update]"

	bzkHome := cmd.String(cli.StringOpt{
		Name:   "home",
		Desc:   "Bazooka's work directory",
		EnvVar: "BZK_HOME",
	})
	scmKey := cmd.String(cli.StringOpt{
		Name:   "scm-key",
		Desc:   "Location of the private SSH Key Bazooka will use for SCM Fetch",
		EnvVar: "BZK_SCM_KEYFILE",
	})
	registry := cmd.String(cli.StringOpt{
		Name:   "registry",
		EnvVar: "BZK_REGISTRY",
	})
	dockerSock := cmd.String(cli.StringOpt{
		Name:   "docker-sock",
		Desc:   "Location of the Docker unix socket, usually /var/run/docker.sock",
		EnvVar: "BZK_DOCKERSOCK",
	})

	forceRestart := cmd.Bool(cli.BoolOpt{
		Name: "r restart",
		Desc: "Restart Bazooka if already running",
	})
	forceUpdate := cmd.Bool(cli.BoolOpt{
		Name: "u update",
		Desc: "Update Bazooka to the latest version by pulling new images from the registry",
	})

	cmd.Action = func() {
		config, err := loadConfig()
		if err != nil {
			log.Fatal(fmt.Errorf("Unable to load Bazooka config, reason is: %v\n", err))
		}
		if len(*bzkHome) == 0 {
			if len(config.Home) == 0 {
				*bzkHome = interactiveInput("Bazooka Home Folder")
				config.Home = *bzkHome
			}
		}

		if len(*dockerSock) == 0 {
			if len(config.DockerSock) == 0 {
				*dockerSock = interactiveInput("Bazooka Home Folder")
				config.DockerSock = *dockerSock
			}
		}

		if len(*scmKey) == 0 {
			if len(config.SCMKey) == 0 {
				*scmKey = interactiveInput("Bazooka Default SCM private key")
				config.SCMKey = *scmKey
			}
		}

		err = saveConfig(config)
		if err != nil {
			log.Fatal(fmt.Errorf("Unable to save Bazooka config, reason is: %v\n", err))
		}

		client, err := docker.NewDocker("")
		if err != nil {
			log.Fatal(err)
		}

		allContainers, err = client.Ps(&docker.PsOptions{
			All: true,
		})
		if err != nil {
			log.Fatal(err)
		}

		if *forceUpdate {
			log.Printf("Pulling Bazooka images to check for new versions\n")
			mandatoryImages := []string{"server", "web", "orchestration", "parser"}
			optionalImages := []string{"parser-java", "parser-golang", "scm-git",
				"runner-java", "runner-java:oraclejdk8", "runner-java:oraclejdk7", "runner-java:oraclejdk6", "runner-java:openjdk8", "runner-java:openjdk7", "runner-java:openjdk6",
				"runner-golang", "runner-golang:1.2.2", "runner-golang:1.3", "runner-golang:1.3.1", "runner-golang:1.3.2", "runner-golang:1.3.3", "runner-golang:1.4"}
			for _, image := range mandatoryImages {
				err = client.Pull(&docker.PullOptions{Image: getImageLocation(*registry, fmt.Sprintf("bazooka/%s", image))})
				if err != nil {
					log.Fatal(fmt.Errorf("Unable to pull required image for Bazooka, reason is: %v\n", err))
				}
			}
			for _, image := range optionalImages {
				err = client.Pull(&docker.PullOptions{Image: getImageLocation(*registry, fmt.Sprintf("bazooka/%s", image))})
				if err != nil {
					log.Printf("Unable to pull image for Bazooka, as it is an optional one, let's move on. Reason is: %v\n", err)
				}
			}
		}

		mongoRestarted, err := ensureContainerIsRestarted(client, &docker.RunOptions{
			Name: "bzk_mongodb",
			// Using the official mongo image from dockerhub, this may need a change later
			Image:  "mongo",
			Detach: true,
		}, false)

		serverRestarted, err := ensureContainerIsRestarted(client, &docker.RunOptions{
			Name:   "bzk_server",
			Image:  getImageLocation(*registry, "bazooka/server"),
			Detach: true,
			VolumeBinds: []string{
				fmt.Sprintf("%s:/bazooka", *bzkHome),
				fmt.Sprintf("%s:/var/run/docker.sock", *dockerSock),
			},
			Links: []string{"bzk_mongodb:mongo"},
			Env:   getServerEnv(*bzkHome, *dockerSock, *scmKey),
			PortBindings: map[dockerclient.Port][]dockerclient.PortBinding{
				"3000/tcp": []dockerclient.PortBinding{
					dockerclient.PortBinding{HostPort: "3000"},
				},
			},
		}, mongoRestarted || *forceRestart || *forceUpdate)

		_, err = ensureContainerIsRestarted(client, &docker.RunOptions{
			Name:   "bzk_web",
			Image:  getImageLocation(*registry, "bazooka/web"),
			Detach: true,
			Links:  []string{"bzk_server:server"},
			PortBindings: map[dockerclient.Port][]dockerclient.PortBinding{
				"80/tcp": []dockerclient.PortBinding{
					dockerclient.PortBinding{HostPort: "8000"},
				},
			},
		}, serverRestarted || *forceRestart || *forceUpdate)

		if err != nil {
			log.Fatal(err)
		}
	}

}

func ensureContainerIsRestarted(client *docker.Docker, options *docker.RunOptions, needRestart bool) (bool, error) {
	container, err := getContainer(allContainers, options.Name)
	if err != nil {
		log.Printf("Container %s not found, Starting it\n", options.Name)
		_, err := client.Run(options)
		return true, err
	}
	if needRestart {
		log.Printf("Restarting Container %s\n", options.Name)
		err = client.Rm(&docker.RmOptions{
			Container: []string{container.ID},
			Force:     true,
		})
		if err != nil {
			return false, err
		}
		_, err := client.Run(options)
		return true, err
	}
	if strings.HasPrefix(container.Status, "Up") {
		log.Printf("Container %s already Up & Running, keeping on\n", options.Name)
		return false, nil
	}
	log.Printf("Container %s is not `Up`, starting it\n", options.Name)
	return true, client.Start(&docker.StartOptions{
		ID: container.ID,
	})

}

func getServerEnv(home, dockerSock, scmKey string) map[string]string {
	envMap := map[string]string{
		"BZK_HOME":       home,
		"BZK_DOCKERSOCK": dockerSock,
	}
	if len(scmKey) > 0 {
		envMap["BZK_SCM_KEYFILE"] = scmKey
	}
	return envMap
}

func getContainer(containers []dockerclient.APIContainers, name string) (dockerclient.APIContainers, error) {
	for _, container := range containers {
		if contains(container.Names, name) || contains(container.Names, "/"+name) {
			return container, nil
		}
	}
	return dockerclient.APIContainers{}, fmt.Errorf("Container not found")
}

func getImageLocation(registry, image string) string {
	if len(registry) > 0 {
		return fmt.Sprintf("%s/%s", registry, image)
	}
	return image
}

func contains(slice []string, item string) bool {
	set := make(map[string]struct{}, len(slice))
	for _, s := range slice {
		set[s] = struct{}{}
	}
	_, ok := set[item]
	return ok
}
<commit_msg>Update run.go<commit_after>package main

import (
	"fmt"
	"log"
	"strings"

	"github.com/jawher/mow.cli"

	docker "github.com/bywan/go-dockercommand"

	dockerclient "github.com/fsouza/go-dockerclient"
)

var allContainers []dockerclient.APIContainers

func run(cmd *cli.Cmd) {
	cmd.Spec = "[--home|--scm-key|--registry]... [--restart|--update]"

	bzkHome := cmd.String(cli.StringOpt{
		Name:   "home",
		Desc:   "Bazooka's work directory",
		EnvVar: "BZK_HOME",
	})
	scmKey := cmd.String(cli.StringOpt{
		Name:   "scm-key",
		Desc:   "Location of the private SSH Key Bazooka will use for SCM Fetch",
		EnvVar: "BZK_SCM_KEYFILE",
	})
	registry := cmd.String(cli.StringOpt{
		Name:   "registry",
		EnvVar: "BZK_REGISTRY",
	})
	dockerSock := cmd.String(cli.StringOpt{
		Name:   "docker-sock",
		Desc:   "Location of the Docker unix socket, usually /var/run/docker.sock",
		EnvVar: "BZK_DOCKERSOCK",
	})

	forceRestart := cmd.Bool(cli.BoolOpt{
		Name: "r restart",
		Desc: "Restart Bazooka if already running",
	})
	forceUpdate := cmd.Bool(cli.BoolOpt{
		Name: "u update",
		Desc: "Update Bazooka to the latest version by pulling new images from the registry",
	})

	cmd.Action = func() {
		config, err := loadConfig()
		if err != nil {
			log.Fatal(fmt.Errorf("Unable to load Bazooka config, reason is: %v\n", err))
		}
		if len(*bzkHome) == 0 {
			if len(config.Home) == 0 {
				*bzkHome = interactiveInput("Bazooka Home Folder")
				config.Home = *bzkHome
			}
		}

		if len(*dockerSock) == 0 {
			if len(config.DockerSock) == 0 {
				*dockerSock = interactiveInput("Docker Socket path")
				config.DockerSock = *dockerSock
			}
		}

		if len(*scmKey) == 0 {
			if len(config.SCMKey) == 0 {
				*scmKey = interactiveInput("Bazooka Default SCM private key")
				config.SCMKey = *scmKey
			}
		}

		err = saveConfig(config)
		if err != nil {
			log.Fatal(fmt.Errorf("Unable to save Bazooka config, reason is: %v\n", err))
		}

		client, err := docker.NewDocker("")
		if err != nil {
			log.Fatal(err)
		}

		allContainers, err = client.Ps(&docker.PsOptions{
			All: true,
		})
		if err != nil {
			log.Fatal(err)
		}

		if *forceUpdate {
			log.Printf("Pulling Bazooka images to check for new versions\n")
			mandatoryImages := []string{"server", "web", "orchestration", "parser"}
			optionalImages := []string{"parser-java", "parser-golang", "scm-git",
				"runner-java", "runner-java:oraclejdk8", "runner-java:oraclejdk7", "runner-java:oraclejdk6", "runner-java:openjdk8", "runner-java:openjdk7", "runner-java:openjdk6",
				"runner-golang", "runner-golang:1.2.2", "runner-golang:1.3", "runner-golang:1.3.1", "runner-golang:1.3.2", "runner-golang:1.3.3", "runner-golang:1.4"}
			for _, image := range mandatoryImages {
				err = client.Pull(&docker.PullOptions{Image: getImageLocation(*registry, fmt.Sprintf("bazooka/%s", image))})
				if err != nil {
					log.Fatal(fmt.Errorf("Unable to pull required image for Bazooka, reason is: %v\n", err))
				}
			}
			for _, image := range optionalImages {
				err = client.Pull(&docker.PullOptions{Image: getImageLocation(*registry, fmt.Sprintf("bazooka/%s", image))})
				if err != nil {
					log.Printf("Unable to pull image for Bazooka, as it is an optional one, let's move on. Reason is: %v\n", err)
				}
			}
		}

		mongoRestarted, err := ensureContainerIsRestarted(client, &docker.RunOptions{
			Name: "bzk_mongodb",
			// Using the official mongo image from dockerhub, this may need a change later
			Image:  "mongo",
			Detach: true,
		}, false)

		serverRestarted, err := ensureContainerIsRestarted(client, &docker.RunOptions{
			Name:   "bzk_server",
			Image:  getImageLocation(*registry, "bazooka/server"),
			Detach: true,
			VolumeBinds: []string{
				fmt.Sprintf("%s:/bazooka", *bzkHome),
				fmt.Sprintf("%s:/var/run/docker.sock", *dockerSock),
			},
			Links: []string{"bzk_mongodb:mongo"},
			Env:   getServerEnv(*bzkHome, *dockerSock, *scmKey),
			PortBindings: map[dockerclient.Port][]dockerclient.PortBinding{
				"3000/tcp": []dockerclient.PortBinding{
					dockerclient.PortBinding{HostPort: "3000"},
				},
			},
		}, mongoRestarted || *forceRestart || *forceUpdate)

		_, err = ensureContainerIsRestarted(client, &docker.RunOptions{
			Name:   "bzk_web",
			Image:  getImageLocation(*registry, "bazooka/web"),
			Detach: true,
			Links:  []string{"bzk_server:server"},
			PortBindings: map[dockerclient.Port][]dockerclient.PortBinding{
				"80/tcp": []dockerclient.PortBinding{
					dockerclient.PortBinding{HostPort: "8000"},
				},
			},
		}, serverRestarted || *forceRestart || *forceUpdate)

		if err != nil {
			log.Fatal(err)
		}
	}

}

func ensureContainerIsRestarted(client *docker.Docker, options *docker.RunOptions, needRestart bool) (bool, error) {
	container, err := getContainer(allContainers, options.Name)
	if err != nil {
		log.Printf("Container %s not found, Starting it\n", options.Name)
		_, err := client.Run(options)
		return true, err
	}
	if needRestart {
		log.Printf("Restarting Container %s\n", options.Name)
		err = client.Rm(&docker.RmOptions{
			Container: []string{container.ID},
			Force:     true,
		})
		if err != nil {
			return false, err
		}
		_, err := client.Run(options)
		return true, err
	}
	if strings.HasPrefix(container.Status, "Up") {
		log.Printf("Container %s already Up & Running, keeping on\n", options.Name)
		return false, nil
	}
	log.Printf("Container %s is not `Up`, starting it\n", options.Name)
	return true, client.Start(&docker.StartOptions{
		ID: container.ID,
	})

}

func getServerEnv(home, dockerSock, scmKey string) map[string]string {
	envMap := map[string]string{
		"BZK_HOME":       home,
		"BZK_DOCKERSOCK": dockerSock,
	}
	if len(scmKey) > 0 {
		envMap["BZK_SCM_KEYFILE"] = scmKey
	}
	return envMap
}

func getContainer(containers []dockerclient.APIContainers, name string) (dockerclient.APIContainers, error) {
	for _, container := range containers {
		if contains(container.Names, name) || contains(container.Names, "/"+name) {
			return container, nil
		}
	}
	return dockerclient.APIContainers{}, fmt.Errorf("Container not found")
}

func getImageLocation(registry, image string) string {
	if len(registry) > 0 {
		return fmt.Sprintf("%s/%s", registry, image)
	}
	return image
}

func contains(slice []string, item string) bool {
	set := make(map[string]struct{}, len(slice))
	for _, s := range slice {
		set[s] = struct{}{}
	}
	_, ok := set[item]
	return ok
}
<|endoftext|>