<commit_before>package cmap

import (
	"encoding/json"
	"errors"
	"sync"
)

const defaultShardCount = 32

const bufferSize = 64

// A "thread" safe map of type uint16:Anything.
// To avoid lock bottlenecks this map is dived to several (shardCount) map shards.
type ConcurrentMap struct {
	shards []*ConcurrentMapShared
	count  int
}

// A "thread" safe uint16 to anything map.
type ConcurrentMapShared struct {
	items map[uint16]interface{}
	mux   *sync.RWMutex
}

// Creates a new concurrent map.
func New(shardCount int) (*ConcurrentMap, error) {
	if shardCount < 1 {
		return nil, errors.New("invalid shard count: less than 1")
	}

	shards := make([]*ConcurrentMapShared, shardCount)

	for i := 0; i < shardCount; i++ {
		shards[i] = &ConcurrentMapShared{
			items: make(map[uint16]interface{}),
			mux:   &sync.RWMutex{},
		}
	}

	return &ConcurrentMap{
		shards: shards,
		count:  shardCount,
	}, nil
}

func NewDefault() *ConcurrentMap {
	m, err := New(defaultShardCount)
	if err != nil {
		panic(err)
	}
	return m
}

// Returns shard under given key
func (m *ConcurrentMap) getShard(key uint16) *ConcurrentMapShared {
	return m.shards[m.getShardIndex(key)]
}

func (m *ConcurrentMap) getShardIndex(key uint16) uint16 {
	return key % uint16(m.count)
}

func (m *ConcurrentMap) sortShardsTuples(data map[uint16]interface{}) map[uint16][]Tuple {
	shardsTuples := map[uint16][]Tuple{}

	for key, value := range data {
		shardIndex := m.getShardIndex(key)

		if shardTuples, ok := shardsTuples[shardIndex]; ok {
			shardsTuples[shardIndex] = append(shardTuples, Tuple{
				Key: key,
				Val: value,
			})
		} else {
			shardsTuples[shardIndex] = make([]Tuple, 0, bufferSize)
			shardsTuples[shardIndex] = append(shardsTuples[shardIndex], Tuple{
				Key: key,
				Val: value,
			})
		}
	}

	return shardsTuples
}

func (m *ConcurrentMap) sortShardsKeys(keys []uint16) map[uint16][]uint16 {
	shardsKeys := map[uint16][]uint16{}

	for _, key := range keys {
		shardIndex := m.getShardIndex(key)

		if shardTuples, ok := shardsKeys[shardIndex]; ok {
			shardsKeys[shardIndex] = append(shardTuples, key)
		} else {
			shardsKeys[shardIndex] = make([]uint16, 0, bufferSize)
			shardsKeys[shardIndex] = append(shardsKeys[shardIndex], key)
		}
	}

	return shardsKeys
}

func (m *ConcurrentMap) MSet(data map[uint16]interface{}) {
	shardsTuples := m.sortShardsTuples(data)

	for shardIndex, tuples := range shardsTuples {
		shard := m.shards[shardIndex]
		shard.mux.Lock()
		for _, tuple := range tuples {
			shard.items[tuple.Key] = tuple.Val
		}
		shard.mux.Unlock()
	}
}

func (m *ConcurrentMap) MSetIfAbsent(data map[uint16]interface{}) bool {
	shardsTuples := m.sortShardsTuples(data)
	rollbackKeys := make([]uint16, 0, bufferSize)

	for shardIndex, tuples := range shardsTuples {
		shard := m.shards[shardIndex]
		shard.mux.Lock()
		for _, tuple := range tuples {
			if _, ok := shard.items[tuple.Key]; !ok {
				shard.items[tuple.Key] = tuple.Val
				rollbackKeys = append(rollbackKeys, tuple.Key)
			} else {
				shard.mux.Unlock()

				// Rollback
				m.MRemove(rollbackKeys)

				return false
			}
		}
		shard.mux.Unlock()
	}

	return true
}

// Sets the given value under the specified key.
func (m *ConcurrentMap) Set(key uint16, value interface{}) {
	// Get map shard.
	shard := m.getShard(key)
	shard.mux.Lock()
	shard.items[key] = value
	shard.mux.Unlock()
}

// Callback to return new element to be inserted into the map
// It is called while lock is held, therefore it MUST NOT
// try to access other keys in same map, as it can lead to deadlock since
// Go sync.RWLock is not reentrant
type UpsertCb func(exist bool, valueInMap interface{}, newValue interface{}) interface{}

// Insert or Update - updates existing element or inserts a new one using UpsertCb
func (m *ConcurrentMap) Upsert(key uint16, value interface{}, cb UpsertCb) (res interface{}) {
	shard := m.getShard(key)
	shard.mux.Lock()
	v, ok := shard.items[key]
	res = cb(ok, v, value)
	shard.items[key] = res
	shard.mux.Unlock()
	return res
}

// Sets the given value under the specified key if no value was associated with it.
func (m *ConcurrentMap) SetIfAbsent(key uint16, value interface{}) bool {
	// Get map shard.
	shard := m.getShard(key)
	shard.mux.Lock()
	_, ok := shard.items[key]
	if !ok {
		shard.items[key] = value
	}
	shard.mux.Unlock()
	return !ok
}

// Retrieves an element from map under given key.
func (m *ConcurrentMap) Get(key uint16) (interface{}, bool) {
	// Get shard
	shard := m.getShard(key)
	shard.mux.RLock()
	// Get item from shard.
	val, ok := shard.items[key]
	shard.mux.RUnlock()
	return val, ok
}

// Returns the number of elements within the map.
func (m *ConcurrentMap) Count() int {
	count := 0
	for i := 0; i < m.count; i++ {
		shard := m.shards[i]
		shard.mux.RLock()
		count += len(shard.items)
		shard.mux.RUnlock()
	}
	return count
}

// Looks up an item under specified key
func (m *ConcurrentMap) Has(key uint16) bool {
	// Get shard
	shard := m.getShard(key)
	shard.mux.RLock()
	// See if element is within shard.
	_, ok := shard.items[key]
	shard.mux.RUnlock()
	return ok
}

// Removes an element from the map.
func (m *ConcurrentMap) Remove(key uint16) {
	// Try to get shard.
	shard := m.getShard(key)
	shard.mux.Lock()
	delete(shard.items, key)
	shard.mux.Unlock()
}

func (m *ConcurrentMap) MRemove(keys []uint16) {
	shardsKeys := m.sortShardsKeys(keys)

	for shardIndex, keys := range shardsKeys {
		shard := m.shards[shardIndex]
		shard.mux.Lock()
		for _, key := range keys {
			delete(shard.items, key)
		}
		shard.mux.Unlock()
	}
}

// RemoveCb is a callback executed in a map.RemoveCb() call, while Lock is held
// If returns true, the element will be removed from the map
type RemoveCb func(key uint16, v interface{}, exists bool) bool

// RemoveCb locks the shard containing the key, retrieves its current value and calls the callback with those params
// If callback returns true and element exists, it will remove it from the map
// Returns the value returned by the callback (even if element was not present in the map)
func (m *ConcurrentMap) RemoveCb(key uint16, cb RemoveCb) bool {
	// Try to get shard.
	shard := m.getShard(key)
	shard.mux.Lock()
	v, ok := shard.items[key]
	remove := cb(key, v, ok)
	if remove && ok {
		delete(shard.items, key)
	}
	shard.mux.Unlock()
	return remove
}

// Removes an element from the map and returns it
func (m *ConcurrentMap) Pop(key uint16) (v interface{}, exists bool) {
	// Try to get shard.
	shard := m.getShard(key)
	shard.mux.Lock()
	v, exists = shard.items[key]
	delete(shard.items, key)
	shard.mux.Unlock()
	return v, exists
}

// Checks if map is empty.
func (m *ConcurrentMap) IsEmpty() bool {
	return m.Count() == 0
}

// Used by the Iter & IterBuffered functions to wrap two variables together over a channel,
type Tuple struct {
	Key uint16
	Val interface{}
}

// Returns an iterator which could be used in a for range loop.
//
// Deprecated: using IterBuffered() will get a better performence
func (m *ConcurrentMap) Iter() <-chan Tuple {
	chans := snapshot(m)
	ch := make(chan Tuple)
	go fanIn(chans, ch)
	return ch
}

// Returns a buffered iterator which could be used in a for range loop.
func (m *ConcurrentMap) IterBuffered() <-chan Tuple {
	chans := snapshot(m)
	total := 0
	for _, c := range chans {
		total += cap(c)
	}
	ch := make(chan Tuple, total)
	go fanIn(chans, ch)
	return ch
}

// Returns a array of channels that contains elements in each shard,
// which likely takes a snapshot of `m`.
// It returns once the size of each buffered channel is determined,
// before all the channels are populated using goroutines.
func snapshot(m *ConcurrentMap) (chans []chan Tuple) {
	chans = make([]chan Tuple, m.count)
	wg := sync.WaitGroup{}
	wg.Add(m.count)
	// Foreach shard.
	for index, shard := range m.shards {
		go func(index int, shard *ConcurrentMapShared) {
			// Foreach key, value pair.
			shard.mux.RLock()
			chans[index] = make(chan Tuple, len(shard.items))
			wg.Done()
			for key, val := range shard.items {
				chans[index] <- Tuple{key, val}
			}
			shard.mux.RUnlock()
			close(chans[index])
		}(index, shard)
	}
	wg.Wait()
	return chans
}

// fanIn reads elements from channels `chans` into channel `out`
func fanIn(chans []chan Tuple, out chan Tuple) {
	wg := sync.WaitGroup{}
	wg.Add(len(chans))
	for _, ch := range chans {
		go func(ch chan Tuple) {
			for t := range ch {
				out <- t
			}
			wg.Done()
		}(ch)
	}
	wg.Wait()
	close(out)
}

// Returns all items as map[uint16]interface{}
func (m *ConcurrentMap) Items() map[uint16]interface{} {
	tmp := make(map[uint16]interface{})

	// Insert items to temporary map.
	for item := range m.IterBuffered() {
		tmp[item.Key] = item.Val
	}

	return tmp
}

// Iterator callback,called for every key,value found in
// maps. RLock is held for all calls for a given shard
// therefore callback sess consistent view of a shard,
// but not across the shards
type IterCb func(key uint16, v interface{})

// Callback based iterator, cheapest way to read
// all elements in a map.
func (m *ConcurrentMap) IterCb(fn IterCb) {
	for idx := range m.shards {
		shard := m.shards[idx]
		shard.mux.RLock()
		for key, value := range shard.items {
			fn(key, value)
		}
		shard.mux.RUnlock()
	}
}

// Return all keys as []uint16
func (m *ConcurrentMap) Keys() []uint16 {
	count := m.Count()
	ch := make(chan uint16, count)
	go func() {
		// Foreach shard.
		wg := sync.WaitGroup{}
		wg.Add(m.count)
		for _, shard := range m.shards {
			go func(shard *ConcurrentMapShared) {
				// Foreach key, value pair.
				shard.mux.RLock()
				for key := range shard.items {
					ch <- key
				}
				shard.mux.RUnlock()
				wg.Done()
			}(shard)
		}
		wg.Wait()
		close(ch)
	}()

	// Generate keys
	keys := make([]uint16, 0, count)
	for k := range ch {
		keys = append(keys, k)
	}
	return keys
}

//Reviles ConcurrentMap "private" variables to json marshal.
func (m *ConcurrentMap) MarshalJSON() ([]byte, error) {
	// Create a temporary map, which will hold all item spread across shards.
	tmp := make(map[uint16]interface{})

	// Insert items to temporary map.
	for item := range m.IterBuffered() {
		tmp[item.Key] = item.Val
	}
	return json.Marshal(tmp)
}

// Concurrent map uses Interface{} as its value, therefor JSON Unmarshal
// will probably won't know which to type to unmarshal into, in such case
// we'll end up with a value of type map[uint16]interface{}, In most cases this isn't
// out value type, this is why we've decided to remove this functionality.

// func (m *ConcurrentMap) UnmarshalJSON(b []byte) (err error) {
// 	// Reverse process of Marshal.

// 	tmp := make(map[uint16]interface{})

// 	// Unmarshal into a single map.
// 	if err := json.Unmarshal(b, &tmp); err != nil {
// 		return nil
// 	}

// 	// foreach key,value pair in temporary map insert into our concurrent map.
// 	for key, val := range tmp {
// 		m.Set(key, val)
// 	}
// 	return nil
// }
<commit_msg>CMap: Fix shard sorting methods<commit_after>package cmap

import (
	"encoding/json"
	"errors"
	"sync"
)

const defaultShardCount = 32

const bufferSize = 64

// A "thread" safe map of type uint16:Anything.
// To avoid lock bottlenecks this map is dived to several (shardCount) map shards.
type ConcurrentMap struct {
	shards []*ConcurrentMapShared
	count  int
}

// A "thread" safe uint16 to anything map.
type ConcurrentMapShared struct {
	items map[uint16]interface{}
	mux   *sync.RWMutex
}

// Creates a new concurrent map.
func New(shardCount int) (*ConcurrentMap, error) {
	if shardCount < 1 {
		return nil, errors.New("invalid shard count: less than 1")
	}

	shards := make([]*ConcurrentMapShared, shardCount)

	for i := 0; i < shardCount; i++ {
		shards[i] = &ConcurrentMapShared{
			items: make(map[uint16]interface{}),
			mux:   &sync.RWMutex{},
		}
	}

	return &ConcurrentMap{
		shards: shards,
		count:  shardCount,
	}, nil
}

func NewDefault() *ConcurrentMap {
	m, err := New(defaultShardCount)
	if err != nil {
		panic(err)
	}
	return m
}

// Returns shard under given key
func (m *ConcurrentMap) getShard(key uint16) *ConcurrentMapShared {
	return m.shards[m.getShardIndex(key)]
}

func (m *ConcurrentMap) getShardIndex(key uint16) uint16 {
	return key % uint16(m.count)
}

func (m *ConcurrentMap) sortShardsTuples(data map[uint16]interface{}) map[uint16][]Tuple {
	shardsTuples := map[uint16][]Tuple{}

	for key, value := range data {
		shardIndex := m.getShardIndex(key)

		if _, ok := shardsTuples[shardIndex]; !ok {
			shardsTuples[shardIndex] = make([]Tuple, 0, bufferSize)
		}

		shardsTuples[shardIndex] = append(shardsTuples[shardIndex], Tuple{
			Key: key,
			Val: value,
		})
	}

	return shardsTuples
}

func (m *ConcurrentMap) sortShardsKeys(keys []uint16) map[uint16][]uint16 {
	shardsKeys := map[uint16][]uint16{}

	for _, key := range keys {
		shardIndex := m.getShardIndex(key)

		if _, ok := shardsKeys[shardIndex]; !ok {
			shardsKeys[shardIndex] = make([]uint16, 0, bufferSize)
		}

		shardsKeys[shardIndex] = append(shardsKeys[shardIndex], key)
	}

	return shardsKeys
}

func (m *ConcurrentMap) MSet(data map[uint16]interface{}) {
	shardsTuples := m.sortShardsTuples(data)

	for shardIndex, tuples := range shardsTuples {
		shard := m.shards[shardIndex]
		shard.mux.Lock()
		for _, tuple := range tuples {
			shard.items[tuple.Key] = tuple.Val
		}
		shard.mux.Unlock()
	}
}

func (m *ConcurrentMap) MSetIfAbsent(data map[uint16]interface{}) bool {
	shardsTuples := m.sortShardsTuples(data)
	rollbackKeys := make([]uint16, 0, bufferSize)

	for shardIndex, tuples := range shardsTuples {
		shard := m.shards[shardIndex]
		shard.mux.Lock()
		for _, tuple := range tuples {
			if _, ok := shard.items[tuple.Key]; !ok {
				shard.items[tuple.Key] = tuple.Val
				rollbackKeys = append(rollbackKeys, tuple.Key)
			} else {
				shard.mux.Unlock()

				// Rollback
				m.MRemove(rollbackKeys)

				return false
			}
		}
		shard.mux.Unlock()
	}

	return true
}

// Sets the given value under the specified key.
func (m *ConcurrentMap) Set(key uint16, value interface{}) {
	// Get map shard.
	shard := m.getShard(key)
	shard.mux.Lock()
	shard.items[key] = value
	shard.mux.Unlock()
}

// Callback to return new element to be inserted into the map
// It is called while lock is held, therefore it MUST NOT
// try to access other keys in same map, as it can lead to deadlock since
// Go sync.RWLock is not reentrant
type UpsertCb func(exist bool, valueInMap interface{}, newValue interface{}) interface{}

// Insert or Update - updates existing element or inserts a new one using UpsertCb
func (m *ConcurrentMap) Upsert(key uint16, value interface{}, cb UpsertCb) (res interface{}) {
	shard := m.getShard(key)
	shard.mux.Lock()
	v, ok := shard.items[key]
	res = cb(ok, v, value)
	shard.items[key] = res
	shard.mux.Unlock()
	return res
}

// Sets the given value under the specified key if no value was associated with it.
func (m *ConcurrentMap) SetIfAbsent(key uint16, value interface{}) bool {
	// Get map shard.
	shard := m.getShard(key)
	shard.mux.Lock()
	_, ok := shard.items[key]
	if !ok {
		shard.items[key] = value
	}
	shard.mux.Unlock()
	return !ok
}

// Retrieves an element from map under given key.
func (m *ConcurrentMap) Get(key uint16) (interface{}, bool) {
	// Get shard
	shard := m.getShard(key)
	shard.mux.RLock()
	// Get item from shard.
	val, ok := shard.items[key]
	shard.mux.RUnlock()
	return val, ok
}

// Returns the number of elements within the map.
func (m *ConcurrentMap) Count() int {
	count := 0
	for i := 0; i < m.count; i++ {
		shard := m.shards[i]
		shard.mux.RLock()
		count += len(shard.items)
		shard.mux.RUnlock()
	}
	return count
}

// Looks up an item under specified key
func (m *ConcurrentMap) Has(key uint16) bool {
	// Get shard
	shard := m.getShard(key)
	shard.mux.RLock()
	// See if element is within shard.
	_, ok := shard.items[key]
	shard.mux.RUnlock()
	return ok
}

// Removes an element from the map.
func (m *ConcurrentMap) Remove(key uint16) {
	// Try to get shard.
	shard := m.getShard(key)
	shard.mux.Lock()
	delete(shard.items, key)
	shard.mux.Unlock()
}

func (m *ConcurrentMap) MRemove(keys []uint16) {
	shardsKeys := m.sortShardsKeys(keys)

	for shardIndex, keys := range shardsKeys {
		shard := m.shards[shardIndex]
		shard.mux.Lock()
		for _, key := range keys {
			delete(shard.items, key)
		}
		shard.mux.Unlock()
	}
}

// RemoveCb is a callback executed in a map.RemoveCb() call, while Lock is held
// If returns true, the element will be removed from the map
type RemoveCb func(key uint16, v interface{}, exists bool) bool

// RemoveCb locks the shard containing the key, retrieves its current value and calls the callback with those params
// If callback returns true and element exists, it will remove it from the map
// Returns the value returned by the callback (even if element was not present in the map)
func (m *ConcurrentMap) RemoveCb(key uint16, cb RemoveCb) bool {
	// Try to get shard.
	shard := m.getShard(key)
	shard.mux.Lock()
	v, ok := shard.items[key]
	remove := cb(key, v, ok)
	if remove && ok {
		delete(shard.items, key)
	}
	shard.mux.Unlock()
	return remove
}

// Removes an element from the map and returns it
func (m *ConcurrentMap) Pop(key uint16) (v interface{}, exists bool) {
	// Try to get shard.
	shard := m.getShard(key)
	shard.mux.Lock()
	v, exists = shard.items[key]
	delete(shard.items, key)
	shard.mux.Unlock()
	return v, exists
}

// Checks if map is empty.
func (m *ConcurrentMap) IsEmpty() bool {
	return m.Count() == 0
}

// Used by the Iter & IterBuffered functions to wrap two variables together over a channel,
type Tuple struct {
	Key uint16
	Val interface{}
}

// Returns an iterator which could be used in a for range loop.
//
// Deprecated: using IterBuffered() will get a better performence
func (m *ConcurrentMap) Iter() <-chan Tuple {
	chans := snapshot(m)
	ch := make(chan Tuple)
	go fanIn(chans, ch)
	return ch
}

// Returns a buffered iterator which could be used in a for range loop.
func (m *ConcurrentMap) IterBuffered() <-chan Tuple {
	chans := snapshot(m)
	total := 0
	for _, c := range chans {
		total += cap(c)
	}
	ch := make(chan Tuple, total)
	go fanIn(chans, ch)
	return ch
}

// Returns a array of channels that contains elements in each shard,
// which likely takes a snapshot of `m`.
// It returns once the size of each buffered channel is determined,
// before all the channels are populated using goroutines.
func snapshot(m *ConcurrentMap) (chans []chan Tuple) {
	chans = make([]chan Tuple, m.count)
	wg := sync.WaitGroup{}
	wg.Add(m.count)
	// Foreach shard.
	for index, shard := range m.shards {
		go func(index int, shard *ConcurrentMapShared) {
			// Foreach key, value pair.
			shard.mux.RLock()
			chans[index] = make(chan Tuple, len(shard.items))
			wg.Done()
			for key, val := range shard.items {
				chans[index] <- Tuple{key, val}
			}
			shard.mux.RUnlock()
			close(chans[index])
		}(index, shard)
	}
	wg.Wait()
	return chans
}

// fanIn reads elements from channels `chans` into channel `out`
func fanIn(chans []chan Tuple, out chan Tuple) {
	wg := sync.WaitGroup{}
	wg.Add(len(chans))
	for _, ch := range chans {
		go func(ch chan Tuple) {
			for t := range ch {
				out <- t
			}
			wg.Done()
		}(ch)
	}
	wg.Wait()
	close(out)
}

// Returns all items as map[uint16]interface{}
func (m *ConcurrentMap) Items() map[uint16]interface{} {
	tmp := make(map[uint16]interface{})

	// Insert items to temporary map.
	for item := range m.IterBuffered() {
		tmp[item.Key] = item.Val
	}

	return tmp
}

// Iterator callback,called for every key,value found in
// maps. RLock is held for all calls for a given shard
// therefore callback sess consistent view of a shard,
// but not across the shards
type IterCb func(key uint16, v interface{})

// Callback based iterator, cheapest way to read
// all elements in a map.
func (m *ConcurrentMap) IterCb(fn IterCb) {
	for idx := range m.shards {
		shard := m.shards[idx]
		shard.mux.RLock()
		for key, value := range shard.items {
			fn(key, value)
		}
		shard.mux.RUnlock()
	}
}

// Return all keys as []uint16
func (m *ConcurrentMap) Keys() []uint16 {
	count := m.Count()
	ch := make(chan uint16, count)
	go func() {
		// Foreach shard.
		wg := sync.WaitGroup{}
		wg.Add(m.count)
		for _, shard := range m.shards {
			go func(shard *ConcurrentMapShared) {
				// Foreach key, value pair.
				shard.mux.RLock()
				for key := range shard.items {
					ch <- key
				}
				shard.mux.RUnlock()
				wg.Done()
			}(shard)
		}
		wg.Wait()
		close(ch)
	}()

	// Generate keys
	keys := make([]uint16, 0, count)
	for k := range ch {
		keys = append(keys, k)
	}
	return keys
}

//Reviles ConcurrentMap "private" variables to json marshal.
func (m *ConcurrentMap) MarshalJSON() ([]byte, error) {
	// Create a temporary map, which will hold all item spread across shards.
	tmp := make(map[uint16]interface{})

	// Insert items to temporary map.
	for item := range m.IterBuffered() {
		tmp[item.Key] = item.Val
	}
	return json.Marshal(tmp)
}

// Concurrent map uses Interface{} as its value, therefor JSON Unmarshal
// will probably won't know which to type to unmarshal into, in such case
// we'll end up with a value of type map[uint16]interface{}, In most cases this isn't
// out value type, this is why we've decided to remove this functionality.

// func (m *ConcurrentMap) UnmarshalJSON(b []byte) (err error) {
// 	// Reverse process of Marshal.

// 	tmp := make(map[uint16]interface{})

// 	// Unmarshal into a single map.
// 	if err := json.Unmarshal(b, &tmp); err != nil {
// 		return nil
// 	}

// 	// foreach key,value pair in temporary map insert into our concurrent map.
// 	for key, val := range tmp {
// 		m.Set(key, val)
// 	}
// 	return nil
// }
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"os"
)

// LogFail is the failure log formatter
// prints text to stderr and exits with status 1
func LogFail(text string) {
	fmt.Fprintln(os.Stderr, fmt.Sprintf("FAILED: %s", text))
	os.Exit(1)
}

// LogInfo1 is the info1 header formatter
func LogInfo1(text string) {
	fmt.Fprintln(os.Stdout, fmt.Sprintf("-----> %s", text))
}

// LogInfo1Quiet is the info1 header formatter (with quiet option)
func LogInfo1Quiet(text string) {
	if os.Getenv("DOKKU_QUIET_OUTPUT") != "" {
		LogInfo1(text)
	}
}

// LogInfo2 is the info2 header formatter
func LogInfo2(text string) {
	fmt.Fprintln(os.Stdout, fmt.Sprintf("=====> %s", text))
}

// LogInfo2Quiet is the info2 header formatter (with quiet option)
func LogInfo2Quiet(text string) {
	if os.Getenv("DOKKU_QUIET_OUTPUT") == "" {
		LogInfo2(text)
	}
}

// LogVerbose is the verbose log formatter
// prints indented text to stdout
func LogVerbose(text string) {
	fmt.Fprintln(os.Stdout, fmt.Sprintf("       %s", text))
}

// LogVerboseQuiet is the verbose log formatter
// prints indented text to stdout (with quiet option)
func LogVerboseQuiet(text string) {
	if os.Getenv("DOKKU_QUIET_OUTPUT") != "" {
		LogVerbose(text)
	}
}

// LogWarn is the warning log formatter
func LogWarn(text string) {
	fmt.Fprintln(os.Stderr, fmt.Sprintf(" !     %s", text))
}
<commit_msg>fix: correct the DOKKU_QUIET_OUTPUT env var check for golang log methods<commit_after>package common

import (
	"fmt"
	"os"
)

// LogFail is the failure log formatter
// prints text to stderr and exits with status 1
func LogFail(text string) {
	fmt.Fprintln(os.Stderr, fmt.Sprintf("FAILED: %s", text))
	os.Exit(1)
}

// LogInfo1 is the info1 header formatter
func LogInfo1(text string) {
	fmt.Fprintln(os.Stdout, fmt.Sprintf("-----> %s", text))
}

// LogInfo1Quiet is the info1 header formatter (with quiet option)
func LogInfo1Quiet(text string) {
	if os.Getenv("DOKKU_QUIET_OUTPUT") == "" {
		LogInfo1(text)
	}
}

// LogInfo2 is the info2 header formatter
func LogInfo2(text string) {
	fmt.Fprintln(os.Stdout, fmt.Sprintf("=====> %s", text))
}

// LogInfo2Quiet is the info2 header formatter (with quiet option)
func LogInfo2Quiet(text string) {
	if os.Getenv("DOKKU_QUIET_OUTPUT") == "" {
		LogInfo2(text)
	}
}

// LogVerbose is the verbose log formatter
// prints indented text to stdout
func LogVerbose(text string) {
	fmt.Fprintln(os.Stdout, fmt.Sprintf("       %s", text))
}

// LogVerboseQuiet is the verbose log formatter
// prints indented text to stdout (with quiet option)
func LogVerboseQuiet(text string) {
	if os.Getenv("DOKKU_QUIET_OUTPUT") == "" {
		LogVerbose(text)
	}
}

// LogWarn is the warning log formatter
func LogWarn(text string) {
	fmt.Fprintln(os.Stderr, fmt.Sprintf(" !     %s", text))
}
<|endoftext|>


<commit_before>package durafmt

import (
	"testing"
	"time"
)

var (
	testStrings []struct {
		test     string
		expected string
	}
	testTimes []struct {
		test     time.Duration
		expected string
	}
)

// TestParse for durafmt time.Duration conversion.
func TestParse(t *testing.T) {
	testTimes = []struct {
		test     time.Duration
		expected string
	}{
		{1 * time.Second, "1 second"},
		{1 * time.Minute, "1 minute"},
		{2 * time.Second, "2 seconds"},
		{2 * time.Minute, "2 minutes"},
		{1 * time.Hour, "1 hour"},
		{2 * time.Hour, "2 hours"},
		{10 * time.Hour, "10 hours"},
		{24 * time.Hour, "1 day"},
		{48 * time.Hour, "2 days"},
		{120 * time.Hour, "5 days"},
		{168 * time.Hour, "1 week"},
		{672 * time.Hour, "4 weeks"},
		{8760 * time.Hour, "1 year"},
		{17520 * time.Hour, "2 years"},
		{-1 * time.Second, "-1 second"},
		{-10 * time.Second, "-10 seconds"},
	}

	for _, table := range testTimes {
		result := Parse(table.test).String()
		if result != table.expected {
			t.Errorf("Parse(%q).String() = %q. got %q, expected %q",
				table.test, result, result, table.expected)
		}
	}
}

// TestParseString for durafmt duration string conversion.
func TestParseString(t *testing.T) {
	testStrings = []struct {
		test     string
		expected string
	}{
		{"1s", "1 second"},
		{"1m", "1 minute"},
		{"2m", "2 minutes"},
		{"1h", "1 hour"},
		{"10h", "10 hours"},
		{"24h", "1 day"},
		{"48h", "2 days"},
		{"120h", "5 days"},
		{"168h", "1 week"},
		{"672h", "4 weeks"},
		{"8760h", "1 year"},
		{"17520h", "2 years"},
		{"1m0s", "1 minute"},
		{"1m2s", "1 minute 2 seconds"},
		{"3h4m5s", "3 hours 4 minutes 5 seconds"},
		{"0s", "0 seconds"},
		{"0m", "0 minutes"},
		{"0h", "0 hours"},
		{"0m2s", "2 seconds"},
		{"0m2m", "2 minutes"},
		{"0m2m3h", "3 hours 2 minutes"},
		{"0m2m34h", "1 day 10 hours 2 minutes"},
		{"-1s", "-1 second"},
		{"-1m", "-1 minute"},
		{"-2m", "-2 minutes"},
		{"-1h", "-1 hour"},
		{"-10h", "-10 hours"},
		{"-24h", "-1 day"},
		{"-48h", "-2 days"},
		{"-120h", "-5 days"},
		{"-168h", "-1 week"},
		{"-672h", "-4 weeks"},
		{"-8760h", "-1 year"},
		{"-1m0s", "-1 minute"},
		{"-0m2s", "-2 seconds"},
		{"-0m2m", "-2 minutes"},
		{"-0m2m3h", "-3 hours 2 minutes"},
		{"-0m2m34h", "-1 day 10 hours 2 minutes"},
		{"-0s", "-0 seconds"},
		{"-0m", "-0 minutes"},
		{"-0h", "-0 hours"},
	}

	for _, table := range testStrings {
		d, err := ParseString(table.test)
		if err != nil {
			t.Errorf("%q", err)
		}
		result := d.String()
		if result != table.expected {
			t.Errorf("d.String() = %q. got %q, expected %q",
				table.test, result, table.expected)
		}
	}
}

// TestInvalidDuration for invalid inputs.
func TestInvalidDuration(t *testing.T) {
	testStrings = []struct {
		test     string
		expected string
	}{
		{"1", ""},
		{"1d", ""},
		{"1w", ""},
		{"1wk", ""},
		{"1y", ""},
		{"", ""},
		{"m1", ""},
		{"1nmd", ""},
		{"0", ""},
		{"-0", ""},
	}

	for _, table := range testStrings {
		_, err := ParseString(table.test)
		if err == nil {
			t.Errorf("NewDurable(%q). got %q, expected %q",
				table.test, err, table.expected)
		}
	}
}
<commit_msg>test modified for millisecond<commit_after>package durafmt

import (
	"testing"
	"time"
)

var (
	testStrings []struct {
		test     string
		expected string
	}
	testTimes []struct {
		test     time.Duration
		expected string
	}
)

// TestParse for durafmt time.Duration conversion.
func TestParse(t *testing.T) {
	testTimes = []struct {
		test     time.Duration
		expected string
	}{
		{1 * time.Second, "1 second"},
		{1 * time.Minute, "1 minute"},
		{2 * time.Second, "2 seconds"},
		{2 * time.Minute, "2 minutes"},
		{1 * time.Hour, "1 hour"},
		{2 * time.Hour, "2 hours"},
		{10 * time.Hour, "10 hours"},
		{24 * time.Hour, "1 day"},
		{48 * time.Hour, "2 days"},
		{120 * time.Hour, "5 days"},
		{168 * time.Hour, "1 week"},
		{672 * time.Hour, "4 weeks"},
		{8760 * time.Hour, "1 year"},
		{17520 * time.Hour, "2 years"},
		{-1 * time.Second, "-1 second"},
		{-10 * time.Second, "-10 seconds"},
	}

	for _, table := range testTimes {
		result := Parse(table.test).String()
		if result != table.expected {
			t.Errorf("Parse(%q).String() = %q. got %q, expected %q",
				table.test, result, result, table.expected)
		}
	}
}

// TestParseString for durafmt duration string conversion.
func TestParseString(t *testing.T) {
	testStrings = []struct {
		test     string
		expected string
	}{
		{"1s", "1 second"},
		{"1m", "1 minute"},
		{"2m", "2 minutes"},
		{"1h", "1 hour"},
		{"10h", "10 hours"},
		{"24h", "1 day"},
		{"48h", "2 days"},
		{"120h", "5 days"},
		{"168h", "1 week"},
		{"672h", "4 weeks"},
		{"8760h", "1 year"},
		{"17520h", "2 years"},
		{"1m0s", "1 minute"},
		{"1m2s", "1 minute 2 seconds"},
		{"3h4m5s", "3 hours 4 minutes 5 seconds"},
		{"0s", "0 seconds"},
		{"0m", "0 minutes0 milliseconds"},
		{"0h", "0 hours"},
		{"0m2s", "2 seconds"},
		{"0m2m", "2 minutes"},
		{"0m2m3h", "3 hours 2 minutes"},
		{"0m2m34h", "1 day 10 hours 2 minutes"},
		{"-1s", "-1 second"},
		{"-1m", "-1 minute"},
		{"-2m", "-2 minutes"},
		{"-1h", "-1 hour"},
		{"-10h", "-10 hours"},
		{"-24h", "-1 day"},
		{"-48h", "-2 days"},
		{"-120h", "-5 days"},
		{"-168h", "-1 week"},
		{"-672h", "-4 weeks"},
		{"-8760h", "-1 year"},
		{"-1m0s", "-1 minute"},
		{"-0m2s", "-2 seconds"},
		{"-0m2m", "-2 minutes"},
		{"-0m2m3h", "-3 hours 2 minutes"},
		{"-0m2m34h", "-1 day 10 hours 2 minutes"},
		{"-0s", "-0 seconds"},
		{"-0m", "-0 minutes0 milliseconds"},
		{"-0h", "-0 hours"},
	}

	for _, table := range testStrings {
		d, err := ParseString(table.test)
		if err != nil {
			t.Errorf("%q", err)
		}
		result := d.String()
		if result != table.expected {
			t.Errorf("d.String() = %q. got %q, expected %q",
				table.test, result, table.expected)
		}
	}
}

// TestInvalidDuration for invalid inputs.
func TestInvalidDuration(t *testing.T) {
	testStrings = []struct {
		test     string
		expected string
	}{
		{"1", ""},
		{"1d", ""},
		{"1w", ""},
		{"1wk", ""},
		{"1y", ""},
		{"", ""},
		{"m1", ""},
		{"1nmd", ""},
		{"0", ""},
		{"-0", ""},
	}

	for _, table := range testStrings {
		_, err := ParseString(table.test)
		if err == nil {
			t.Errorf("NewDurable(%q). got %q, expected %q",
				table.test, err, table.expected)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"image"
	"image/color"
	"image/draw"
	"os"
	"time"

	. "github.com/eaburns/quart/geom"
	"github.com/eaburns/quart/phys"

	"github.com/skelterjohn/go.wde"
)

const (
	width  = 640
	height = 480

	speed         = 5
	gravity       = -3
	stopThreshold = 1
)

var (
	vel    Vector
	circle = Circle{Center: Point{200, 200}, Radius: 50}

	// Sides is the set of polygon sides.
	sides = []Side{
		{{0, height - 1}, {0, 0}},
		{{0, 0}, {width - 1, 0}},
		{{width - 1, 0}, {width - 1, height - 1}},
		{{width - 1, height - 1}, {0, height - 1}},
	}

	// Click is the position of the latest mouse click.
	click = Point{-1, -1}

	// Cursor is the current cursor position.
	cursor Point

	// Stopped is true if the circle has effectively stopped moving.
	stopped bool
)

func main() {
	go mainLoop()
	wde.Run()
}

func mainLoop() {
	win, err := wde.NewWindow(width, height)
	if err != nil {
		panic(err)
	}
	win.SetTitle("geom test")
	win.Show()

	drawScene(win)

	tick := time.NewTicker(40 * time.Millisecond)
	for {
		select {
		case ev, ok := <-win.EventChan():
			if !ok {
				os.Exit(0)
			}
			switch ev := ev.(type) {
			case wde.CloseEvent:
				os.Exit(0)
			case wde.KeyTypedEvent:
				keyTyped(ev)
			case wde.KeyDownEvent:
				keyDown(wde.KeyEvent(ev))
			case wde.KeyUpEvent:
				keyUp(wde.KeyEvent(ev))
			case wde.MouseDraggedEvent:
				mouseMove(ev.MouseEvent)
			case wde.MouseMovedEvent:
				mouseMove(ev.MouseEvent)
			case wde.MouseDownEvent:
				mouseDown(wde.MouseButtonEvent(ev))
			case wde.MouseUpEvent:
				mouseUp(wde.MouseButtonEvent(ev))
			}

		case <-tick.C:
			if !stopped {
				start := circle.Center
				circle = phys.MoveCircle(circle, vel, sides)
				circle = phys.MoveCircle(circle, Vector{0, gravity}, sides)
				dist := start.Minus(circle.Center).Magnitude()
				stopped = dist < stopThreshold
			}
			drawScene(win)
		}
	}
}

func mouseMove(ev wde.MouseEvent) {
	cursor = Point{float64(ev.Where.X), float64(height - ev.Where.Y - 1)}
}

func mouseDown(ev wde.MouseButtonEvent) {
	switch ev.Which {
	case wde.LeftButton:
		click = Point{float64(ev.Where.X), float64(height - ev.Where.Y - 1)}
	}
}

func mouseUp(ev wde.MouseButtonEvent) {
	switch ev.Which {
	case wde.LeftButton:
		sides = append(sides, Side{click, cursor})
		click = Point{-1, -1}
	}
}

func keyTyped(ev wde.KeyTypedEvent) {
	switch ev.Key {
	case "d":
		if len(sides) > 4 {
			sides = sides[:len(sides)-1]
		}
	}
}

func keyDown(ev wde.KeyEvent) {
	switch ev.Key {
	case "left_arrow":
		vel[0] = -speed
	case "right_arrow":
		vel[0] = speed
	case "up_arrow":
		vel[1] = speed
	case "down_arrow":
		vel[1] = -speed
	}
	stopped = false
}

func keyUp(ev wde.KeyEvent) {
	switch ev.Key {
	case "left_arrow", "right_arrow":
		vel[0] = 0
	case "up_arrow", "down_arrow":
		vel[1] = 0
	}
}

func drawScene(win wde.Window) {
	clear(win)
	cv := ImageCanvas{win.Screen()}

	for _, s := range sides {
		s.Draw(cv, color.Black)
	}
	circle.Draw(cv, color.Black)

	if click[0] >= 0 {
		Side{click, cursor}.Draw(cv, color.RGBA{B: 255, A: 255})
	}

	win.FlushImage()
}

func clear(win wde.Window) {
	img := win.Screen()
	draw.Draw(img, img.Bounds(), image.NewUniform(color.White), image.ZP, draw.Src)
}
<commit_msg>Only set stopped=true when velocity is zero.<commit_after>package main

import (
	"image"
	"image/color"
	"image/draw"
	"os"
	"time"

	. "github.com/eaburns/quart/geom"
	"github.com/eaburns/quart/phys"

	"github.com/skelterjohn/go.wde"
)

const (
	width  = 640
	height = 480

	speed         = 5
	gravity       = -3
	stopThreshold = 1
)

var (
	vel    Vector
	circle = Circle{Center: Point{200, 200}, Radius: 50}

	// Sides is the set of polygon sides.
	sides = []Side{
		{{0, height - 1}, {0, 0}},
		{{0, 0}, {width - 1, 0}},
		{{width - 1, 0}, {width - 1, height - 1}},
		{{width - 1, height - 1}, {0, height - 1}},
	}

	// Click is the position of the latest mouse click.
	click = Point{-1, -1}

	// Cursor is the current cursor position.
	cursor Point

	// Stopped is true if the circle has effectively stopped moving.
	stopped bool
)

func main() {
	go mainLoop()
	wde.Run()
}

func mainLoop() {
	win, err := wde.NewWindow(width, height)
	if err != nil {
		panic(err)
	}
	win.SetTitle("geom test")
	win.Show()

	drawScene(win)

	tick := time.NewTicker(40 * time.Millisecond)
	for {
		select {
		case ev, ok := <-win.EventChan():
			if !ok {
				os.Exit(0)
			}
			switch ev := ev.(type) {
			case wde.CloseEvent:
				os.Exit(0)
			case wde.KeyTypedEvent:
				keyTyped(ev)
			case wde.KeyDownEvent:
				keyDown(wde.KeyEvent(ev))
			case wde.KeyUpEvent:
				keyUp(wde.KeyEvent(ev))
			case wde.MouseDraggedEvent:
				mouseMove(ev.MouseEvent)
			case wde.MouseMovedEvent:
				mouseMove(ev.MouseEvent)
			case wde.MouseDownEvent:
				mouseDown(wde.MouseButtonEvent(ev))
			case wde.MouseUpEvent:
				mouseUp(wde.MouseButtonEvent(ev))
			}

		case <-tick.C:
			if !stopped {
				start := circle.Center
				circle = phys.MoveCircle(circle, vel, sides)
				circle = phys.MoveCircle(circle, Vector{0, gravity}, sides)
				dist := start.Minus(circle.Center).Magnitude()
				stopped = vel.Equals(Vector{}) && dist < stopThreshold
			}
			drawScene(win)
		}
	}
}

func mouseMove(ev wde.MouseEvent) {
	cursor = Point{float64(ev.Where.X), float64(height - ev.Where.Y - 1)}
}

func mouseDown(ev wde.MouseButtonEvent) {
	switch ev.Which {
	case wde.LeftButton:
		click = Point{float64(ev.Where.X), float64(height - ev.Where.Y - 1)}
	}
}

func mouseUp(ev wde.MouseButtonEvent) {
	switch ev.Which {
	case wde.LeftButton:
		sides = append(sides, Side{click, cursor})
		click = Point{-1, -1}
	}
}

func keyTyped(ev wde.KeyTypedEvent) {
	switch ev.Key {
	case "d":
		if len(sides) > 4 {
			sides = sides[:len(sides)-1]
		}
	}
}

func keyDown(ev wde.KeyEvent) {
	switch ev.Key {
	case "left_arrow":
		vel[0] = -speed
	case "right_arrow":
		vel[0] = speed
	case "up_arrow":
		vel[1] = speed
	case "down_arrow":
		vel[1] = -speed
	}
	stopped = false
}

func keyUp(ev wde.KeyEvent) {
	switch ev.Key {
	case "left_arrow", "right_arrow":
		vel[0] = 0
	case "up_arrow", "down_arrow":
		vel[1] = 0
	}
}

func drawScene(win wde.Window) {
	clear(win)
	cv := ImageCanvas{win.Screen()}

	for _, s := range sides {
		s.Draw(cv, color.Black)
	}
	circle.Draw(cv, color.Black)

	if click[0] >= 0 {
		Side{click, cursor}.Draw(cv, color.RGBA{B: 255, A: 255})
	}

	win.FlushImage()
}

func clear(win wde.Window) {
	img := win.Screen()
	draw.Draw(img, img.Bounds(), image.NewUniform(color.White), image.ZP, draw.Src)
}
<|endoftext|>


<commit_before>package util

import (
	"bytes"
	"fmt"
	"testing"
)

var (
	bs    = []byte{0, 11, 22, 33, 44, 55, 66, 77}
	bsBuf = bytes.NewReader(bs)
	n     = len(bs)
)

func TestReadBytes(t *testing.T) {
	bsReader := NewReader(bsBuf)

	read, err := bsReader.ReadBytes(n)
	if err != nil {
		t.Error(err)
	}
	if n != len(read) {
		t.Errorf("Expecting to read: %v, got: %v", n, read)
	}
	if !bytes.Equal(bs, read) {
		t.Error("Expecting: %v, got: %v", bs, read)
	}

	fmt.Println(bsReader.buf.Buffered())
	if bsReader.buf.Buffered() != 0 {
		t.Errorf("Expecting buffered: %v, got: %v", 0, bsReader.buf.Buffered())
	}
}
<commit_msg>Add test to Reader.ReadTillAndWithDelims<commit_after>package util

import (
	"bytes"
	"testing"
)

var (
	bs   = []byte{0, 11, 22, 33, 44, 55, 77, 88, 55, 66, 77, 88}
	bsRd = bytes.NewReader(bs)
)

func TestReadBytes(t *testing.T) {
	bsRd.Reset(bs)
	bsReader := NewReader(bsRd)
	n := 5 // Read 5 elements

	read, err := bsReader.ReadBytes(n)
	if err != nil {
		t.Error(err)
	}
	if n != len(read) {
		t.Errorf("Expecting to read: %v, got: %v", n, read)
	}
	if !bytes.Equal(bs[:n], read) {
		t.Error("Expecting: %v, got: %v", bs[:n], len(read))
	}

	if bsReader.buf.Buffered() != len(bs)-n {
		t.Errorf("Expecting buffered: %v, got: %v", len(bs)-n, bsReader.buf.Buffered())
	}
}

func TestReadTillAndWithDelims(t *testing.T) {
	bsRd.Reset(bs)
	bsReader := NewReader(bsRd)
	n := 10
	delims := []byte{bs[n-2], bs[n-1]}

	read, err := bsReader.ReadTillAndWithDelims(delims)
	if err != nil {
		t.Error(err)
	}
	if n != len(read) {
		t.Errorf("Expecting to read: %v, got: %v", n, len(read))
	}
	if !bytes.Equal(bs[:n], read) {
		t.Error("Expecting: %v, got: %v", bs[:n], read)
	}
}
<|endoftext|>


<commit_before>package main

func main() {
	ex1()

}

func ex1() {
	var i interface{} = nil
	println("nil interface is nil?:", i == nil) // true
	println(i)                                  // (0x0, 0x0)

}
<commit_msg>nilinterface: add ex2<commit_after>package main

func main() {

	ex1()

	ex2()

}

func ex1() {
	var i interface{} = nil
	println("ex1")
	println("nil interface is nil?:", i == nil) // true
	println(i)                                  // (0x0, 0x0)
	println()

}

func ex2() {
	var v int = 10
	var i interface{} = v
	println("ex2")
	println("interface-to-value is nil?:", i == nil) // false
	println("value address:", &v)
	println(i)
	println()
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
	"bytes"
	"fmt"
	"math/rand"
	"sort"
	"sync"
	"time"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/errors"
	"k8s.io/kubernetes/pkg/util/workqueue"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates"
	schedulerapi "k8s.io/kubernetes/plugin/pkg/scheduler/api"
	"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"
)

type FailedPredicateMap map[string]string

type FitError struct {
	Pod              *api.Pod
	FailedPredicates FailedPredicateMap
}

var ErrNoNodesAvailable = fmt.Errorf("no nodes available to schedule pods")

// Error returns detailed information of why the pod failed to fit on each node
func (f *FitError) Error() string {
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("pod (%s) failed to fit in any node\n", f.Pod.Name))
	for node, predicate := range f.FailedPredicates {
		reason := fmt.Sprintf("fit failure on node (%s): %s\n", node, predicate)
		buf.WriteString(reason)
	}
	return buf.String()
}

type genericScheduler struct {
	cache         schedulercache.Cache
	predicates    map[string]algorithm.FitPredicate
	prioritizers  []algorithm.PriorityConfig
	extenders     []algorithm.SchedulerExtender
	pods          algorithm.PodLister
	random        *rand.Rand
	randomLock    sync.Mutex
	lastNodeIndex uint64
}

// Schedule tries to schedule the given pod to one of node in the node list.
// If it succeeds, it will return the name of the node.
// If it fails, it will return a Fiterror error with reasons.
func (g *genericScheduler) Schedule(pod *api.Pod, nodeLister algorithm.NodeLister) (string, error) {
	var trace *util.Trace
	if pod != nil {
		trace = util.NewTrace(fmt.Sprintf("Scheduling %s/%s", pod.Namespace, pod.Name))
	} else {
		trace = util.NewTrace("Scheduling <nil> pod")
	}
	defer trace.LogIfLong(20 * time.Millisecond)

	nodes, err := nodeLister.List()
	if err != nil {
		return "", err
	}
	if len(nodes.Items) == 0 {
		return "", ErrNoNodesAvailable
	}

	// Used for all fit and priority funcs.
	nodeNameToInfo, err := g.cache.GetNodeNameToInfoMap()
	if err != nil {
		return "", err
	}

	trace.Step("Computing predicates")
	filteredNodes, failedPredicateMap, err := findNodesThatFit(pod, nodeNameToInfo, g.predicates, nodes, g.extenders)
	if err != nil {
		return "", err
	}

	if len(filteredNodes.Items) == 0 {
		return "", &FitError{
			Pod:              pod,
			FailedPredicates: failedPredicateMap,
		}
	}

	trace.Step("Prioritizing")
	priorityList, err := PrioritizeNodes(pod, nodeNameToInfo, g.prioritizers, algorithm.FakeNodeLister(filteredNodes), g.extenders)
	if err != nil {
		return "", err
	}

	trace.Step("Selecting host")
	return g.selectHost(priorityList)
}

// selectHost takes a prioritized list of nodes and then picks one
// randomly from the nodes that had the highest score.
func (g *genericScheduler) selectHost(priorityList schedulerapi.HostPriorityList) (string, error) {
	if len(priorityList) == 0 {
		return "", fmt.Errorf("empty priorityList")
	}

	sort.Sort(sort.Reverse(priorityList))
	maxScore := priorityList[0].Score
	firstAfterMaxScore := sort.Search(len(priorityList), func(i int) bool { return priorityList[i].Score < maxScore })

	g.randomLock.Lock()
	ix := int(g.lastNodeIndex % uint64(firstAfterMaxScore))
	g.lastNodeIndex++
	g.randomLock.Unlock()

	return priorityList[ix].Host, nil
}

// Filters the nodes to find the ones that fit based on the given predicate functions
// Each node is passed through the predicate functions to determine if it is a fit
func findNodesThatFit(pod *api.Pod, nodeNameToInfo map[string]*schedulercache.NodeInfo, predicateFuncs map[string]algorithm.FitPredicate, nodes api.NodeList, extenders []algorithm.SchedulerExtender) (api.NodeList, FailedPredicateMap, error) {
	predicateResultLock := sync.Mutex{}
	filtered := []api.Node{}
	failedPredicateMap := FailedPredicateMap{}
	errs := []error{}

	checkNode := func(i int) {
		nodeName := nodes.Items[i].Name
		fits, failedPredicate, err := podFitsOnNode(pod, nodeNameToInfo[nodeName], predicateFuncs)

		predicateResultLock.Lock()
		defer predicateResultLock.Unlock()
		if err != nil {
			errs = append(errs, err)
			return
		}
		if fits {
			filtered = append(filtered, nodes.Items[i])
		} else {
			failedPredicateMap[nodeName] = failedPredicate
		}
	}
	workqueue.Parallelize(16, len(nodes.Items), checkNode)
	if len(errs) > 0 {
		return api.NodeList{}, FailedPredicateMap{}, errors.NewAggregate(errs)
	}

	if len(filtered) > 0 && len(extenders) != 0 {
		for _, extender := range extenders {
			filteredList, err := extender.Filter(pod, &api.NodeList{Items: filtered})
			if err != nil {
				return api.NodeList{}, FailedPredicateMap{}, err
			}
			filtered = filteredList.Items
			if len(filtered) == 0 {
				break
			}
		}
	}
	return api.NodeList{Items: filtered}, failedPredicateMap, nil
}

// Checks whether node with a given name and NodeInfo satisfies all predicateFuncs.
func podFitsOnNode(pod *api.Pod, info *schedulercache.NodeInfo, predicateFuncs map[string]algorithm.FitPredicate) (bool, string, error) {
	for _, predicate := range predicateFuncs {
		fit, err := predicate(pod, info)
		if err != nil {
			switch e := err.(type) {
			case *predicates.InsufficientResourceError:
				if fit {
					err := fmt.Errorf("got InsufficientResourceError: %v, but also fit='true' which is unexpected", e)
					return false, "", err
				}
			case *predicates.PredicateFailureError:
				if fit {
					err := fmt.Errorf("got PredicateFailureError: %v, but also fit='true' which is unexpected", e)
					return false, "", err
				}
			default:
				return false, "", err
			}
		}
		if !fit {
			if re, ok := err.(*predicates.InsufficientResourceError); ok {
				return false, fmt.Sprintf("Insufficient %s", re.ResourceName), nil
			}
			if re, ok := err.(*predicates.PredicateFailureError); ok {
				return false, re.PredicateName, nil
			} else {
				err := fmt.Errorf("SchedulerPredicates failed due to %v, which is unexpected.", err)
				return false, "", err
			}
		}
	}
	return true, "", nil
}

// Prioritizes the nodes by running the individual priority functions in parallel.
// Each priority function is expected to set a score of 0-10
// 0 is the lowest priority score (least preferred node) and 10 is the highest
// Each priority function can also have its own weight
// The node scores returned by the priority function are multiplied by the weights to get weighted scores
// All scores are finally combined (added) to get the total weighted scores of all nodes
func PrioritizeNodes(
	pod *api.Pod,
	nodeNameToInfo map[string]*schedulercache.NodeInfo,
	priorityConfigs []algorithm.PriorityConfig,
	nodeLister algorithm.NodeLister,
	extenders []algorithm.SchedulerExtender,
) (schedulerapi.HostPriorityList, error) {
	result := schedulerapi.HostPriorityList{}

	// If no priority configs are provided, then the EqualPriority function is applied
	// This is required to generate the priority list in the required format
	if len(priorityConfigs) == 0 && len(extenders) == 0 {
		return EqualPriority(pod, nodeNameToInfo, nodeLister)
	}

	var (
		mu             = sync.Mutex{}
		wg             = sync.WaitGroup{}
		combinedScores = map[string]int{}
		errs           []error
	)

	for _, priorityConfig := range priorityConfigs {
		// skip the priority function if the weight is specified as 0
		if priorityConfig.Weight == 0 {
			continue
		}

		wg.Add(1)
		go func(config algorithm.PriorityConfig) {
			defer wg.Done()
			weight := config.Weight
			priorityFunc := config.Function
			prioritizedList, err := priorityFunc(pod, nodeNameToInfo, nodeLister)

			mu.Lock()
			defer mu.Unlock()
			if err != nil {
				errs = append(errs, err)
				return
			}
			for i := range prioritizedList {
				host, score := prioritizedList[i].Host, prioritizedList[i].Score
				combinedScores[host] += score * weight
			}
		}(priorityConfig)
	}
	if len(errs) != 0 {
		return schedulerapi.HostPriorityList{}, errors.NewAggregate(errs)
	}

	// wait for all go routines to finish
	wg.Wait()

	if len(extenders) != 0 && nodeLister != nil {
		nodes, err := nodeLister.List()
		if err != nil {
			return schedulerapi.HostPriorityList{}, err
		}
		for _, extender := range extenders {
			wg.Add(1)
			go func(ext algorithm.SchedulerExtender) {
				defer wg.Done()
				prioritizedList, weight, err := ext.Prioritize(pod, &nodes)
				if err != nil {
					// Prioritization errors from extender can be ignored, let k8s/other extenders determine the priorities
					return
				}
				mu.Lock()
				for i := range *prioritizedList {
					host, score := (*prioritizedList)[i].Host, (*prioritizedList)[i].Score
					combinedScores[host] += score * weight
				}
				mu.Unlock()
			}(extender)
		}
	}
	// wait for all go routines to finish
	wg.Wait()

	for host, score := range combinedScores {
		glog.V(10).Infof("Host %s Score %d", host, score)
		result = append(result, schedulerapi.HostPriority{Host: host, Score: score})
	}
	return result, nil
}

// EqualPriority is a prioritizer function that gives an equal weight of one to all nodes
func EqualPriority(_ *api.Pod, nodeNameToInfo map[string]*schedulercache.NodeInfo, nodeLister algorithm.NodeLister) (schedulerapi.HostPriorityList, error) {
	nodes, err := nodeLister.List()
	if err != nil {
		glog.Errorf("Failed to list nodes: %v", err)
		return []schedulerapi.HostPriority{}, err
	}

	result := []schedulerapi.HostPriority{}
	for _, node := range nodes.Items {
		result = append(result, schedulerapi.HostPriority{
			Host:  node.Name,
			Score: 1,
		})
	}
	return result, nil
}

func NewGenericScheduler(cache schedulercache.Cache, predicates map[string]algorithm.FitPredicate, prioritizers []algorithm.PriorityConfig, extenders []algorithm.SchedulerExtender, random *rand.Rand) algorithm.ScheduleAlgorithm {
	return &genericScheduler{
		cache:        cache,
		predicates:   predicates,
		prioritizers: prioritizers,
		extenders:    extenders,
		random:       random,
	}
}
<commit_msg>Add the length detection of the "predicateFuncs" in generic_scheduler.go<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
	"bytes"
	"fmt"
	"math/rand"
	"sort"
	"sync"
	"time"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/errors"
	"k8s.io/kubernetes/pkg/util/workqueue"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates"
	schedulerapi "k8s.io/kubernetes/plugin/pkg/scheduler/api"
	"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"
)

type FailedPredicateMap map[string]string

type FitError struct {
	Pod              *api.Pod
	FailedPredicates FailedPredicateMap
}

var ErrNoNodesAvailable = fmt.Errorf("no nodes available to schedule pods")

// Error returns detailed information of why the pod failed to fit on each node
func (f *FitError) Error() string {
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("pod (%s) failed to fit in any node\n", f.Pod.Name))
	for node, predicate := range f.FailedPredicates {
		reason := fmt.Sprintf("fit failure on node (%s): %s\n", node, predicate)
		buf.WriteString(reason)
	}
	return buf.String()
}

type genericScheduler struct {
	cache         schedulercache.Cache
	predicates    map[string]algorithm.FitPredicate
	prioritizers  []algorithm.PriorityConfig
	extenders     []algorithm.SchedulerExtender
	pods          algorithm.PodLister
	random        *rand.Rand
	randomLock    sync.Mutex
	lastNodeIndex uint64
}

// Schedule tries to schedule the given pod to one of node in the node list.
// If it succeeds, it will return the name of the node.
// If it fails, it will return a Fiterror error with reasons.
func (g *genericScheduler) Schedule(pod *api.Pod, nodeLister algorithm.NodeLister) (string, error) {
	var trace *util.Trace
	if pod != nil {
		trace = util.NewTrace(fmt.Sprintf("Scheduling %s/%s", pod.Namespace, pod.Name))
	} else {
		trace = util.NewTrace("Scheduling <nil> pod")
	}
	defer trace.LogIfLong(20 * time.Millisecond)

	nodes, err := nodeLister.List()
	if err != nil {
		return "", err
	}
	if len(nodes.Items) == 0 {
		return "", ErrNoNodesAvailable
	}

	// Used for all fit and priority funcs.
	nodeNameToInfo, err := g.cache.GetNodeNameToInfoMap()
	if err != nil {
		return "", err
	}

	trace.Step("Computing predicates")
	filteredNodes, failedPredicateMap, err := findNodesThatFit(pod, nodeNameToInfo, g.predicates, nodes, g.extenders)
	if err != nil {
		return "", err
	}

	if len(filteredNodes.Items) == 0 {
		return "", &FitError{
			Pod:              pod,
			FailedPredicates: failedPredicateMap,
		}
	}

	trace.Step("Prioritizing")
	priorityList, err := PrioritizeNodes(pod, nodeNameToInfo, g.prioritizers, algorithm.FakeNodeLister(filteredNodes), g.extenders)
	if err != nil {
		return "", err
	}

	trace.Step("Selecting host")
	return g.selectHost(priorityList)
}

// selectHost takes a prioritized list of nodes and then picks one
// randomly from the nodes that had the highest score.
func (g *genericScheduler) selectHost(priorityList schedulerapi.HostPriorityList) (string, error) {
	if len(priorityList) == 0 {
		return "", fmt.Errorf("empty priorityList")
	}

	sort.Sort(sort.Reverse(priorityList))
	maxScore := priorityList[0].Score
	firstAfterMaxScore := sort.Search(len(priorityList), func(i int) bool { return priorityList[i].Score < maxScore })

	g.randomLock.Lock()
	ix := int(g.lastNodeIndex % uint64(firstAfterMaxScore))
	g.lastNodeIndex++
	g.randomLock.Unlock()

	return priorityList[ix].Host, nil
}

// Filters the nodes to find the ones that fit based on the given predicate functions
// Each node is passed through the predicate functions to determine if it is a fit
func findNodesThatFit(pod *api.Pod, nodeNameToInfo map[string]*schedulercache.NodeInfo, predicateFuncs map[string]algorithm.FitPredicate, nodes api.NodeList, extenders []algorithm.SchedulerExtender) (api.NodeList, FailedPredicateMap, error) {
	filtered := []api.Node{}
	failedPredicateMap := FailedPredicateMap{}

	if len(predicateFuncs) == 0 {
		filtered = nodes.Items
	} else {
		predicateResultLock := sync.Mutex{}
		errs := []error{}
		checkNode := func(i int) {
			nodeName := nodes.Items[i].Name
			fits, failedPredicate, err := podFitsOnNode(pod, nodeNameToInfo[nodeName], predicateFuncs)

			predicateResultLock.Lock()
			defer predicateResultLock.Unlock()
			if err != nil {
				errs = append(errs, err)
				return
			}
			if fits {
				filtered = append(filtered, nodes.Items[i])
			} else {
				failedPredicateMap[nodeName] = failedPredicate
			}
		}
		workqueue.Parallelize(16, len(nodes.Items), checkNode)
		if len(errs) > 0 {
			return api.NodeList{}, FailedPredicateMap{}, errors.NewAggregate(errs)
		}
	}

	if len(filtered) > 0 && len(extenders) != 0 {
		for _, extender := range extenders {
			filteredList, err := extender.Filter(pod, &api.NodeList{Items: filtered})
			if err != nil {
				return api.NodeList{}, FailedPredicateMap{}, err
			}
			filtered = filteredList.Items
			if len(filtered) == 0 {
				break
			}
		}
	}
	return api.NodeList{Items: filtered}, failedPredicateMap, nil
}

// Checks whether node with a given name and NodeInfo satisfies all predicateFuncs.
func podFitsOnNode(pod *api.Pod, info *schedulercache.NodeInfo, predicateFuncs map[string]algorithm.FitPredicate) (bool, string, error) {
	for _, predicate := range predicateFuncs {
		fit, err := predicate(pod, info)
		if err != nil {
			switch e := err.(type) {
			case *predicates.InsufficientResourceError:
				if fit {
					err := fmt.Errorf("got InsufficientResourceError: %v, but also fit='true' which is unexpected", e)
					return false, "", err
				}
			case *predicates.PredicateFailureError:
				if fit {
					err := fmt.Errorf("got PredicateFailureError: %v, but also fit='true' which is unexpected", e)
					return false, "", err
				}
			default:
				return false, "", err
			}
		}
		if !fit {
			if re, ok := err.(*predicates.InsufficientResourceError); ok {
				return false, fmt.Sprintf("Insufficient %s", re.ResourceName), nil
			}
			if re, ok := err.(*predicates.PredicateFailureError); ok {
				return false, re.PredicateName, nil
			} else {
				err := fmt.Errorf("SchedulerPredicates failed due to %v, which is unexpected.", err)
				return false, "", err
			}
		}
	}
	return true, "", nil
}

// Prioritizes the nodes by running the individual priority functions in parallel.
// Each priority function is expected to set a score of 0-10
// 0 is the lowest priority score (least preferred node) and 10 is the highest
// Each priority function can also have its own weight
// The node scores returned by the priority function are multiplied by the weights to get weighted scores
// All scores are finally combined (added) to get the total weighted scores of all nodes
func PrioritizeNodes(
	pod *api.Pod,
	nodeNameToInfo map[string]*schedulercache.NodeInfo,
	priorityConfigs []algorithm.PriorityConfig,
	nodeLister algorithm.NodeLister,
	extenders []algorithm.SchedulerExtender,
) (schedulerapi.HostPriorityList, error) {
	result := schedulerapi.HostPriorityList{}

	// If no priority configs are provided, then the EqualPriority function is applied
	// This is required to generate the priority list in the required format
	if len(priorityConfigs) == 0 && len(extenders) == 0 {
		return EqualPriority(pod, nodeNameToInfo, nodeLister)
	}

	var (
		mu             = sync.Mutex{}
		wg             = sync.WaitGroup{}
		combinedScores = map[string]int{}
		errs           []error
	)

	for _, priorityConfig := range priorityConfigs {
		// skip the priority function if the weight is specified as 0
		if priorityConfig.Weight == 0 {
			continue
		}

		wg.Add(1)
		go func(config algorithm.PriorityConfig) {
			defer wg.Done()
			weight := config.Weight
			priorityFunc := config.Function
			prioritizedList, err := priorityFunc(pod, nodeNameToInfo, nodeLister)

			mu.Lock()
			defer mu.Unlock()
			if err != nil {
				errs = append(errs, err)
				return
			}
			for i := range prioritizedList {
				host, score := prioritizedList[i].Host, prioritizedList[i].Score
				combinedScores[host] += score * weight
			}
		}(priorityConfig)
	}
	if len(errs) != 0 {
		return schedulerapi.HostPriorityList{}, errors.NewAggregate(errs)
	}

	// wait for all go routines to finish
	wg.Wait()

	if len(extenders) != 0 && nodeLister != nil {
		nodes, err := nodeLister.List()
		if err != nil {
			return schedulerapi.HostPriorityList{}, err
		}
		for _, extender := range extenders {
			wg.Add(1)
			go func(ext algorithm.SchedulerExtender) {
				defer wg.Done()
				prioritizedList, weight, err := ext.Prioritize(pod, &nodes)
				if err != nil {
					// Prioritization errors from extender can be ignored, let k8s/other extenders determine the priorities
					return
				}
				mu.Lock()
				for i := range *prioritizedList {
					host, score := (*prioritizedList)[i].Host, (*prioritizedList)[i].Score
					combinedScores[host] += score * weight
				}
				mu.Unlock()
			}(extender)
		}
	}
	// wait for all go routines to finish
	wg.Wait()

	for host, score := range combinedScores {
		glog.V(10).Infof("Host %s Score %d", host, score)
		result = append(result, schedulerapi.HostPriority{Host: host, Score: score})
	}
	return result, nil
}

// EqualPriority is a prioritizer function that gives an equal weight of one to all nodes
func EqualPriority(_ *api.Pod, nodeNameToInfo map[string]*schedulercache.NodeInfo, nodeLister algorithm.NodeLister) (schedulerapi.HostPriorityList, error) {
	nodes, err := nodeLister.List()
	if err != nil {
		glog.Errorf("Failed to list nodes: %v", err)
		return []schedulerapi.HostPriority{}, err
	}

	result := []schedulerapi.HostPriority{}
	for _, node := range nodes.Items {
		result = append(result, schedulerapi.HostPriority{
			Host:  node.Name,
			Score: 1,
		})
	}
	return result, nil
}

func NewGenericScheduler(cache schedulercache.Cache, predicates map[string]algorithm.FitPredicate, prioritizers []algorithm.PriorityConfig, extenders []algorithm.SchedulerExtender, random *rand.Rand) algorithm.ScheduleAlgorithm {
	return &genericScheduler{
		cache:        cache,
		predicates:   predicates,
		prioritizers: prioritizers,
		extenders:    extenders,
		random:       random,
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 CodeIgnition. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package policy

import (
	"errors"
	"testing"
	"time"

	"golang.org/x/net/context"
)

func fakePolicyHandler(ctx context.Context, p Policy) (<-chan Event, error) {
	foo, ok := p.M["foo"]
	if !ok {
		return nil, errors.New(`"foo" key missing in fake policy`)
	}
	bar, ok := p.M["bar"]
	if !ok {
		return nil, errors.New(`"bar" key missing in fake policy`)
	}
	out := make(chan Event)
	go func() {
		out <- Event{
			Time:   time.Now(),
			Policy: p,
			Data: map[string]interface{}{
				"foo": foo,
				"bar": bar,
			},
		}
		out <- Event{
			Time:   time.Now(),
			Policy: p,
			Data: map[string]interface{}{
				"foo": foo,
				"bar": bar,
			},
		}
		close(out)
	}()
	return out, nil
}

func TestNewHandler(t *testing.T) {
	err := NewHandler("", fakePolicyHandler)
	if err == nil {
		t.Fatal(errors.New("NewHandler should return an error when the type is empty"))
	}
	err = NewHandler("fake", fakePolicyHandler)
	if err != nil {
		t.Fatal(err)
	}
}

func TestValid(t *testing.T) {
	p := Policy{
		Name: "",
	}
	if err := p.Valid(); err == nil {
		t.Error(`want error "policy name can't be empty" got nil`)
	}
	p = Policy{
		Name: "dummy",
		Type: "unknownDummyType",
	}
	if err := p.Valid(); err == nil {
		t.Error(`want error "policy type unknown" got nil`)
	}
}

func TestExecute(t *testing.T) {
	p := Policy{
		Name: "dummy",
		Type: "fake",
		M: map[string]string{
			"foo": "foo_value",
			"bar": "bar_value",
		},
	}
	out, err := p.Execute(context.TODO())
	if err != nil {
		t.Error(err)
	}
	var count int
	for evt := range out {
		count++
		if evt.Data["foo"] != "foo_value" {
			t.Errorf(`want evt.Data["foo"] = %s; got %s`, "foo", evt.Data["foo"])
		}
		if evt.Data["bar"] != "bar_value" {
			t.Errorf(`want evt.Data["bar"] = %s; got %s`, "bar", evt.Data["bar"])
		}
	}
	if count != 2 {
		t.Errorf(`expected %d events; got %d events`, 2, count)
	}
}
<commit_msg>policy: test stopping policy execution using context cancel<commit_after>// Copyright 2015 CodeIgnition. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package policy

import (
	"errors"
	"testing"
	"time"

	"golang.org/x/net/context"
)

func fakePolicyHandler(ctx context.Context, p Policy) (<-chan Event, error) {
	foo, ok := p.M["foo"]
	if !ok {
		return nil, errors.New(`"foo" key missing in fake policy`)
	}
	interval, ok := p.M["interval"]
	if !ok {
		return nil, errors.New(`"interval" key missing in fake policy`)
	}
	d, err := time.ParseDuration(interval)
	if err != nil {
		return nil, err
	}

	// This check is here to ensure time.Ticker(d) doesn't panic
	if d <= 0 {
		return nil, errors.New("frequency must be a positive quantity")
	}

	out := make(chan Event)
	go func() {
		t := time.NewTicker(d)
		for {
			select {
			case <-ctx.Done():
				t.Stop()
				close(out)
				return
			case <-t.C:
				out <- Event{
					Time:   time.Now(),
					Policy: p,
					Data: map[string]interface{}{
						"foo": foo,
					},
				}
			}
		}
	}()
	return out, nil
}

func TestNewHandler(t *testing.T) {
	err := NewHandler("", fakePolicyHandler)
	if err == nil {
		t.Fatal(errors.New("NewHandler should return an error when the type is empty"))
	}
	err = NewHandler("fake", fakePolicyHandler)
	if err != nil {
		t.Fatal(err)
	}
}

func TestValid(t *testing.T) {
	p := Policy{
		Name: "",
	}
	if err := p.Valid(); err == nil {
		t.Error(`want error "policy name can't be empty" got nil`)
	}
	p = Policy{
		Name: "dummy",
		Type: "unknownDummyType",
	}
	if err := p.Valid(); err == nil {
		t.Error(`want error "policy type unknown" got nil`)
	}
}

func TestExecute(t *testing.T) {
	p := Policy{
		Name: "dummy",
		Type: "fake",
		M: map[string]string{
			"foo":      "foo_value",
			"interval": "200ms",
		},
	}
	ctx, cancel := context.WithCancel(context.Background())
	out, err := p.Execute(ctx)
	if err != nil {
		t.Error(err)
	}
	go func() {
		time.Sleep(1 * time.Second)
		cancel()
	}()

	var count int
	// Here, we are also able to test whether out is being
	// closed when cancel() is called. If out is not closed,
	// this test should have been running forever.
	for evt := range out {
		count++
		if evt.Data["foo"] != "foo_value" {
			t.Errorf(`want evt.Data["foo"] = %s; got %s`, "foo", evt.Data["foo"])
		}
	}

	// The interval for the dummy policy is 200ms.
	// We are calling cancel after 1 sec. Typically, we receive
	// 4 or 5 events in that duration.
	if count != 4 && count != 5 {
		t.Errorf(`want count to be either 4 or 5; got %d`, count)
	}
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package action

import (
	"fmt"
	"io/ioutil"
	"os"
	"syscall"

	"github.com/Masterminds/semver/v3"
	"github.com/pkg/errors"
	"golang.org/x/crypto/ssh/terminal"

	"helm.sh/helm/v3/pkg/chart"
	"helm.sh/helm/v3/pkg/chart/loader"
	"helm.sh/helm/v3/pkg/chartutil"
	"helm.sh/helm/v3/pkg/provenance"
)

// Package is the action for packaging a chart.
//
// It provides the implementation of 'helm package'.
type Package struct {
	Sign             bool
	Key              string
	Keyring          string
	Version          string
	AppVersion       string
	Destination      string
	DependencyUpdate bool

	RepositoryConfig string
	RepositoryCache  string
}

// NewPackage creates a new Package object with the given configuration.
func NewPackage() *Package {
	return &Package{}
}

// Run executes 'helm package' against the given chart and returns the path to the packaged chart.
func (p *Package) Run(path string, vals map[string]interface{}) (string, error) {
	ch, err := loader.LoadDir(path)
	if err != nil {
		return "", err
	}

	// If version is set, modify the version.
	if p.Version != "" {
		if err := setVersion(ch, p.Version); err != nil {
			return "", err
		}
	}

	if p.AppVersion != "" {
		ch.Metadata.AppVersion = p.AppVersion
	}

	if reqs := ch.Metadata.Dependencies; reqs != nil {
		if err := CheckDependencies(ch, reqs); err != nil {
			return "", err
		}
	}

	var dest string
	if p.Destination == "." {
		// Save to the current working directory.
		dest, err = os.Getwd()
		if err != nil {
			return "", err
		}
	} else {
		// Otherwise save to set destination
		dest = p.Destination
	}

	name, err := chartutil.Save(ch, dest)
	if err != nil {
		return "", errors.Wrap(err, "failed to save")
	}

	if p.Sign {
		err = p.Clearsign(name)
	}

	return name, err
}

func setVersion(ch *chart.Chart, ver string) error {
	// Verify that version is a Version, and error out if it is not.
	if _, err := semver.NewVersion(ver); err != nil {
		return err
	}

	// Set the version field on the chart.
	ch.Metadata.Version = ver
	return nil
}

// Clearsign signs a chart
func (p *Package) Clearsign(filename string) error {
	// Load keyring
	signer, err := provenance.NewFromKeyring(p.Keyring, p.Key)
	if err != nil {
		return err
	}

	if err := signer.DecryptKey(promptUser); err != nil {
		return err
	}

	sig, err := signer.ClearSign(filename)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(filename+".prov", []byte(sig), 0644)
}

// promptUser implements provenance.PassphraseFetcher
func promptUser(name string) ([]byte, error) {
	fmt.Printf("Password for key %q >  ", name)
	pw, err := terminal.ReadPassword(syscall.Stdin)
	fmt.Println()
	return pw, err
}
<commit_msg>Fixing failing CI for windows<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package action

import (
	"fmt"
	"io/ioutil"
	"os"
	"syscall"

	"github.com/Masterminds/semver/v3"
	"github.com/pkg/errors"
	"golang.org/x/crypto/ssh/terminal"

	"helm.sh/helm/v3/pkg/chart"
	"helm.sh/helm/v3/pkg/chart/loader"
	"helm.sh/helm/v3/pkg/chartutil"
	"helm.sh/helm/v3/pkg/provenance"
)

// Package is the action for packaging a chart.
//
// It provides the implementation of 'helm package'.
type Package struct {
	Sign             bool
	Key              string
	Keyring          string
	Version          string
	AppVersion       string
	Destination      string
	DependencyUpdate bool

	RepositoryConfig string
	RepositoryCache  string
}

// NewPackage creates a new Package object with the given configuration.
func NewPackage() *Package {
	return &Package{}
}

// Run executes 'helm package' against the given chart and returns the path to the packaged chart.
func (p *Package) Run(path string, vals map[string]interface{}) (string, error) {
	ch, err := loader.LoadDir(path)
	if err != nil {
		return "", err
	}

	// If version is set, modify the version.
	if p.Version != "" {
		if err := setVersion(ch, p.Version); err != nil {
			return "", err
		}
	}

	if p.AppVersion != "" {
		ch.Metadata.AppVersion = p.AppVersion
	}

	if reqs := ch.Metadata.Dependencies; reqs != nil {
		if err := CheckDependencies(ch, reqs); err != nil {
			return "", err
		}
	}

	var dest string
	if p.Destination == "." {
		// Save to the current working directory.
		dest, err = os.Getwd()
		if err != nil {
			return "", err
		}
	} else {
		// Otherwise save to set destination
		dest = p.Destination
	}

	name, err := chartutil.Save(ch, dest)
	if err != nil {
		return "", errors.Wrap(err, "failed to save")
	}

	if p.Sign {
		err = p.Clearsign(name)
	}

	return name, err
}

func setVersion(ch *chart.Chart, ver string) error {
	// Verify that version is a Version, and error out if it is not.
	if _, err := semver.NewVersion(ver); err != nil {
		return err
	}

	// Set the version field on the chart.
	ch.Metadata.Version = ver
	return nil
}

// Clearsign signs a chart
func (p *Package) Clearsign(filename string) error {
	// Load keyring
	signer, err := provenance.NewFromKeyring(p.Keyring, p.Key)
	if err != nil {
		return err
	}

	if err := signer.DecryptKey(promptUser); err != nil {
		return err
	}

	sig, err := signer.ClearSign(filename)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(filename+".prov", []byte(sig), 0644)
}

// promptUser implements provenance.PassphraseFetcher
func promptUser(name string) ([]byte, error) {
	fmt.Printf("Password for key %q >  ", name)
	// syscall.Stdin is not an int in all environments and needs to be coerced
	// into one there (e.g., Windows)
	pw, err := terminal.ReadPassword(int(syscall.Stdin))
	fmt.Println()
	return pw, err
}
<|endoftext|>


<commit_before>package api

import (
	"context"
	"net/http"
	"time"

	"github.com/go-chi/chi/middleware"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/satori/go.uuid"
)

type contextValue string

const contextReqID contextValue = "request_id"

//NewRequestID creates a middleware that passes X-Request-ID via contenxt or creates a new ID
func NewRequestID(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		reqID := r.Header.Get("X-Request-ID")
		if reqID == "" {
			reqID = uuid.NewV4().String()
		}

		r = r.WithContext(context.WithValue(r.Context(), contextReqID, reqID))

		next.ServeHTTP(w, r)
	})
}

//GetRequestID returns the request ID from context.Context
func GetRequestID(ctx context.Context) string {
	value := ctx.Value(contextReqID)
	if reqID, ok := value.(string); ok {
		return reqID
	}
	return ""
}

//NewRequestLogger creates a middleware that logs all HTTP request information
func NewRequestLogger(logger log.Logger) func(next http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			start := time.Now()

			ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
			next.ServeHTTP(ww, r)

			level.Debug(logger).Log(
				"request", GetRequestID(r.Context()),
				"proto", r.Proto,
				"method", r.Method,
				"status", ww.Status(),
				"path", r.URL.Path,
				"duration", time.Since(start),
				"bytes", ww.BytesWritten(),
			)
		})
	}
}
<commit_msg>Fix typo in pkg/api/request_id.go<commit_after>package api

import (
	"context"
	"net/http"
	"time"

	"github.com/go-chi/chi/middleware"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/satori/go.uuid"
)

type contextValue string

const contextReqID contextValue = "request_id"

//NewRequestID creates a middleware that passes X-Request-ID via context or creates a new ID
func NewRequestID(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		reqID := r.Header.Get("X-Request-ID")
		if reqID == "" {
			reqID = uuid.NewV4().String()
		}

		r = r.WithContext(context.WithValue(r.Context(), contextReqID, reqID))

		next.ServeHTTP(w, r)
	})
}

//GetRequestID returns the request ID from context.Context
func GetRequestID(ctx context.Context) string {
	value := ctx.Value(contextReqID)
	if reqID, ok := value.(string); ok {
		return reqID
	}
	return ""
}

//NewRequestLogger creates a middleware that logs all HTTP request information
func NewRequestLogger(logger log.Logger) func(next http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			start := time.Now()

			ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
			next.ServeHTTP(ww, r)

			level.Debug(logger).Log(
				"request", GetRequestID(r.Context()),
				"proto", r.Proto,
				"method", r.Method,
				"status", ww.Status(),
				"path", r.URL.Path,
				"duration", time.Since(start),
				"bytes", ww.BytesWritten(),
			)
		})
	}
}
<|endoftext|>


<commit_before>package api

import "testing"

func TestVolumeListSet(t *testing.T) {
	table := map[string][]VolumeSpec{
		"/test:":             {{Source: "/test", Destination: "."}},
		"/test:/test":        {{Source: "/test", Destination: "/test"}},
		"/test/foo:/etc/ssl": {{Source: "/test/foo", Destination: "/etc/ssl"}},
		":/foo":              {{Source: ".", Destination: "/foo"}},
		"/foo":               {{Source: "/foo", Destination: "."}},
		":":                  {{Source: ".", Destination: "."}},
		"/t est/foo:":        {{Source: "/t est/foo", Destination: "."}},
		`"/test":"/foo"`:     {{Source: "/test", Destination: "/foo"}},
		`'/test':"/foo"`:     {{Source: "/test", Destination: "/foo"}},
		`"/te"st":"/foo"`:    {},
		"/test/foo:/ss;ss":   {},
		"/test;foo:/ssss":    {},
	}
	for v, expected := range table {
		got := VolumeList{}
		err := got.Set(v)
		if len(expected) == 0 {
			if err == nil {
				t.Errorf("Expected error for %q, got %#v", v, got)
			} else {
				continue
			}
		}
		if len(got) != len(expected) {
			t.Errorf("Expected %d injection in the list for %q, got %d", len(expected), v, len(got))
		}
		for _, exp := range expected {
			found := false
			for _, g := range got {
				if g.Source == exp.Source && g.Destination == exp.Destination {
					found = true
				}
			}
			if !found {
				t.Errorf("Expected %+v injection found in %#v list", exp, got)
			}
		}
	}
}

func TestEnvironmentSet(t *testing.T) {
	table := map[string][]EnvironmentSpec{
		"FOO=bar":  {{Name: "FOO", Value: "bar"}},
		"FOO=":     {{Name: "FOO", Value: ""}},
		"FOO":      {},
		"=":        {},
		"FOO=bar,": {{Name: "FOO", Value: "bar,"}},
		// Users should get a deprecation warning in this case
		// TODO: Create fake glog interface to be able to verify this.
		"FOO=bar,BAR=foo": {{Name: "FOO", Value: "bar,BAR=foo"}},
	}

	for v, expected := range table {
		got := EnvironmentList{}
		err := got.Set(v)
		if len(expected) == 0 && err == nil {
			t.Errorf("Expected error for env %q", v)
			continue
		}
		if len(expected) != len(got) {
			t.Errorf("got %d items, expected %d items in the list for %q", len(got), len(expected), v)
			continue
		}
		for _, exp := range expected {
			found := false
			for _, g := range got {
				if g.Name == exp.Name && g.Value == exp.Value {
					found = true
				}
			}
			if !found {
				t.Errorf("Expected %+v environment found in %#v list", exp, got)
			}
		}
	}
}
<commit_msg>if found = true, add break is better<commit_after>package api

import "testing"

func TestVolumeListSet(t *testing.T) {
	table := map[string][]VolumeSpec{
		"/test:":             {{Source: "/test", Destination: "."}},
		"/test:/test":        {{Source: "/test", Destination: "/test"}},
		"/test/foo:/etc/ssl": {{Source: "/test/foo", Destination: "/etc/ssl"}},
		":/foo":              {{Source: ".", Destination: "/foo"}},
		"/foo":               {{Source: "/foo", Destination: "."}},
		":":                  {{Source: ".", Destination: "."}},
		"/t est/foo:":        {{Source: "/t est/foo", Destination: "."}},
		`"/test":"/foo"`:     {{Source: "/test", Destination: "/foo"}},
		`'/test':"/foo"`:     {{Source: "/test", Destination: "/foo"}},
		`"/te"st":"/foo"`:    {},
		"/test/foo:/ss;ss":   {},
		"/test;foo:/ssss":    {},
	}
	for v, expected := range table {
		got := VolumeList{}
		err := got.Set(v)
		if len(expected) == 0 {
			if err == nil {
				t.Errorf("Expected error for %q, got %#v", v, got)
			} else {
				continue
			}
		}
		if len(got) != len(expected) {
			t.Errorf("Expected %d injection in the list for %q, got %d", len(expected), v, len(got))
		}
		for _, exp := range expected {
			found := false
			for _, g := range got {
				if g.Source == exp.Source && g.Destination == exp.Destination {
					found = true
					break
				}
			}
			if !found {
				t.Errorf("Expected %+v injection found in %#v list", exp, got)
			}
		}
	}
}

func TestEnvironmentSet(t *testing.T) {
	table := map[string][]EnvironmentSpec{
		"FOO=bar":  {{Name: "FOO", Value: "bar"}},
		"FOO=":     {{Name: "FOO", Value: ""}},
		"FOO":      {},
		"=":        {},
		"FOO=bar,": {{Name: "FOO", Value: "bar,"}},
		// Users should get a deprecation warning in this case
		// TODO: Create fake glog interface to be able to verify this.
		"FOO=bar,BAR=foo": {{Name: "FOO", Value: "bar,BAR=foo"}},
	}

	for v, expected := range table {
		got := EnvironmentList{}
		err := got.Set(v)
		if len(expected) == 0 && err == nil {
			t.Errorf("Expected error for env %q", v)
			continue
		}
		if len(expected) != len(got) {
			t.Errorf("got %d items, expected %d items in the list for %q", len(got), len(expected), v)
			continue
		}
		for _, exp := range expected {
			found := false
			for _, g := range got {
				if g.Name == exp.Name && g.Value == exp.Value {
					found = true
					break
				}
			}
			if !found {
				t.Errorf("Expected %+v environment found in %#v list", exp, got)
			}
		}
	}
}
<|endoftext|>


<commit_before>package utils

import (
	"io"
	"net/http"
	"net/textproto"
	"strconv"
	"strings"
)

// ServeContent replies to the request using the content in the provided
// reader. The Content-Length and Content-Type headers are added with the
// provided values.
func ServeContent(w http.ResponseWriter, r *http.Request, contentType string, size int64, content io.Reader) {
	h := w.Header()
	if size > 0 {
		h.Set("Content-Length", strconv.FormatInt(size, 10))
	}
	if contentType != "" {
		h.Set("Content-Type", contentType)
	}
	w.WriteHeader(http.StatusOK)
	if r.Method != "HEAD" {
		io.Copy(w, content)
	}
}

// CheckPreconditions evaluates request preconditions based only on the Etag
// values.
func CheckPreconditions(w http.ResponseWriter, r *http.Request, etag string) (done bool) {
	if etag != "" && !checkIfNoneMatch(w, r, etag) {
		writeNotModified(w)
		return true
	}
	return false
}

func checkIfNoneMatch(w http.ResponseWriter, r *http.Request, definedETag string) (match bool) {
	inm := r.Header.Get("If-None-Match")
	if inm == "" {
		return false
	}
	buf := inm
	for {
		buf = textproto.TrimString(buf)
		if len(buf) == 0 {
			break
		}
		if buf[0] == ',' {
			buf = buf[1:]
		}
		if buf[0] == '*' {
			return false
		}
		etag, remain := scanETag(buf)
		if etag == "" {
			break
		}
		if etagWeakMatch(etag, definedETag) {
			return false
		}
		buf = remain
	}
	return true
}

// etagWeakMatch reports whether a and b match using weak ETag comparison.
// Assumes a and b are valid ETags.
func etagWeakMatch(a, b string) bool {
	return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/")
}

// etagStrongMatch reports whether a and b match using strong ETag comparison.
// Assumes a and b are valid ETags.
func etagStrongMatch(a, b string) bool {
	return a == b && a != "" && a[0] == '"'
}

// scanETag determines if a syntactically valid ETag is present at s. If so,
// the ETag and remaining text after consuming ETag is returned. Otherwise,
// it returns "", "".
func scanETag(s string) (etag string, remain string) {
	start := 0
	if len(s) >= 2 && s[0] == 'W' && s[1] == '/' {
		start = 2
	}
	if len(s[start:]) < 2 || s[start] != '"' {
		return "", ""
	}
	// ETag is either W/"text" or "text".
	// See RFC 7232 2.3.
	for i := start + 1; i < len(s); i++ {
		c := s[i]
		switch {
		// Character values allowed in ETags.
		case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80:
		case c == '"':
			return s[:i+1], s[i+1:]
		default:
			return "", ""
		}
	}
	return "", ""
}

func writeNotModified(w http.ResponseWriter) {
	h := w.Header()
	delete(h, "Content-Type")
	delete(h, "Content-Length")
	w.WriteHeader(http.StatusNotModified)
}
<commit_msg>Keep content-type and content-length in unmodified response<commit_after>package utils

import (
	"io"
	"net/http"
	"net/textproto"
	"strconv"
	"strings"
)

// ServeContent replies to the request using the content in the provided
// reader. The Content-Length and Content-Type headers are added with the
// provided values.
func ServeContent(w http.ResponseWriter, r *http.Request, contentType string, size int64, content io.Reader) {
	h := w.Header()
	if size > 0 {
		h.Set("Content-Length", strconv.FormatInt(size, 10))
	}
	if contentType != "" {
		h.Set("Content-Type", contentType)
	}
	w.WriteHeader(http.StatusOK)
	if r.Method != "HEAD" {
		io.Copy(w, content)
	}
}

// CheckPreconditions evaluates request preconditions based only on the Etag
// values.
func CheckPreconditions(w http.ResponseWriter, r *http.Request, etag string) (done bool) {
	if etag != "" && !checkIfNoneMatch(w, r, etag) {
		w.WriteHeader(http.StatusNotModified)
		return true
	}
	return false
}

func checkIfNoneMatch(w http.ResponseWriter, r *http.Request, definedETag string) (match bool) {
	inm := r.Header.Get("If-None-Match")
	if inm == "" {
		return false
	}
	buf := inm
	for {
		buf = textproto.TrimString(buf)
		if len(buf) == 0 {
			break
		}
		if buf[0] == ',' {
			buf = buf[1:]
		}
		if buf[0] == '*' {
			return false
		}
		etag, remain := scanETag(buf)
		if etag == "" {
			break
		}
		if etagWeakMatch(etag, definedETag) {
			return false
		}
		buf = remain
	}
	return true
}

// etagWeakMatch reports whether a and b match using weak ETag comparison.
// Assumes a and b are valid ETags.
func etagWeakMatch(a, b string) bool {
	return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/")
}

// etagStrongMatch reports whether a and b match using strong ETag comparison.
// Assumes a and b are valid ETags.
func etagStrongMatch(a, b string) bool {
	return a == b && a != "" && a[0] == '"'
}

// scanETag determines if a syntactically valid ETag is present at s. If so,
// the ETag and remaining text after consuming ETag is returned. Otherwise,
// it returns "", "".
func scanETag(s string) (etag string, remain string) {
	start := 0
	if len(s) >= 2 && s[0] == 'W' && s[1] == '/' {
		start = 2
	}
	if len(s[start:]) < 2 || s[start] != '"' {
		return "", ""
	}
	// ETag is either W/"text" or "text".
	// See RFC 7232 2.3.
	for i := start + 1; i < len(s); i++ {
		c := s[i]
		switch {
		// Character values allowed in ETags.
		case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80:
		case c == '"':
			return s[:i+1], s[i+1:]
		default:
			return "", ""
		}
	}
	return "", ""
}
<|endoftext|>