<commit_before>package memberlist

import (
	"bytes"
	"compress/lzw"
	"encoding/binary"
	"fmt"
	"io"
	"math"
	"math/rand"
	"net"
	"strings"
	"time"

	"github.com/hashicorp/go-msgpack/codec"
)

// pushPullScale is the minimum number of nodes
// before we start scaling the push/pull timing. The scale
// effect is the log2(Nodes) - log2(pushPullScale). This means
// that the 33rd node will cause us to double the interval,
// while the 65th will triple it.
const pushPullScaleThreshold = 32

/*
 * Contains an entry for each private block:
 * 10.0.0.0/8
 * 100.64.0.0/10
 * 127.0.0.0/8
 * 169.254.0.0/16
 * 172.16.0.0/12
 * 192.168.0.0/16
 */
var privateBlocks []*net.IPNet

var loopbackBlock *net.IPNet

const (
	// Constant litWidth 2-8
	lzwLitWidth = 8
)

func init() {
	// Seed the random number generator
	rand.Seed(time.Now().UnixNano())

	// Add each private block
	privateBlocks = make([]*net.IPNet, 6)

	_, block, err := net.ParseCIDR("10.0.0.0/8")
	if err != nil {
		panic(fmt.Sprintf("Bad cidr. Got %v", err))
	}
	privateBlocks[0] = block

	_, block, err = net.ParseCIDR("100.64.0.0/10")
	if err != nil {
		panic(fmt.Sprintf("Bad cidr. Got %v", err))
	}
	privateBlocks[1] = block

	_, block, err = net.ParseCIDR("127.0.0.0/8")
	if err != nil {
		panic(fmt.Sprintf("Bad cidr. Got %v", err))
	}
	privateBlocks[2] = block

	_, block, err = net.ParseCIDR("169.254.0.0/16")
	if err != nil {
		panic(fmt.Sprintf("Bad cidr. Got %v", err))
	}
	privateBlocks[3] = block

	_, block, err = net.ParseCIDR("172.16.0.0/12")
	if err != nil {
		panic(fmt.Sprintf("Bad cidr. Got %v", err))
	}
	privateBlocks[4] = block

	_, block, err = net.ParseCIDR("192.168.0.0/16")
	if err != nil {
		panic(fmt.Sprintf("Bad cidr. Got %v", err))
	}
	privateBlocks[5] = block

	_, block, err = net.ParseCIDR("127.0.0.0/8")
	if err != nil {
		panic(fmt.Sprintf("Bad cidr. Got %v", err))
	}
	loopbackBlock = block
}

// Decode reverses the encode operation on a byte slice input
func decode(buf []byte, out interface{}) error {
	r := bytes.NewReader(buf)
	hd := codec.MsgpackHandle{}
	dec := codec.NewDecoder(r, &hd)
	return dec.Decode(out)
}

// Encode writes an encoded object to a new bytes buffer
func encode(msgType messageType, in interface{}) (*bytes.Buffer, error) {
	buf := bytes.NewBuffer(nil)
	buf.WriteByte(uint8(msgType))
	hd := codec.MsgpackHandle{}
	enc := codec.NewEncoder(buf, &hd)
	err := enc.Encode(in)
	return buf, err
}

// GetPrivateIP returns the first private IP address found in a list of
// addresses.
func GetPrivateIP(addresses []net.Addr) (net.IP, error) {
	var candidates []net.IP

	// Find private IPv4 address
	for _, rawAddr := range addresses {
		var ip net.IP
		switch addr := rawAddr.(type) {
		case *net.IPAddr:
			ip = addr.IP
		case *net.IPNet:
			ip = addr.IP
		default:
			continue
		}

		if ip.To4() == nil {
			continue
		}
		if !IsPrivateIP(ip.String()) {
			continue
		}
		candidates = append(candidates, ip)
	}
	numIps := len(candidates)
	switch numIps {
	case 0:
		return nil, fmt.Errorf("No private IP address found")
	case 1:
		return candidates[0], nil
	default:
		return nil, fmt.Errorf("Multiple private IPs found. Please configure one.")
	}
}

// Returns a random offset between 0 and n
func randomOffset(n int) int {
	if n == 0 {
		return 0
	}
	return int(rand.Uint32() % uint32(n))
}

// suspicionTimeout computes the timeout that should be used when
// a node is suspected
func suspicionTimeout(suspicionMult, n int, interval time.Duration) time.Duration {
	nodeScale := math.Ceil(math.Log10(float64(n + 1)))
	timeout := time.Duration(suspicionMult) * time.Duration(nodeScale) * interval
	return timeout
}

// retransmitLimit computes the limit of retransmissions
func retransmitLimit(retransmitMult, n int) int {
	nodeScale := math.Ceil(math.Log10(float64(n + 1)))
	limit := retransmitMult * int(nodeScale)
	return limit
}

// shuffleNodes randomly shuffles the input nodes using the Fisher-Yates shuffle
func shuffleNodes(nodes []*nodeState) {
	n := len(nodes)
	for i := n - 1; i > 0; i-- {
		j := rand.Intn(i + 1)
		nodes[i], nodes[j] = nodes[j], nodes[i]
	}
}

// pushPushScale is used to scale the time interval at which push/pull
// syncs take place. It is used to prevent network saturation as the
// cluster size grows
func pushPullScale(interval time.Duration, n int) time.Duration {
	// Don't scale until we cross the threshold
	if n <= pushPullScaleThreshold {
		return interval
	}

	multiplier := math.Ceil(math.Log2(float64(n))-math.Log2(pushPullScaleThreshold)) + 1.0
	return time.Duration(multiplier) * interval
}

// moveDeadNodes moves all the nodes in the dead state
// to the end of the slice and returns the index of the first dead node.
func moveDeadNodes(nodes []*nodeState) int {
	numDead := 0
	n := len(nodes)
	for i := 0; i < n-numDead; i++ {
		if nodes[i].State != stateDead {
			continue
		}

		// Move this node to the end
		nodes[i], nodes[n-numDead-1] = nodes[n-numDead-1], nodes[i]
		numDead++
		i--
	}
	return n - numDead
}

// kRandomNodes is used to select up to k random nodes, excluding a given
// node and any non-alive nodes. It is possible that less than k nodes are returned.
func kRandomNodes(k int, excludes []string, nodes []*nodeState) []*nodeState {
	n := len(nodes)
	kNodes := make([]*nodeState, 0, k)
OUTER:
	// Probe up to 3*n times, with large n this is not necessary
	// since k << n, but with small n we want search to be
	// exhaustive
	for i := 0; i < 3*n && len(kNodes) < k; i++ {
		// Get random node
		idx := randomOffset(n)
		node := nodes[idx]

		// Exclude node if match
		for _, exclude := range excludes {
			if node.Name == exclude {
				continue OUTER
			}
		}

		// Exclude if not alive
		if node.State != stateAlive {
			continue
		}

		// Check if we have this node already
		for j := 0; j < len(kNodes); j++ {
			if node == kNodes[j] {
				continue OUTER
			}
		}

		// Append the node
		kNodes = append(kNodes, node)
	}
	return kNodes
}

// makeCompoundMessage takes a list of messages and generates
// a single compound message containing all of them
func makeCompoundMessage(msgs [][]byte) *bytes.Buffer {
	// Create a local buffer
	buf := bytes.NewBuffer(nil)

	// Write out the type
	buf.WriteByte(uint8(compoundMsg))

	// Write out the number of message
	buf.WriteByte(uint8(len(msgs)))

	// Add the message lengths
	for _, m := range msgs {
		binary.Write(buf, binary.BigEndian, uint16(len(m)))
	}

	// Append the messages
	for _, m := range msgs {
		buf.Write(m)
	}

	return buf
}

// decodeCompoundMessage splits a compound message and returns
// the slices of individual messages. Also returns the number
// of truncated messages and any potential error
func decodeCompoundMessage(buf []byte) (trunc int, parts [][]byte, err error) {
	if len(buf) < 1 {
		err = fmt.Errorf("missing compound length byte")
		return
	}
	numParts := uint8(buf[0])
	buf = buf[1:]

	// Check we have enough bytes
	if len(buf) < int(numParts*2) {
		err = fmt.Errorf("truncated len slice")
		return
	}

	// Decode the lengths
	lengths := make([]uint16, numParts)
	for i := 0; i < int(numParts); i++ {
		lengths[i] = binary.BigEndian.Uint16(buf[i*2 : i*2+2])
	}
	buf = buf[numParts*2:]

	// Split each message
	for idx, msgLen := range lengths {
		if len(buf) < int(msgLen) {
			trunc = int(numParts) - idx
			return
		}

		// Extract the slice, seek past on the buffer
		slice := buf[:msgLen]
		buf = buf[msgLen:]
		parts = append(parts, slice)
	}
	return
}

// Returns if the given IP is in a private block
func IsPrivateIP(ip_str string) bool {
	ip := net.ParseIP(ip_str)
	for _, priv := range privateBlocks {
		if priv.Contains(ip) {
			return true
		}
	}
	return false
}

// Returns if the given IP is in a loopback block
func isLoopbackIP(ip_str string) bool {
	ip := net.ParseIP(ip_str)
	return loopbackBlock.Contains(ip)
}

// Given a string of the form "host", "host:port",
// "ipv6::addr" or "[ipv6::address]:port",
// return true if the string includes a port.
func hasPort(s string) bool {
	if s == "" {
		return false
	}
	last := strings.LastIndex(s, ":")
	if s[0] == '[' {
		return last > strings.LastIndex(s, "]")
	}
	first := strings.Index(s, ":")
	return first != -1 && first == last
}

// compressPayload takes an opaque input buffer, compresses it
// and wraps it in a compress{} message that is encoded.
func compressPayload(inp []byte) (*bytes.Buffer, error) {
	var buf bytes.Buffer
	compressor := lzw.NewWriter(&buf, lzw.LSB, lzwLitWidth)

	_, err := compressor.Write(inp)
	if err != nil {
		return nil, err
	}

	// Ensure we flush everything out
	if err := compressor.Close(); err != nil {
		return nil, err
	}

	// Create a compressed message
	c := compress{
		Algo: lzwAlgo,
		Buf:  buf.Bytes(),
	}
	return encode(compressMsg, &c)
}

// decompressPayload is used to unpack an encoded compress{}
// message and return its payload uncompressed
func decompressPayload(msg []byte) ([]byte, error) {
	// Decode the message
	var c compress
	if err := decode(msg, &c); err != nil {
		return nil, err
	}
	return decompressBuffer(&c)
}

// decompressBuffer is used to decompress the buffer of
// a single compress message, handling multiple algorithms
func decompressBuffer(c *compress) ([]byte, error) {
	// Verify the algorithm
	if c.Algo != lzwAlgo {
		return nil, fmt.Errorf("Cannot decompress unknown algorithm %d", c.Algo)
	}

	// Create a uncompressor
	uncomp := lzw.NewReader(bytes.NewReader(c.Buf), lzw.LSB, lzwLitWidth)
	defer uncomp.Close()

	// Read all the data
	var b bytes.Buffer
	_, err := io.Copy(&b, uncomp)
	if err != nil {
		return nil, err
	}

	// Return the uncompressed bytes
	return b.Bytes(), nil
}
<commit_msg>more effective solution<commit_after>package memberlist

import (
	"bytes"
	"compress/lzw"
	"encoding/binary"
	"fmt"
	"io"
	"math"
	"math/rand"
	"net"
	"strings"
	"time"

	"github.com/hashicorp/go-msgpack/codec"
)

// pushPullScale is the minimum number of nodes
// before we start scaling the push/pull timing. The scale
// effect is the log2(Nodes) - log2(pushPullScale). This means
// that the 33rd node will cause us to double the interval,
// while the 65th will triple it.
const pushPullScaleThreshold = 32

/*
 * Contains an entry for each private block:
 * 10.0.0.0/8
 * 100.64.0.0/10
 * 127.0.0.0/8
 * 169.254.0.0/16
 * 172.16.0.0/12
 * 192.168.0.0/16
 */
var privateBlocks []*net.IPNet

var loopbackBlock *net.IPNet

const (
	// Constant litWidth 2-8
	lzwLitWidth = 8
)

func init() {
	// Seed the random number generator
	rand.Seed(time.Now().UnixNano())

	// Add each private block
	privateBlocks = make([]*net.IPNet, 6)

	_, block, err := net.ParseCIDR("10.0.0.0/8")
	if err != nil {
		panic(fmt.Sprintf("Bad cidr. Got %v", err))
	}
	privateBlocks[0] = block

	_, block, err = net.ParseCIDR("100.64.0.0/10")
	if err != nil {
		panic(fmt.Sprintf("Bad cidr. Got %v", err))
	}
	privateBlocks[1] = block

	_, block, err = net.ParseCIDR("127.0.0.0/8")
	if err != nil {
		panic(fmt.Sprintf("Bad cidr. Got %v", err))
	}
	privateBlocks[2] = block

	_, block, err = net.ParseCIDR("169.254.0.0/16")
	if err != nil {
		panic(fmt.Sprintf("Bad cidr. Got %v", err))
	}
	privateBlocks[3] = block

	_, block, err = net.ParseCIDR("172.16.0.0/12")
	if err != nil {
		panic(fmt.Sprintf("Bad cidr. Got %v", err))
	}
	privateBlocks[4] = block

	_, block, err = net.ParseCIDR("192.168.0.0/16")
	if err != nil {
		panic(fmt.Sprintf("Bad cidr. Got %v", err))
	}
	privateBlocks[5] = block

	_, block, err = net.ParseCIDR("127.0.0.0/8")
	if err != nil {
		panic(fmt.Sprintf("Bad cidr. Got %v", err))
	}
	loopbackBlock = block
}

// Decode reverses the encode operation on a byte slice input
func decode(buf []byte, out interface{}) error {
	r := bytes.NewReader(buf)
	hd := codec.MsgpackHandle{}
	dec := codec.NewDecoder(r, &hd)
	return dec.Decode(out)
}

// Encode writes an encoded object to a new bytes buffer
func encode(msgType messageType, in interface{}) (*bytes.Buffer, error) {
	buf := bytes.NewBuffer(nil)
	buf.WriteByte(uint8(msgType))
	hd := codec.MsgpackHandle{}
	enc := codec.NewEncoder(buf, &hd)
	err := enc.Encode(in)
	return buf, err
}

// GetPrivateIP returns the first private IP address found in a list of
// addresses.
func GetPrivateIP(addresses []net.Addr) (net.IP, error) {
	var candidates []net.IP

	// Find private IPv4 address
	for _, rawAddr := range addresses {
		var ip net.IP
		switch addr := rawAddr.(type) {
		case *net.IPAddr:
			ip = addr.IP
		case *net.IPNet:
			ip = addr.IP
		default:
			continue
		}

		if ip.To4() == nil {
			continue
		}
		if !IsPrivateIP(ip.String()) {
			continue
		}
		candidates = append(candidates, ip)
	}
	numIps := len(candidates)
	switch numIps {
	case 0:
		return nil, fmt.Errorf("No private IP address found")
	case 1:
		return candidates[0], nil
	default:
		return nil, fmt.Errorf("Multiple private IPs found. Please configure one.")
	}
}

// Returns a random offset between 0 and n
func randomOffset(n int) int {
	if n == 0 {
		return 0
	}
	return int(rand.Uint32() % uint32(n))
}

// suspicionTimeout computes the timeout that should be used when
// a node is suspected
func suspicionTimeout(suspicionMult, n int, interval time.Duration) time.Duration {
	nodeScale := math.Ceil(math.Log10(float64(n + 1)))
	timeout := time.Duration(suspicionMult) * time.Duration(nodeScale) * interval
	return timeout
}

// retransmitLimit computes the limit of retransmissions
func retransmitLimit(retransmitMult, n int) int {
	nodeScale := math.Ceil(math.Log10(float64(n + 1)))
	limit := retransmitMult * int(nodeScale)
	return limit
}

// shuffleNodes randomly shuffles the input nodes using the Fisher-Yates shuffle
func shuffleNodes(nodes []*nodeState) {
	n := len(nodes)
	for i := n - 1; i > 0; i-- {
		j := rand.Intn(i + 1)
		nodes[i], nodes[j] = nodes[j], nodes[i]
	}
}

// pushPushScale is used to scale the time interval at which push/pull
// syncs take place. It is used to prevent network saturation as the
// cluster size grows
func pushPullScale(interval time.Duration, n int) time.Duration {
	// Don't scale until we cross the threshold
	if n <= pushPullScaleThreshold {
		return interval
	}

	multiplier := math.Ceil(math.Log2(float64(n))-math.Log2(pushPullScaleThreshold)) + 1.0
	return time.Duration(multiplier) * interval
}

// moveDeadNodes moves all the nodes in the dead state
// to the end of the slice and returns the index of the first dead node.
func moveDeadNodes(nodes []*nodeState) int {
	numDead := 0
	n := len(nodes)
	for i := 0; i < n-numDead; i++ {
		if nodes[i].State != stateDead {
			continue
		}

		// Move this node to the end
		nodes[i], nodes[n-numDead-1] = nodes[n-numDead-1], nodes[i]
		numDead++
		i--
	}
	return n - numDead
}

// kRandomNodes is used to select up to k random nodes, excluding a given
// node and any non-alive nodes. It is possible that less than k nodes are returned.
func kRandomNodes(k int, excludes []string, nodes []*nodeState) []*nodeState {
	n := len(nodes)
	kNodes := make([]*nodeState, 0, k)
OUTER:
	// Probe up to 3*n times, with large n this is not necessary
	// since k << n, but with small n we want search to be
	// exhaustive
	for i := 0; i < 3*n && len(kNodes) < k; i++ {
		// Get random node
		idx := randomOffset(n)
		node := nodes[idx]

		// Exclude node if match
		for _, exclude := range excludes {
			if node.Name == exclude {
				continue OUTER
			}
		}

		// Exclude if not alive
		if node.State != stateAlive {
			continue
		}

		// Check if we have this node already
		for j := 0; j < len(kNodes); j++ {
			if node == kNodes[j] {
				continue OUTER
			}
		}

		// Append the node
		kNodes = append(kNodes, node)
	}
	return kNodes
}

// makeCompoundMessage takes a list of messages and generates
// a single compound message containing all of them
func makeCompoundMessage(msgs [][]byte) *bytes.Buffer {
	// Create a local buffer
	buf := bytes.NewBuffer(nil)

	// Write out the type
	buf.WriteByte(uint8(compoundMsg))

	// Write out the number of message
	buf.WriteByte(uint8(len(msgs)))

	// Add the message lengths
	for _, m := range msgs {
		binary.Write(buf, binary.BigEndian, uint16(len(m)))
	}

	// Append the messages
	for _, m := range msgs {
		buf.Write(m)
	}

	return buf
}

// decodeCompoundMessage splits a compound message and returns
// the slices of individual messages. Also returns the number
// of truncated messages and any potential error
func decodeCompoundMessage(buf []byte) (trunc int, parts [][]byte, err error) {
	if len(buf) < 1 {
		err = fmt.Errorf("missing compound length byte")
		return
	}
	numParts := uint8(buf[0])
	buf = buf[1:]

	// Check we have enough bytes
	if len(buf) < int(numParts*2) {
		err = fmt.Errorf("truncated len slice")
		return
	}

	// Decode the lengths
	lengths := make([]uint16, numParts)
	for i := 0; i < int(numParts); i++ {
		lengths[i] = binary.BigEndian.Uint16(buf[i*2 : i*2+2])
	}
	buf = buf[numParts*2:]

	// Split each message
	for idx, msgLen := range lengths {
		if len(buf) < int(msgLen) {
			trunc = int(numParts) - idx
			return
		}

		// Extract the slice, seek past on the buffer
		slice := buf[:msgLen]
		buf = buf[msgLen:]
		parts = append(parts, slice)
	}
	return
}

// Returns if the given IP is in a private block
func IsPrivateIP(ip_str string) bool {
	ip := net.ParseIP(ip_str)
	for _, priv := range privateBlocks {
		if priv.Contains(ip) {
			return true
		}
	}
	return false
}

// Returns if the given IP is in a loopback block
func isLoopbackIP(ip_str string) bool {
	ip := net.ParseIP(ip_str)
	return loopbackBlock.Contains(ip)
}

// Given a string of the form "host", "host:port",
// "ipv6::addr" or "[ipv6::address]:port",
// return true if the string includes a port.
func hasPort(s string) bool {
	last := strings.LastIndex(s, ":")
	if last == -1 {
		return false
	}
	if s[0] == '[' {
		return s[last-1] == ']'
	}
	return strings.Index(s, ":") == last
}

// compressPayload takes an opaque input buffer, compresses it
// and wraps it in a compress{} message that is encoded.
func compressPayload(inp []byte) (*bytes.Buffer, error) {
	var buf bytes.Buffer
	compressor := lzw.NewWriter(&buf, lzw.LSB, lzwLitWidth)

	_, err := compressor.Write(inp)
	if err != nil {
		return nil, err
	}

	// Ensure we flush everything out
	if err := compressor.Close(); err != nil {
		return nil, err
	}

	// Create a compressed message
	c := compress{
		Algo: lzwAlgo,
		Buf:  buf.Bytes(),
	}
	return encode(compressMsg, &c)
}

// decompressPayload is used to unpack an encoded compress{}
// message and return its payload uncompressed
func decompressPayload(msg []byte) ([]byte, error) {
	// Decode the message
	var c compress
	if err := decode(msg, &c); err != nil {
		return nil, err
	}
	return decompressBuffer(&c)
}

// decompressBuffer is used to decompress the buffer of
// a single compress message, handling multiple algorithms
func decompressBuffer(c *compress) ([]byte, error) {
	// Verify the algorithm
	if c.Algo != lzwAlgo {
		return nil, fmt.Errorf("Cannot decompress unknown algorithm %d", c.Algo)
	}

	// Create a uncompressor
	uncomp := lzw.NewReader(bytes.NewReader(c.Buf), lzw.LSB, lzwLitWidth)
	defer uncomp.Close()

	// Read all the data
	var b bytes.Buffer
	_, err := io.Copy(&b, uncomp)
	if err != nil {
		return nil, err
	}

	// Return the uncompressed bytes
	return b.Bytes(), nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Alex Browne.  All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

// File util.go contains miscellaneous utility functions used throughout
// the zoom library.

package zoom

import (
	"fmt"
	"github.com/dchest/uniuri"
	"math/cmplx"
	"math/rand"
	"reflect"
	"strconv"
	"time"
)

var (
	// delString is used as a suffix for string index tricks. This is a string which equals the ASCII
	// DEL character and is the highest possible value (in terms of codepoint, which is also
	// how redis sorts strings) for an ASCII character.
	delString = string([]byte{byte(127)})
	// nullString is used as a suffix for string index tricks. This is a string which equals the ASCII
	// NULL character and is the lowest possible value (in terms of codepoint, which is also
	// how redis sorts strings) for an ASCII character.
	nullString = string([]byte{byte(0)})
)

// Models converts in to []Model. It will panic if the underlying type
// of in is not a slice of some concrete type which implements Model.
func Models(in interface{}) []Model {
	typ := reflect.TypeOf(in)
	if !typeIsSliceOrArray(typ) {
		msg := fmt.Sprintf("zoom: panic in Models() - attempt to convert invalid type %T to []Model.\nArgument must be slice or array.", in)
		panic(msg)
	}
	elemTyp := typ.Elem()
	if !typeIsPointerToStruct(elemTyp) {
		msg := fmt.Sprintf("zoom: panic in Models() - attempt to convert invalid type %T to []Model.\nSlice or array must have elements of type pointer to struct.", in)
		panic(msg)
	}
	if !typeIsRegistered(elemTyp) {
		msg := fmt.Sprintf("zoom: panic in Models() - attempt to convert invalid type %T to []Model.\nType %s is not registered.", in, elemTyp)
		panic(msg)
	}
	val := reflect.ValueOf(in)
	length := val.Len()
	results := make([]Model, length)
	for i := 0; i < length; i++ {
		elemVal := val.Index(i)
		model, ok := elemVal.Interface().(Model)
		if !ok {
			msg := fmt.Sprintf("zoom: panic in Models() - cannot convert type %T to Model", elemVal.Interface())
			panic(msg)
		}
		results[i] = model
	}
	return results
}

// Interfaces converts in to []interface{}. It will panic if the underlying type
// of in is not a slice.
func Interfaces(in interface{}) []interface{} {
	val := reflect.ValueOf(in)
	length := val.Len()
	results := make([]interface{}, length)
	for i := 0; i < length; i++ {
		elemVal := val.Index(i)
		results[i] = elemVal.Interface()
	}
	return results
}

// indexOfStringSlice returns the index of s in strings, or
// -1 if a is not found in strings
func indexOfStringSlice(strings []string, s string) int {
	for i, b := range strings {
		if b == s {
			return i
		}
	}
	return -1
}

// stringSliceContains returns true iff strings contains s
func stringSliceContains(strings []string, s string) bool {
	return indexOfStringSlice(strings, s) != -1
}

// removeElementFromStringSlice removes elem from list and returns
// the new slice.
func removeElementFromStringSlice(list []string, elem string) []string {
	for i, e := range list {
		if e == elem {
			return append(list[:i], list[i+1:]...)
		}
	}
	return list
}

// compareAsStringSet compares expecteds and gots as if they were sets, i.e.,
// it checks if they contain the same values, regardless of order. It returns true
// and an empty string if expecteds and gots contain all the same values and false
// and a detailed message if they do not.
func compareAsStringSet(expecteds, gots []string) (bool, string) {
	// make sure everything in expecteds is also in gots
	for _, e := range expecteds {
		index := indexOfStringSlice(gots, e)
		if index == -1 {
			msg := fmt.Sprintf("Missing expected element: %v", e)
			return false, msg
		}
	}

	// make sure everything in gots is also in expecteds
	for _, g := range gots {
		index := indexOfStringSlice(expecteds, g)
		if index == -1 {
			msg := fmt.Sprintf("Found extra element: %v", g)
			return false, msg
		}
	}

	return true, "ok"
}

// typeIsSliceOrArray returns true iff typ is a slice or array
func typeIsSliceOrArray(typ reflect.Type) bool {
	k := typ.Kind()
	return (k == reflect.Slice || k == reflect.Array) && typ.Elem().Kind() != reflect.Uint8
}

// typeIsPointerToStruct returns true iff typ is a pointer to a struct
func typeIsPointerToStruct(typ reflect.Type) bool {
	return typ.Kind() == reflect.Ptr && typ.Elem().Kind() == reflect.Struct
}

// typeIsString returns true iff typ is a string or an array or slice of bytes
// (which is freely castable to a string)
func typeIsString(typ reflect.Type) bool {
	k := typ.Kind()
	return k == reflect.String || ((k == reflect.Slice || k == reflect.Array) && typ.Elem().Kind() == reflect.Uint8)
}

// typeIsNumeric returns true iff typ is one of the numeric primative types
func typeIsNumeric(typ reflect.Type) bool {
	k := typ.Kind()
	switch k {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Float32, reflect.Float64:
		return true
	default:
		return false
	}
}

// typeIsBool returns true iff typ is a bool
func typeIsBool(typ reflect.Type) bool {
	k := typ.Kind()
	return k == reflect.Bool
}

// typeIsPrimative returns true iff typ is a primative type, i.e. either a
// string, bool, or numeric type.
func typeIsPrimative(typ reflect.Type) bool {
	return typeIsString(typ) || typeIsNumeric(typ) || typeIsBool(typ)
}

// numericScore returns a float64 which is the score for val in a sorted set.
// If val is a pointer, it will keep dereferencing until it reaches the underlying
// value. It panics if val is not a numeric type or a pointer to a numeric type.
func numericScore(val reflect.Value) float64 {
	for val.Kind() == reflect.Ptr {
		val = val.Elem()
	}
	switch val.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		integer := val.Int()
		return float64(integer)
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		uinteger := val.Uint()
		return float64(uinteger)
	case reflect.Float32, reflect.Float64:
		return val.Float()
	default:
		msg := fmt.Sprintf("zoom: attempt to call numericScore on non-numeric type %s", val.Type().String())
		panic(msg)
	}
}

// boolScore returns an int which is the score for val in a sorted set.
// If val is a pointer, it will keep dereferencing until it reaches the underlying
// value. It panics if val is not a boolean or a pointer to a boolean.
func boolScore(val reflect.Value) int {
	for val.Kind() == reflect.Ptr {
		val = val.Elem()
	}
	if val.Kind() != reflect.Bool {
		msg := fmt.Sprintf("zoom: attempt to call boolScore on non-boolean type %s", val.Type().String())
		panic(msg)
	}
	return convertBoolToInt(val.Bool())
}

// convertBoolToInt converts a bool to an int using the following rule:
// false = 0
// true = 1
func convertBoolToInt(b bool) int {
	if b {
		return 1
	} else {
		return 0
	}
}

// modelIds returns the ids for models
func modelIds(models []Model) []string {
	results := make([]string, len(models))
	for i, m := range models {
		results[i] = m.ModelId()
	}
	return results
}

// generateRandomId generates a random string that is more or less
// garunteed to be unique. Used as Ids for records where an Id is
// not otherwise provided.
func generateRandomId() string {
	timeInt := time.Now().Unix()
	timeString := strconv.FormatInt(timeInt, 36)
	randomString := uniuri.NewLen(16)
	return randomString + timeString
}

// randomInt returns a pseudo-random int between the minimum and maximum
// possible values.
func randomInt() int {
	return rand.Int()
}

// randomString returns a random string of length 16
func randomString() string {
	return uniuri.NewLen(16)
}

// randomBool returns a random bool
func randomBool() bool {
	return rand.Int()%2 == 0
}

// randomFloat returns a random float64
func randomFloat() float64 {
	return rand.Float64()
}

// randomComplex returns a random complex128
func randomComplex() complex128 {
	return cmplx.Rect(randomFloat(), randomFloat())
}

// decrementString subtracts 1 to the last codepoint in s and returns the new string
// E.g. if the input string is "abc" the return will be "abb" because the codepoint
// for 'c' is 99, 99-1 = 98, and the codepoint 98 corresponds to 'b'.
func decrementString(s string) string {
	codepoints := []uint8(s)
	codepoints[len(codepoints)-1] = codepoints[len(codepoints)-1] + 1
	return string(codepoints)
}

// incrementString adds 1 to the last codepoint in s and returns the new string
// E.g. if the input string is "abc" the return will be "abd" because the codepoint
// for 'c' is 99, 99+1 = 100, and the codepoint 100 corresponds to 'd'.
func incrementString(s string) string {
	codepoints := []uint8(s)
	codepoints[len(codepoints)-1] = codepoints[len(codepoints)-1] + 1
	return string(codepoints)
}
<commit_msg>Drastically decrease the possiblity of collisions for randomly generated ids<commit_after>// Copyright 2015 Alex Browne.  All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

// File util.go contains miscellaneous utility functions used throughout
// the zoom library.

package zoom

import (
	"fmt"
	"github.com/dchest/uniuri"
	"github.com/tv42/base58"
	"hash/crc32"
	"math/big"
	"math/cmplx"
	"math/rand"
	"net"
	"reflect"
	"sync/atomic"
	"time"
)

var (
	// delString is used as a suffix for string index tricks. This is a string which equals the ASCII
	// DEL character and is the highest possible value (in terms of codepoint, which is also
	// how redis sorts strings) for an ASCII character.
	delString = string([]byte{byte(127)})
	// nullString is used as a suffix for string index tricks. This is a string which equals the ASCII
	// NULL character and is the lowest possible value (in terms of codepoint, which is also
	// how redis sorts strings) for an ASCII character.
	nullString = string([]byte{byte(0)})
	// hardwareId is a unique id for the current machine. Right now it uses the MAC address.
	hardwareId = ""
)

func init() {
	// Set chars to the 58 non-ambiguous characters use by base58 encoding
	uniuri.StdChars = []byte("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz")
}

// Models converts in to []Model. It will panic if the underlying type
// of in is not a slice of some concrete type which implements Model.
func Models(in interface{}) []Model {
	typ := reflect.TypeOf(in)
	if !typeIsSliceOrArray(typ) {
		msg := fmt.Sprintf("zoom: panic in Models() - attempt to convert invalid type %T to []Model.\nArgument must be slice or array.", in)
		panic(msg)
	}
	elemTyp := typ.Elem()
	if !typeIsPointerToStruct(elemTyp) {
		msg := fmt.Sprintf("zoom: panic in Models() - attempt to convert invalid type %T to []Model.\nSlice or array must have elements of type pointer to struct.", in)
		panic(msg)
	}
	if !typeIsRegistered(elemTyp) {
		msg := fmt.Sprintf("zoom: panic in Models() - attempt to convert invalid type %T to []Model.\nType %s is not registered.", in, elemTyp)
		panic(msg)
	}
	val := reflect.ValueOf(in)
	length := val.Len()
	results := make([]Model, length)
	for i := 0; i < length; i++ {
		elemVal := val.Index(i)
		model, ok := elemVal.Interface().(Model)
		if !ok {
			msg := fmt.Sprintf("zoom: panic in Models() - cannot convert type %T to Model", elemVal.Interface())
			panic(msg)
		}
		results[i] = model
	}
	return results
}

// Interfaces converts in to []interface{}. It will panic if the underlying type
// of in is not a slice.
func Interfaces(in interface{}) []interface{} {
	val := reflect.ValueOf(in)
	length := val.Len()
	results := make([]interface{}, length)
	for i := 0; i < length; i++ {
		elemVal := val.Index(i)
		results[i] = elemVal.Interface()
	}
	return results
}

// indexOfStringSlice returns the index of s in strings, or
// -1 if a is not found in strings
func indexOfStringSlice(strings []string, s string) int {
	for i, b := range strings {
		if b == s {
			return i
		}
	}
	return -1
}

// stringSliceContains returns true iff strings contains s
func stringSliceContains(strings []string, s string) bool {
	return indexOfStringSlice(strings, s) != -1
}

// removeElementFromStringSlice removes elem from list and returns
// the new slice.
func removeElementFromStringSlice(list []string, elem string) []string {
	for i, e := range list {
		if e == elem {
			return append(list[:i], list[i+1:]...)
		}
	}
	return list
}

// compareAsStringSet compares expecteds and gots as if they were sets, i.e.,
// it checks if they contain the same values, regardless of order. It returns true
// and an empty string if expecteds and gots contain all the same values and false
// and a detailed message if they do not.
func compareAsStringSet(expecteds, gots []string) (bool, string) {
	// make sure everything in expecteds is also in gots
	for _, e := range expecteds {
		index := indexOfStringSlice(gots, e)
		if index == -1 {
			msg := fmt.Sprintf("Missing expected element: %v", e)
			return false, msg
		}
	}

	// make sure everything in gots is also in expecteds
	for _, g := range gots {
		index := indexOfStringSlice(expecteds, g)
		if index == -1 {
			msg := fmt.Sprintf("Found extra element: %v", g)
			return false, msg
		}
	}

	return true, "ok"
}

// typeIsSliceOrArray returns true iff typ is a slice or array
func typeIsSliceOrArray(typ reflect.Type) bool {
	k := typ.Kind()
	return (k == reflect.Slice || k == reflect.Array) && typ.Elem().Kind() != reflect.Uint8
}

// typeIsPointerToStruct returns true iff typ is a pointer to a struct
func typeIsPointerToStruct(typ reflect.Type) bool {
	return typ.Kind() == reflect.Ptr && typ.Elem().Kind() == reflect.Struct
}

// typeIsString returns true iff typ is a string or an array or slice of bytes
// (which is freely castable to a string)
func typeIsString(typ reflect.Type) bool {
	k := typ.Kind()
	return k == reflect.String || ((k == reflect.Slice || k == reflect.Array) && typ.Elem().Kind() == reflect.Uint8)
}

// typeIsNumeric returns true iff typ is one of the numeric primative types
func typeIsNumeric(typ reflect.Type) bool {
	k := typ.Kind()
	switch k {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Float32, reflect.Float64:
		return true
	default:
		return false
	}
}

// typeIsBool returns true iff typ is a bool
func typeIsBool(typ reflect.Type) bool {
	k := typ.Kind()
	return k == reflect.Bool
}

// typeIsPrimative returns true iff typ is a primative type, i.e. either a
// string, bool, or numeric type.
func typeIsPrimative(typ reflect.Type) bool {
	return typeIsString(typ) || typeIsNumeric(typ) || typeIsBool(typ)
}

// numericScore returns a float64 which is the score for val in a sorted set.
// If val is a pointer, it will keep dereferencing until it reaches the underlying
// value. It panics if val is not a numeric type or a pointer to a numeric type.
func numericScore(val reflect.Value) float64 {
	for val.Kind() == reflect.Ptr {
		val = val.Elem()
	}
	switch val.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		integer := val.Int()
		return float64(integer)
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		uinteger := val.Uint()
		return float64(uinteger)
	case reflect.Float32, reflect.Float64:
		return val.Float()
	default:
		msg := fmt.Sprintf("zoom: attempt to call numericScore on non-numeric type %s", val.Type().String())
		panic(msg)
	}
}

// boolScore returns an int which is the score for val in a sorted set.
// If val is a pointer, it will keep dereferencing until it reaches the underlying
// value. It panics if val is not a boolean or a pointer to a boolean.
func boolScore(val reflect.Value) int {
	for val.Kind() == reflect.Ptr {
		val = val.Elem()
	}
	if val.Kind() != reflect.Bool {
		msg := fmt.Sprintf("zoom: attempt to call boolScore on non-boolean type %s", val.Type().String())
		panic(msg)
	}
	return convertBoolToInt(val.Bool())
}

// convertBoolToInt converts a bool to an int using the following rule:
// false = 0
// true = 1
func convertBoolToInt(b bool) int {
	if b {
		return 1
	} else {
		return 0
	}
}

// modelIds returns the ids for models
func modelIds(models []Model) []string {
	results := make([]string, len(models))
	for i, m := range models {
		results[i] = m.ModelId()
	}
	return results
}

// generateRandomId generates a pseudo-random string that is highly likely to be unique.
// The string is base58 encoded and consists of 4 components:
//   1. The current UTC unix time with second precision
//   2. An atomic counter which is always 4 characters long and cycles
//      through the range of 0 to 11,316,496
//   3. A unique hardware identifier based on the MAC address of the
//      current machine
//   4. A pseudo-randomly generated sequence of 6 characters
func generateRandomId() string {
	// str := getTimeString() + getAtomicCounter() + getHardwareId() + uniuri.NewLen(4)
	// fmt.Println(str)
	// return str
	return getTimeString() + getAtomicCounter() + getHardwareId() + uniuri.NewLen(6)
}

// getTimeString returns the current UTC unix time with second precision encoded
// with base58 encoding.
func getTimeString() string {
	timeInt := time.Now().UTC().Unix()
	timeBytes := base58.EncodeBig(nil, big.NewInt(timeInt))
	return string(timeBytes)
}

// getHardwareId returns a unique identifier for the current machine. It does this
// by iterating through the network interfaces of the machine and picking the first
// one that has a non-empty hardware (MAC) address. Then it takes the crc32 checksum
// of the MAC address and encodes it in base58 encoding. getHardwareId caches results,
// so subsequent calls will return the previously calculated result. If no MAC address
// could be found, the function will use "0" as the MAC address. This is not ideal, but
// generateRandomId uses other means to try and avoid collisions.
func getHardwareId() string {
	if hardwareId != "" {
		return hardwareId
	}
	address := ""
	inters, err := net.Interfaces()
	if err == nil {
		for _, inter := range inters {
			if inter.HardwareAddr.String() != "" {
				address = inter.HardwareAddr.String()
				break
			}
		}
	}
	if address == "" {
		address = "0"
	}
	check32 := crc32.ChecksumIEEE([]byte(address))
	id58 := base58.EncodeBig(nil, big.NewInt(int64(check32)))
	hardwareId = string(id58)
	return hardwareId
}

var counter int32 = 0

// getAtomicCounter returns the base58 encoding of a counter which cycles through
// the values in the range 0 to 11,316,496. This is the range that can be represented
// with 4 base58 characters. The returned result will be padded with zeros such that
// it is always 4 characters long.
func getAtomicCounter() string {
	atomic.AddInt32(&counter, 1)
	if counter > 58*58*58*58-1 {
		// Reset the counter if we're beyond what we
		// can represent with 4 base58 characters
		atomic.StoreInt32(&counter, 0)
	}
	counterBytes := base58.EncodeBig(nil, big.NewInt(int64(counter)))
	counterStr := string(counterBytes)
	switch len(counterStr) {
	case 0:
		return "0000"
	case 1:
		return "000" + counterStr
	case 2:
		return "00" + counterStr
	case 3:
		return "0" + counterStr
	default:
		return counterStr[0:4]
	}
}

// randomInt returns a pseudo-random int between the minimum and maximum
// possible values.
func randomInt() int {
	return rand.Int()
}

// randomString returns a random string of length 16
func randomString() string {
	return uniuri.NewLen(16)
}

// randomBool returns a random bool
func randomBool() bool {
	return rand.Int()%2 == 0
}

// randomFloat returns a random float64
func randomFloat() float64 {
	return rand.Float64()
}

// randomComplex returns a random complex128
func randomComplex() complex128 {
	return cmplx.Rect(randomFloat(), randomFloat())
}

// decrementString subtracts 1 to the last codepoint in s and returns the new string
// E.g. if the input string is "abc" the return will be "abb" because the codepoint
// for 'c' is 99, 99-1 = 98, and the codepoint 98 corresponds to 'b'.
func decrementString(s string) string {
	codepoints := []uint8(s)
	codepoints[len(codepoints)-1] = codepoints[len(codepoints)-1] + 1
	return string(codepoints)
}

// incrementString adds 1 to the last codepoint in s and returns the new string
// E.g. if the input string is "abc" the return will be "abd" because the codepoint
// for 'c' is 99, 99+1 = 100, and the codepoint 100 corresponds to 'd'.
func incrementString(s string) string {
	codepoints := []uint8(s)
	codepoints[len(codepoints)-1] = codepoints[len(codepoints)-1] + 1
	return string(codepoints)
}
<|endoftext|>


<commit_before>// Package str provides some routines to convert an unsigned integer value to
// its English textual representation. This facilitates database testing by
// supporting the generation of a large number of records with generally
// unrelated sort orders.
package str

import (
	"bytes"
	"fmt"
	"strings"
)

// Delimit converts 'ABCDEFG' to, for example, 'A,BCD,EFG'
func Delimit(str string, sepstr string, sepcount int) string {
	pos := len(str) - sepcount
	for pos > 0 {
		str = str[:pos] + sepstr + str[pos:]
		pos = pos - sepcount
	}
	return str
}

var lookup = [...]string{
	"",           // 0
	"eight ",     // 1
	"eighteen ",  // 2
	"eighty ",    // 3
	"eleven ",    // 4
	"fifteen ",   // 5
	"fifty ",     // 6
	"five ",      // 7
	"forty ",     // 8
	"four ",      // 9
	"fourteen ",  // 10
	"hundred ",   // 11
	"million ",   // 12
	"nine ",      // 13
	"nineteen ",  // 14
	"ninety ",    // 15
	"one ",       // 16
	"seven ",     // 17
	"seventeen ", // 18
	"seventy ",   // 19
	"six ",       // 20
	"sixteen ",   // 21
	"sixty ",     // 22
	"ten ",       // 23
	"ten ",       // 24
	"thirteen ",  // 25
	"thirty ",    // 26
	"thousand ",  // 27
	"three ",     // 28
	"twelve ",    // 29
	"twenty ",    // 30
	"two ",       // 31
	"zero ",      // 32
}

// QuantityDecode returns the textual expansion of the byte sequence specified
// by sl. See QuantityEncode for more details.
func QuantityDecode(sl []byte) string {
	var buf bytes.Buffer
	for _, c := range sl {
		if c < 33 {
			buf.WriteString(lookup[c])
		}
	}
	return strings.TrimSpace(buf.String())
}

func quantityByte(buf *bytes.Buffer, val uint) {
	var numList = [...]byte{0, 16, 31, 28, 9, 7, 20, 17, 1, 13, 24, 4, 29,
		25, 10, 5, 21, 18, 2, 14}
	var tenList = [...]byte{23, 30, 26, 8, 6, 22, 19, 3, 15}

	if val >= 1000000 {
		quantityByte(buf, val/1000000)
		buf.WriteByte(12)
		quantityByte(buf, val%1000000)
	} else if val >= 1000 {
		quantityByte(buf, val/1000)
		buf.WriteByte(27)
		quantityByte(buf, val%1000)
	} else if val >= 100 {
		quantityByte(buf, val/100)
		buf.WriteByte(11)
		quantityByte(buf, val%100)
	} else if val >= 20 {
		buf.WriteByte(tenList[val/10-1])
		quantityByte(buf, val%10)
	} else {
		buf.WriteByte(numList[val])
	}
}

// QuantityEncode returns the English text equivalent, in byte-encoded form, of
// the number specified by val. For example, if val is 21, the return value is
// "twenty one". Sorting encoded values for various values produces the same
// order as sorting decoded values. QuantityDecode() decodes the packed byte
// sequence into English words. val must be a value less than or equal to
// 999,999,999.
func QuantityEncode(val uint) (sl []byte, err error) {
	const limit = 1000000000
	if val < limit {
		var buf bytes.Buffer
		if val > 0 {
			quantityByte(&buf, val)
		} else {
			buf.WriteByte(32)
		}
		sl = buf.Bytes()
	} else {
		err = fmt.Errorf("expecting val (%d) to be less than %d", val, limit)
	}
	return
}
func quantity(buf *bytes.Buffer, val uint) {
	var numList = [...]string{"", "one ", "two ", "three ", "four ",
		"five ", "six ", "seven ", "eight ", "nine ", "ten ", "eleven ", "twelve ",
		"thirteen ", "fourteen ", "fifteen ", "sixteen ", "seventeen ", "eighteen ",
		"nineteen "}
	var tenList = [...]string{"ten ", "twenty ", "thirty ", "forty ",
		"fifty ", "sixty ", "seventy ", "eighty ", "ninety "}

	if val >= 1000000 {
		quantity(buf, val/1000000)
		fmt.Fprint(buf, "million ")
		quantity(buf, val%1000000)
	} else if val >= 1000 {
		quantity(buf, val/1000)
		fmt.Fprint(buf, "thousand ")
		quantity(buf, val%1000)
	} else if val >= 100 {
		quantity(buf, val/100)
		fmt.Fprint(buf, "hundred ")
		quantity(buf, val%100)
	} else if val >= 20 {
		fmt.Fprint(buf, tenList[val/10-1])
		quantity(buf, val%10)
	} else {
		fmt.Fprint(buf, numList[val])
	}
}

// Quantity returns the English text equivalent of the number specified by
// val. For example, if val is 35 the return value is "thirty five".
func Quantity(val uint) string {
	if val == 0 {
		return "zero"
	} else if val >= 1000000000 {
		return fmt.Sprintf("%d", val)
	} else {
		var buf bytes.Buffer
		quantity(&buf, val)
		return strings.TrimSpace(buf.String())
	}
}
<commit_msg>Clarify documentation for QuantityEncode<commit_after>// Package str provides some routines to convert an unsigned integer value to
// its English textual representation. This facilitates database testing by
// supporting the generation of a large number of records with generally
// unrelated sort orders.
package str

import (
	"bytes"
	"fmt"
	"strings"
)

// Delimit converts 'ABCDEFG' to, for example, 'A,BCD,EFG'
func Delimit(str string, sepstr string, sepcount int) string {
	pos := len(str) - sepcount
	for pos > 0 {
		str = str[:pos] + sepstr + str[pos:]
		pos = pos - sepcount
	}
	return str
}

var lookup = [...]string{
	"",           // 0
	"eight ",     // 1
	"eighteen ",  // 2
	"eighty ",    // 3
	"eleven ",    // 4
	"fifteen ",   // 5
	"fifty ",     // 6
	"five ",      // 7
	"forty ",     // 8
	"four ",      // 9
	"fourteen ",  // 10
	"hundred ",   // 11
	"million ",   // 12
	"nine ",      // 13
	"nineteen ",  // 14
	"ninety ",    // 15
	"one ",       // 16
	"seven ",     // 17
	"seventeen ", // 18
	"seventy ",   // 19
	"six ",       // 20
	"sixteen ",   // 21
	"sixty ",     // 22
	"ten ",       // 23
	"ten ",       // 24
	"thirteen ",  // 25
	"thirty ",    // 26
	"thousand ",  // 27
	"three ",     // 28
	"twelve ",    // 29
	"twenty ",    // 30
	"two ",       // 31
	"zero ",      // 32
}

// QuantityDecode returns the textual expansion of the byte sequence specified
// by sl. See QuantityEncode for more details.
func QuantityDecode(sl []byte) string {
	var buf bytes.Buffer
	for _, c := range sl {
		if c < 33 {
			buf.WriteString(lookup[c])
		}
	}
	return strings.TrimSpace(buf.String())
}

func quantityByte(buf *bytes.Buffer, val uint) {
	var numList = [...]byte{0, 16, 31, 28, 9, 7, 20, 17, 1, 13, 24, 4, 29,
		25, 10, 5, 21, 18, 2, 14}
	var tenList = [...]byte{23, 30, 26, 8, 6, 22, 19, 3, 15}

	if val >= 1000000 {
		quantityByte(buf, val/1000000)
		buf.WriteByte(12)
		quantityByte(buf, val%1000000)
	} else if val >= 1000 {
		quantityByte(buf, val/1000)
		buf.WriteByte(27)
		quantityByte(buf, val%1000)
	} else if val >= 100 {
		quantityByte(buf, val/100)
		buf.WriteByte(11)
		quantityByte(buf, val%100)
	} else if val >= 20 {
		buf.WriteByte(tenList[val/10-1])
		quantityByte(buf, val%10)
	} else {
		buf.WriteByte(numList[val])
	}
}

// QuantityEncode returns the English text equivalent, in byte-encoded form, of
// the number specified by val. For example, if val is 21, the return value is
// a compact byte slice that, when unencoded with QuantityDecode(), is "twenty
// one". Sorting encoded values for various values produces the same order as
// sorting decoded values. QuantityDecode() decodes the packed byte sequence
// into English words. val must be a value less than or equal to 999,999,999.
func QuantityEncode(val uint) (sl []byte, err error) {
	const limit = 1000000000
	if val < limit {
		var buf bytes.Buffer
		if val > 0 {
			quantityByte(&buf, val)
		} else {
			buf.WriteByte(32)
		}
		sl = buf.Bytes()
	} else {
		err = fmt.Errorf("expecting val (%d) to be less than %d", val, limit)
	}
	return
}
func quantity(buf *bytes.Buffer, val uint) {
	var numList = [...]string{"", "one ", "two ", "three ", "four ",
		"five ", "six ", "seven ", "eight ", "nine ", "ten ", "eleven ", "twelve ",
		"thirteen ", "fourteen ", "fifteen ", "sixteen ", "seventeen ", "eighteen ",
		"nineteen "}
	var tenList = [...]string{"ten ", "twenty ", "thirty ", "forty ",
		"fifty ", "sixty ", "seventy ", "eighty ", "ninety "}

	if val >= 1000000 {
		quantity(buf, val/1000000)
		fmt.Fprint(buf, "million ")
		quantity(buf, val%1000000)
	} else if val >= 1000 {
		quantity(buf, val/1000)
		fmt.Fprint(buf, "thousand ")
		quantity(buf, val%1000)
	} else if val >= 100 {
		quantity(buf, val/100)
		fmt.Fprint(buf, "hundred ")
		quantity(buf, val%100)
	} else if val >= 20 {
		fmt.Fprint(buf, tenList[val/10-1])
		quantity(buf, val%10)
	} else {
		fmt.Fprint(buf, numList[val])
	}
}

// Quantity returns the English text equivalent of the number specified by
// val. For example, if val is 35 the return value is "thirty five".
func Quantity(val uint) string {
	if val == 0 {
		return "zero"
	} else if val >= 1000000000 {
		return fmt.Sprintf("%d", val)
	} else {
		var buf bytes.Buffer
		quantity(&buf, val)
		return strings.TrimSpace(buf.String())
	}
}
<|endoftext|>


<commit_before>package interfaces

import (
	"time"
)

type WriteRequest struct {
	Timeseries []*Timeseries
}

type Operation int

const (
	EqualOperation Operation = iota
	NotEqualOperation
	GreaterThanOperation
	GreaterThanOrEqualOperation
	LessThanOperation
	LessThanOrEqualOperation
)

type Condition interface {
	IsCondition()
}

type CombineOperation int

// definition of combining operators with order of precedence
const (
	NotOperation CombineOperation = iota // First has to be nil
	AndOperation
	OrOperation
)

type CombiningCondition struct {
	First     *Condition
	CombineOp CombineOperation
	Second    *Condition
}

type ComparisonCondition struct {
	FieldName string
	Op        Operation
	Value     interface{}
}

// TODO: applying functions and joining time series
type ReadRequest struct {
	Timeseries   string
	IsRegex      bool // is the timeseries name a regex?
	StartTime    time.Time
	EndTime      time.Time
	IsContinuous bool
	Conditions   []*Condition
}

type StorageEngineProcessingI interface {
	WritePoints(request *WriteRequest) error
	ReadPoints(request *ReadRequest, yield func(pts []*Point) error) error
}

type StorageEngineConsensusI interface {
	// TODO: figure out the requirements of this interface. Probably the following
	// 1. Transfer part(s) of the ring to other node(s)
	// 2. Give up ownership of part(s) of the ring
	// 3. Take ownership of part(s) of the ring
}
<commit_msg>Fix the storage engine interface to build with things in protocol<commit_after>package interfaces

import (
	"protocol"
	"time"
)

type WriteRequest struct {
	Timeseries []*protocol.Series
}

type Operation int

const (
	EqualOperation Operation = iota
	NotEqualOperation
	GreaterThanOperation
	GreaterThanOrEqualOperation
	LessThanOperation
	LessThanOrEqualOperation
)

type Condition interface {
	IsCondition()
}

type CombineOperation int

// definition of combining operators with order of precedence
const (
	NotOperation CombineOperation = iota // First has to be nil
	AndOperation
	OrOperation
)

type CombiningCondition struct {
	First     *Condition
	CombineOp CombineOperation
	Second    *Condition
}

type ComparisonCondition struct {
	FieldName string
	Op        Operation
	Value     interface{}
}

// TODO: applying functions and joining time series
type ReadRequest struct {
	Timeseries   string
	IsRegex      bool // is the timeseries name a regex?
	StartTime    time.Time
	EndTime      time.Time
	IsContinuous bool
	Conditions   []*Condition
}

type StorageEngineProcessingI interface {
	WritePoints(request *WriteRequest) error
	ReadPoints(request *ReadRequest, yield func(pts []*protocol.Point) error) error
}

type StorageEngineConsensusI interface {
	// TODO: figure out the requirements of this interface. Probably the following
	// 1. Transfer part(s) of the ring to other node(s)
	// 2. Give up ownership of part(s) of the ring
	// 3. Take ownership of part(s) of the ring
}
<|endoftext|>


<commit_before>package uncertainty

import (
	"errors"
	"math"

	"github.com/ready-steady/infinity"
	"github.com/ready-steady/linear/matrix"
	"github.com/ready-steady/probability/distribution"
	"github.com/turing-complete/laboratory/src/internal/config"
	"github.com/turing-complete/laboratory/src/internal/support"
	"github.com/turing-complete/laboratory/src/internal/system"

	scorrelation "github.com/ready-steady/statistics/correlation"
	icorrelation "github.com/turing-complete/laboratory/src/internal/correlation"
	idistribution "github.com/turing-complete/laboratory/src/internal/distribution"
)

var (
	epsilon  = math.Nextafter(1.0, 2.0) - 1.0
	gaussian = distribution.NewGaussian(0.0, 1.0)
)

type base struct {
	tasks []uint
	lower []float64
	upper []float64

	nt uint
	nu uint
	nz uint

	correlation *correlation
	marginals   []distribution.Continuous
}

type correlation struct {
	R []float64
	C []float64 // x = C * z
	D []float64 // z = D * x
	P []float64 // R^(-1) - I

	detR float64
}

func newBase(system *system.System, reference []float64,
	config *config.Uncertainty) (*base, error) {

	nt := uint(len(reference))

	tasks, err := support.ParseNaturalIndex(config.Tasks, 0, nt-1)
	if err != nil {
		return nil, err
	}

	nu := uint(len(tasks))

	lower := make([]float64, nt)
	upper := make([]float64, nt)

	copy(lower, reference)
	copy(upper, reference)

	for _, tid := range tasks {
		lower[tid] -= config.Deviation * reference[tid]
		upper[tid] += config.Deviation * reference[tid]
	}

	correlation, err := correlate(system, config, tasks)
	if err != nil {
		return nil, err
	}

	nz := uint(len(correlation.C)) / nu

	marginal, err := idistribution.Parse(config.Distribution)
	if err != nil {
		return nil, err
	}

	marginals := make([]distribution.Continuous, nu)
	for i := uint(0); i < nu; i++ {
		marginals[i] = marginal
	}

	return &base{
		tasks: tasks,
		lower: lower,
		upper: upper,

		nt: nt,
		nu: nu,
		nz: nz,

		correlation: correlation,
		marginals:   marginals,
	}, nil
}

func (self *base) Mapping() (uint, uint) {
	return self.nz, self.nt
}

func (self *base) Evaluate(ω []float64) float64 {
	nu, nz := self.nu, self.nz
	lower, upper := self.lower, self.upper

	if nu != nz {
		panic("model-order reduction is not supported")
	}

	amplitude := 1.0

	u := make([]float64, nu)
	for i, tid := range self.tasks {
		ω := (ω[tid] - lower[tid]) / (upper[tid] - lower[tid])
		u[i] = gaussian.Invert(self.marginals[i].Cumulate(ω))
		amplitude *= self.marginals[i].Weigh(ω)
	}

	exponent := -0.5 * infinity.Quadratic(self.correlation.P, u, nu)
	normalization := math.Sqrt(self.correlation.detR)

	return amplitude * math.Exp(exponent) / normalization
}

func (self *base) Forward(ω []float64) []float64 {
	nu, nz := self.nu, self.nz
	lower, upper := self.lower, self.upper

	z := make([]float64, nz)
	u := make([]float64, nu)

	// Dependent desired to dependent uniform
	for i, tid := range self.tasks {
		ω := (ω[tid] - lower[tid]) / (upper[tid] - lower[tid])
		u[i] = self.marginals[i].Cumulate(ω)
	}

	// Dependent uniform to dependent Gaussian
	for i := range u {
		u[i] = gaussian.Invert(u[i])
	}

	// Dependent Gaussian to independent Gaussian
	n := infinity.Linear(self.correlation.D, u, nz, nu)

	// Independent Gaussian to independent uniform
	for i := range n {
		z[i] = gaussian.Cumulate(n[i])
	}

	return z
}

func (self *base) Backward(z []float64) []float64 {
	nu, nz := self.nu, self.nz
	lower, upper := self.lower, self.upper

	ω := append([]float64(nil), lower...)
	n := make([]float64, nz)

	// Independent uniform to independent Gaussian
	for i := range n {
		n[i] = gaussian.Invert(z[i])
	}

	// Independent Gaussian to dependent Gaussian
	u := infinity.Linear(self.correlation.C, n, nu, nz)

	// Dependent Gaussian to dependent uniform
	for i := range u {
		u[i] = gaussian.Cumulate(u[i])
	}

	// Dependent uniform to dependent desired
	for i, tid := range self.tasks {
		ω[tid] += (upper[tid] - lower[tid]) * self.marginals[i].Invert(u[i])
	}

	return ω
}

func correlate(system *system.System, config *config.Uncertainty,
	tasks []uint) (*correlation, error) {

	ε := math.Sqrt(epsilon)

	nu := uint(len(tasks))

	if config.Correlation == 0.0 {
		return &correlation{
			R: matrix.Identity(nu),
			C: matrix.Identity(nu),
			D: matrix.Identity(nu),
			P: make([]float64, nu*nu),

			detR: 1.0,
		}, nil
	}
	if config.Correlation < 0.0 {
		return nil, errors.New("the correlation length should be nonnegative")
	}
	if config.Variance <= 0.0 {
		return nil, errors.New("the variance threshold should be positive")
	}

	R := icorrelation.Compute(system.Application, tasks, config.Correlation)

	C, D, U, Λ, err := scorrelation.Decompose(R, nu, config.Variance, ε)
	if err != nil {
		return nil, err
	}

	detR := 1.0
	for _, λ := range Λ {
		if λ <= 0.0 {
			return nil, errors.New("the corelation matrix is invalid or singular")
		}
		detR *= λ
	}

	P, err := invert(U, Λ, nu)
	if err != nil {
		return nil, err
	}
	for i := uint(0); i < nu; i++ {
		P[i*nu+i] -= 1.0
	}

	return &correlation{
		R: R,
		C: C,
		D: D,
		P: P,

		detR: detR,
	}, nil
}
<commit_msg>i/uncertainty: rename correlation to copula<commit_after>package uncertainty

import (
	"errors"
	"math"

	"github.com/ready-steady/infinity"
	"github.com/ready-steady/linear/matrix"
	"github.com/ready-steady/probability/distribution"
	"github.com/ready-steady/statistics/correlation"
	"github.com/turing-complete/laboratory/src/internal/config"
	"github.com/turing-complete/laboratory/src/internal/support"
	"github.com/turing-complete/laboratory/src/internal/system"

	icorrelation "github.com/turing-complete/laboratory/src/internal/correlation"
	idistribution "github.com/turing-complete/laboratory/src/internal/distribution"
)

var (
	epsilon  = math.Nextafter(1.0, 2.0) - 1.0
	gaussian = distribution.NewGaussian(0.0, 1.0)
)

type base struct {
	tasks []uint
	lower []float64
	upper []float64

	nt uint
	nu uint
	nz uint

	copula    *copula
	marginals []distribution.Continuous
}

// x(z) = F^(-1)(u(z))
// u(z) = Φ(C * Φ^(-1)(z))
//
// z(x) = Φ(D * Φ^(-1)(u(x)))
// u(x) = F(x)
//
// f(u) = exp(-0.5 * Φ^(-1)(u)^T * (R^(-1) - I) * Φ^(-1)(u)) / sqrt(det(R))
//      = exp(-0.5 * Φ^(-1)(u)^T * P * Φ^(-1)(u)) / N
//
// f(x) = prod(f(x)) * f(F(x))
type copula struct {
	R []float64
	C []float64
	D []float64
	P []float64
	N float64
}

func newBase(system *system.System, reference []float64,
	config *config.Uncertainty) (*base, error) {

	nt := uint(len(reference))

	tasks, err := support.ParseNaturalIndex(config.Tasks, 0, nt-1)
	if err != nil {
		return nil, err
	}

	nu := uint(len(tasks))

	lower := make([]float64, nt)
	upper := make([]float64, nt)

	copy(lower, reference)
	copy(upper, reference)

	for _, tid := range tasks {
		lower[tid] -= config.Deviation * reference[tid]
		upper[tid] += config.Deviation * reference[tid]
	}

	copula, err := correlate(system, config, tasks)
	if err != nil {
		return nil, err
	}

	nz := uint(len(copula.C)) / nu

	marginal, err := idistribution.Parse(config.Distribution)
	if err != nil {
		return nil, err
	}

	marginals := make([]distribution.Continuous, nu)
	for i := uint(0); i < nu; i++ {
		marginals[i] = marginal
	}

	return &base{
		tasks: tasks,
		lower: lower,
		upper: upper,

		nt: nt,
		nu: nu,
		nz: nz,

		copula:    copula,
		marginals: marginals,
	}, nil
}

func (self *base) Mapping() (uint, uint) {
	return self.nz, self.nt
}

func (self *base) Evaluate(ω []float64) float64 {
	nu, nz := self.nu, self.nz
	lower, upper := self.lower, self.upper

	if nu != nz {
		panic("model-order reduction is not supported")
	}

	amplitude, u := 1.0, make([]float64, nu)
	for i, tid := range self.tasks {
		ω := (ω[tid] - lower[tid]) / (upper[tid] - lower[tid])
		u[i] = gaussian.Invert(self.marginals[i].Cumulate(ω))
		amplitude *= self.marginals[i].Weigh(ω)
	}

	exponent := -0.5 * infinity.Quadratic(self.copula.P, u, nu)

	return amplitude * math.Exp(exponent) / self.copula.N
}

func (self *base) Forward(ω []float64) []float64 {
	nu, nz := self.nu, self.nz
	lower, upper := self.lower, self.upper

	z := make([]float64, nz)
	u := make([]float64, nu)

	// Dependent desired to dependent uniform
	for i, tid := range self.tasks {
		ω := (ω[tid] - lower[tid]) / (upper[tid] - lower[tid])
		u[i] = self.marginals[i].Cumulate(ω)
	}

	// Dependent uniform to dependent Gaussian
	for i := range u {
		u[i] = gaussian.Invert(u[i])
	}

	// Dependent Gaussian to independent Gaussian
	n := infinity.Linear(self.copula.D, u, nz, nu)

	// Independent Gaussian to independent uniform
	for i := range n {
		z[i] = gaussian.Cumulate(n[i])
	}

	return z
}

func (self *base) Backward(z []float64) []float64 {
	nu, nz := self.nu, self.nz
	lower, upper := self.lower, self.upper

	ω := append([]float64(nil), lower...)
	n := make([]float64, nz)

	// Independent uniform to independent Gaussian
	for i := range n {
		n[i] = gaussian.Invert(z[i])
	}

	// Independent Gaussian to dependent Gaussian
	u := infinity.Linear(self.copula.C, n, nu, nz)

	// Dependent Gaussian to dependent uniform
	for i := range u {
		u[i] = gaussian.Cumulate(u[i])
	}

	// Dependent uniform to dependent desired
	for i, tid := range self.tasks {
		ω[tid] += (upper[tid] - lower[tid]) * self.marginals[i].Invert(u[i])
	}

	return ω
}

func correlate(system *system.System, config *config.Uncertainty,
	tasks []uint) (*copula, error) {

	ε := math.Sqrt(epsilon)

	nu := uint(len(tasks))

	if config.Correlation == 0.0 {
		return &copula{
			R: matrix.Identity(nu),
			C: matrix.Identity(nu),
			D: matrix.Identity(nu),
			P: make([]float64, nu*nu),
			N: 1.0,
		}, nil
	}
	if config.Correlation < 0.0 {
		return nil, errors.New("the correlation length should be nonnegative")
	}
	if config.Variance <= 0.0 {
		return nil, errors.New("the variance threshold should be positive")
	}

	R := icorrelation.Compute(system.Application, tasks, config.Correlation)

	C, D, U, Λ, err := correlation.Decompose(R, nu, config.Variance, ε)
	if err != nil {
		return nil, err
	}

	detR := 1.0
	for _, λ := range Λ {
		if λ <= 0.0 {
			return nil, errors.New("the corelation matrix is invalid or singular")
		}
		detR *= λ
	}

	P, err := invert(U, Λ, nu)
	if err != nil {
		return nil, err
	}
	for i := uint(0); i < nu; i++ {
		P[i*nu+i] -= 1.0
	}

	return &copula{
		R: R,
		C: C,
		D: D,
		P: P,
		N: math.Sqrt(detR),
	}, nil
}
<|endoftext|>


<commit_before>// Package walk implements a path walker which reads audio files under a path and
// constructs an index.Library from supported metadata tags (see github.com/dhowden/tag).
package walk

import (
	"crypto/sha1"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/dhowden/tag"
	"github.com/tchaik/tchaik/index"
)

var fileExtensions = []string{".mp3", ".m4a", ".flac", ".ogg"}

// NewLibrary constructs an index.Library by walking through the directory tree under
// the given path.  Any errors are logged to stdout (TODO: fix this!)
func NewLibrary(path string) index.Library {
	tracks := make(map[string]*track)
	files := walk(path)
	for p := range files {
		if validExtension(p) {
			t, err := processPath(p)
			if err != nil {
				log.Printf("error processing '%v': %v\n", p, err)
				continue
			}
			tracks[p] = t
		}
	}

	return &library{
		tracks: tracks,
	}
}

// library is an implementation of index.library.
type library struct {
	tracks map[string]*track
}

func (l *library) Track(id string) (index.Track, bool) {
	t, ok := l.tracks[id]
	return t, ok
}

func (l *library) Tracks() []index.Track {
	tracks := make([]index.Track, 0, len(l.tracks))
	for _, t := range l.tracks {
		tracks = append(tracks, t)
	}
	return tracks
}

// track is a wrapper around tag.Metadata which implements index.Track
type track struct {
	tag.Metadata
	Location    string
	FileInfo    os.FileInfo
	CreatedTime time.Time
}

func (m *track) GetString(name string) string {
	switch name {
	case "Name":
		title := m.Title()
		if title == "" {
			fileName := m.FileInfo.Name()
			ext := filepath.Ext(fileName)
			title = strings.TrimSuffix(fileName, ext)
		}
		return title
	case "Album":
		return m.Album()
	case "Artist":
		return m.Artist()
	case "AlbumArtist":
		return m.AlbumArtist()
	case "Composer":
		return m.Composer()
	case "Genre":
		return m.Genre()
	case "Location":
		return m.Location
	case "TrackID":
		sum := sha1.Sum([]byte(m.Location))
		return string(fmt.Sprintf("%x", sum))
	}
	return ""
}

func (m *track) GetInt(name string) int {
	switch name {
	case "Year":
		return m.Year()
	case "TrackNumber":
		x, _ := m.Track()
		return x
	case "TrackCount":
		_, n := m.Track()
		return n
	case "DiscNumber":
		x, _ := m.Disc()
		return x
	case "DiscCount":
		_, n := m.Disc()
		return n
	}
	return 0
}

func (m *track) GetTime(name string) time.Time {
	switch name {
	case "DateModified":
		return m.FileInfo.ModTime()
	case "DateAdded":
		return m.CreatedTime
	}
	return time.Time{}
}

func validExtension(path string) bool {
	ext := strings.ToLower(filepath.Ext(filepath.Base(path)))
	for _, x := range fileExtensions {
		if ext == x {
			return true
		}
	}
	return false
}

func walk(root string) <-chan string {
	ch := make(chan string)
	fn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		ch <- path
		return nil
	}

	go func() {
		err := filepath.Walk(root, fn)
		if err != nil {
			log.Println(err)
		}
		close(ch)
	}()
	return ch
}

func processPath(path string) (*track, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	m, err := tag.ReadFrom(f)
	if err != nil {
		return nil, err
	}

	fileInfo, err := f.Stat()
	if err != nil {
		return nil, err
	}

	createdTime, err := getCreatedTime(path)
	if err != nil {
		return nil, err
	}

	return &track{
		Metadata:    m,
		Location:    path,
		FileInfo:    fileInfo,
		CreatedTime: createdTime,
	}, nil
}
<commit_msg>Rewrite walk.NewLibrary to use multiple goroutine workers.<commit_after>// Package walk implements a path walker which reads audio files under a path and
// constructs an index.Library from supported metadata tags (see github.com/dhowden/tag).
package walk

import (
	"crypto/sha1"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/dhowden/tag"
	"github.com/tchaik/tchaik/index"
)

var fileExtensions = []string{".mp3", ".m4a", ".flac", ".ogg"}
var workers = 4

type pathTrack struct {
	path  string
	track *track
}

// NewLibrary constructs an index.Library by walking through the directory tree under
// the given path.  Any errors are logged to stdout (TODO: fix this!)
func NewLibrary(path string) index.Library {
	trackCh := make(chan pathTrack)
	errCh := make(chan error)
	files := walk(path)

	go func() {
		for err := range errCh {
			// FIXME
			log.Println(err)
		}
	}()

	process := func(files <-chan string) {
		for p := range files {
			if validExtension(p) {
				t, err := processPath(p)
				if err != nil {
					errCh <- fmt.Errorf("error processing '%v': %v", p, err)
					continue
				}
				trackCh <- pathTrack{path, t}
			}
		}
	}

	wg := &sync.WaitGroup{}
	wg.Add(workers)
	for i := 0; i < workers; i++ {
		go func() {
			process(files)
			wg.Done()
		}()
	}

	go func() {
		wg.Wait()
		close(errCh)
		close(trackCh)
	}()

	tracks := make(map[string]*track)
	for pt := range trackCh {
		tracks[pt.path] = pt.track
	}

	return &library{
		tracks: tracks,
	}
}

// library is an implementation of index.library.
type library struct {
	tracks map[string]*track
}

func (l *library) Track(id string) (index.Track, bool) {
	t, ok := l.tracks[id]
	return t, ok
}

func (l *library) Tracks() []index.Track {
	tracks := make([]index.Track, 0, len(l.tracks))
	for _, t := range l.tracks {
		tracks = append(tracks, t)
	}
	return tracks
}

// track is a wrapper around tag.Metadata which implements index.Track
type track struct {
	tag.Metadata
	Location    string
	FileInfo    os.FileInfo
	CreatedTime time.Time
}

func (m *track) GetString(name string) string {
	switch name {
	case "Name":
		title := m.Title()
		if title == "" {
			fileName := m.FileInfo.Name()
			ext := filepath.Ext(fileName)
			title = strings.TrimSuffix(fileName, ext)
		}
		return title
	case "Album":
		return m.Album()
	case "Artist":
		return m.Artist()
	case "AlbumArtist":
		return m.AlbumArtist()
	case "Composer":
		return m.Composer()
	case "Genre":
		return m.Genre()
	case "Location":
		return m.Location
	case "TrackID":
		sum := sha1.Sum([]byte(m.Location))
		return string(fmt.Sprintf("%x", sum))
	}
	return ""
}

func (m *track) GetInt(name string) int {
	switch name {
	case "Year":
		return m.Year()
	case "TrackNumber":
		x, _ := m.Track()
		return x
	case "TrackCount":
		_, n := m.Track()
		return n
	case "DiscNumber":
		x, _ := m.Disc()
		return x
	case "DiscCount":
		_, n := m.Disc()
		return n
	}
	return 0
}

func (m *track) GetTime(name string) time.Time {
	switch name {
	case "DateModified":
		return m.FileInfo.ModTime()
	case "DateAdded":
		return m.CreatedTime
	}
	return time.Time{}
}

func validExtension(path string) bool {
	ext := strings.ToLower(filepath.Ext(filepath.Base(path)))
	for _, x := range fileExtensions {
		if ext == x {
			return true
		}
	}
	return false
}

func walk(root string) <-chan string {
	ch := make(chan string)
	fn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		ch <- path
		return nil
	}

	go func() {
		err := filepath.Walk(root, fn)
		if err != nil {
			log.Println(err)
		}
		close(ch)
	}()
	return ch
}

func processPath(path string) (*track, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	m, err := tag.ReadFrom(f)
	if err != nil {
		return nil, err
	}

	fileInfo, err := f.Stat()
	if err != nil {
		return nil, err
	}

	createdTime, err := getCreatedTime(path)
	if err != nil {
		return nil, err
	}

	return &track{
		Metadata:    m,
		Location:    path,
		FileInfo:    fileInfo,
		CreatedTime: createdTime,
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

// Package agent implements a service that runs on a serviced node. It is
// responsible for ensuring that a particular node is running the correct services
// and reporting the state and health of those services back to the master
// serviced.

package main

// This is the main entry point for the application. Here we parse command line
// flags and either start a service or execute command line functions.

//svc "github.com/zenoss/serviced/svc"
import (
	"github.com/zenoss/serviced"
	"github.com/zenoss/serviced/dao"
	"github.com/zenoss/serviced/dao/elasticsearch"
	"github.com/zenoss/serviced/isvcs"
	"github.com/zenoss/serviced/web"

	"flag"
	"fmt"
	"github.com/zenoss/glog"
	"net"
	"net/http"
	"net/rpc"
	"os"
	"os/signal"
	"os/user"
	"path"
	"time"
)

// Store the command line options
var options struct {
	port           string
	listen         string
	master         bool
	agent          bool
	muxPort        int
	tls            bool
	keyPEMFile     string
	certPEMFile    string
	varPath        string // Directory to store data, eg isvcs & service volumes
	resourcePath   string
	zookeepers     ListOpts
	repstats       bool
	statshost      string
	statsperiod    int
	mcusername     string
	mcpasswd       string
	mount          ListOpts
	resourceperiod int
}

// Setup flag options (static block)
func init() {
	ip, err := serviced.GetIpAddress()
	if err != nil {
		panic(err)
	}

	flag.StringVar(&options.port, "port", ip+":4979", "port for remote serviced (example.com:8080)")
	flag.StringVar(&options.listen, "listen", ":4979", "port for local serviced (example.com:8080)")
	flag.BoolVar(&options.master, "master", false, "run in master mode, ie the control plane service")
	flag.BoolVar(&options.agent, "agent", false, "run in agent mode, ie a host in a resource pool")
	flag.IntVar(&options.muxPort, "muxport", 22250, "multiplexing port to use")
	flag.BoolVar(&options.tls, "tls", true, "enable TLS")

	varPathDefault := path.Join(os.TempDir(), "serviced")
	if len(os.Getenv("SERVICED_HOME")) > 0 {
		varPathDefault = path.Join(os.Getenv("SERVICED_HOME"), "var")
	} else {
		if user, err := user.Current(); err == nil {
			varPathDefault = path.Join(os.TempDir(), "serviced-"+user.Username, "var")
		}
	}
	flag.StringVar(&options.varPath, "varPath", varPathDefault, "path to store serviced data")

	flag.StringVar(&options.keyPEMFile, "keyfile", "", "path to private key file (defaults to compiled in private key)")
	flag.StringVar(&options.certPEMFile, "certfile", "", "path to public certificate file (defaults to compiled in public cert)")
	options.zookeepers = make(ListOpts, 0)
	flag.Var(&options.zookeepers, "zk", "Specify a zookeeper instance to connect to (e.g. -zk localhost:2181 )")
	flag.BoolVar(&options.repstats, "reportstats", false, "report container statistics")
	flag.StringVar(&options.statshost, "statshost", "127.0.0.1:8443", "host:port for container statistics")
	flag.IntVar(&options.statsperiod, "statsperiod", 5, "Period (minutes) for container statistics reporting")
	flag.IntVar(&options.resourceperiod, "resourceperiod", 360, "Period (minutes) for for registering host resources")
	flag.StringVar(&options.mcusername, "mcusername", "scott", "Username for the Zenoss metric consumer")
	flag.StringVar(&options.mcpasswd, "mcpasswd", "tiger", "Password for the Zenoss metric consumer")
	options.mount = make(ListOpts, 0)
	flag.Var(&options.mount, "mount", "bind mount: container_image:host_path:container_path (e.g. -mount zenoss/zenoss5x:/home/zenoss/zenhome/zenoss/Products/:/opt/zenoss/Products/)")

	flag.Usage = func() {
		flag.PrintDefaults()
	}
}

func compareVersion(a, b []int) int {
	astr := ""
	for _, s := range a {
		astr += fmt.Sprintf("%12d", s)
	}
	bstr := ""
	for _, s := range b {
		bstr += fmt.Sprintf("%12d", s)
	}
	if astr > bstr {
		return -1
	}
	if astr < bstr {
		return 1
	}
	return 0
}

// Start the agent or master services on this host.
func startServer() {
	l, err := net.Listen("tcp", options.listen)
	if err != nil {
		glog.Fatalf("Could not bind to port %v. Is another instance running", err)
	}

	isvcs.Init()
	isvcs.Mgr.SetVolumesDir(options.varPath + "/isvcs")

	dockerVersion, err := serviced.GetDockerVersion()
	if err != nil {
		glog.Fatalf("Could not determine docker version: %s", err)
	}

	atLeast := []int{0, 7, 5}
	if compareVersion(atLeast, dockerVersion.Client) < 0 {
		glog.Fatal("serviced needs at least docker 0.7.5")
	}

	if options.master {
		var master dao.ControlPlane
		var err error
		master, err = elasticsearch.NewControlSvc("localhost", 9200, options.zookeepers)

		if err != nil {
			glog.Fatalf("Could not start ControlPlane service: %v", err)
		}
		// register the API
		glog.V(0).Infoln("registering ControlPlane service")
		rpc.RegisterName("LoadBalancer", master)
		rpc.RegisterName("ControlPlane", master)

		// TODO: Make bind port for web server optional?
		cpserver := web.NewServiceConfig(":8787", options.port, options.zookeepers, options.repstats)
		go cpserver.Serve()
	}
	if options.agent {
		mux := serviced.TCPMux{}

		mux.CertPEMFile = options.certPEMFile
		mux.KeyPEMFile = options.keyPEMFile
		mux.Enabled = true
		mux.Port = options.muxPort
		mux.UseTLS = options.tls

		agent, err := serviced.NewHostAgent(options.port, options.varPath, options.mount, options.zookeepers, mux)
		if err != nil {
			glog.Fatalf("Could not start ControlPlane agent: %v", err)
		}
		// register the API
		glog.V(0).Infoln("registering ControlPlaneAgent service")
		rpc.RegisterName("ControlPlaneAgent", agent)

		go func() {
			signalChan := make(chan os.Signal, 10)
			signal.Notify(signalChan, os.Interrupt)
			<-signalChan
			glog.V(0).Info("Shutting down due to interrupt")
			err = agent.Shutdown()
			if err != nil {
				glog.V(1).Infof("Agent shutdown with error: %v", err)
			}
			isvcs.Mgr.Stop()
			os.Exit(0)
		}()

		resourceDuration := time.Duration(options.resourceperiod) * time.Minute
		go agent.RegisterIPResources(resourceDuration)

	}

	rpc.HandleHTTP()

	if options.repstats {
		statsdest := fmt.Sprintf("http://%s/api/metrics/store", options.statshost)
		sr := StatsReporter{statsdest, options.mcusername, options.mcpasswd}

		glog.V(1).Infoln("Staring containter statistics reporter")
		statsduration := time.Duration(options.statsperiod) * time.Minute
		go sr.Report(statsduration)
	}

	glog.V(0).Infof("Listening on %s", l.Addr().String())
	http.Serve(l, nil) // start the server
}

// main entry point of the product
func main() {

	// parse the command line flags
	flag.Parse()

	// are we in server mode
	if (options.master || options.agent) && len(flag.Args()) == 0 {
		startServer()
	} else {
		// we are in command line mode
		if len(flag.Args()) == 0 {
			// no arguments were give, show help
			cli := ServicedCli{}
			cli.CmdHelp(flag.Args()...)
			flag.Usage()
		} else {
			ParseCommands(flag.Args()...)
		}
	}
	glog.Flush()
}
<commit_msg>update supported versions<commit_after>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

// Package agent implements a service that runs on a serviced node. It is
// responsible for ensuring that a particular node is running the correct services
// and reporting the state and health of those services back to the master
// serviced.

package main

// This is the main entry point for the application. Here we parse command line
// flags and either start a service or execute command line functions.

//svc "github.com/zenoss/serviced/svc"
import (
	"github.com/zenoss/serviced"
	"github.com/zenoss/serviced/dao"
	"github.com/zenoss/serviced/dao/elasticsearch"
	"github.com/zenoss/serviced/isvcs"
	"github.com/zenoss/serviced/web"

	"flag"
	"fmt"
	"github.com/zenoss/glog"
	"net"
	"net/http"
	"net/rpc"
	"os"
	"os/signal"
	"os/user"
	"path"
	"time"
)

// Store the command line options
var options struct {
	port           string
	listen         string
	master         bool
	agent          bool
	muxPort        int
	tls            bool
	keyPEMFile     string
	certPEMFile    string
	varPath        string // Directory to store data, eg isvcs & service volumes
	resourcePath   string
	zookeepers     ListOpts
	repstats       bool
	statshost      string
	statsperiod    int
	mcusername     string
	mcpasswd       string
	mount          ListOpts
	resourceperiod int
}

// Setup flag options (static block)
func init() {
	ip, err := serviced.GetIpAddress()
	if err != nil {
		panic(err)
	}

	flag.StringVar(&options.port, "port", ip+":4979", "port for remote serviced (example.com:8080)")
	flag.StringVar(&options.listen, "listen", ":4979", "port for local serviced (example.com:8080)")
	flag.BoolVar(&options.master, "master", false, "run in master mode, ie the control plane service")
	flag.BoolVar(&options.agent, "agent", false, "run in agent mode, ie a host in a resource pool")
	flag.IntVar(&options.muxPort, "muxport", 22250, "multiplexing port to use")
	flag.BoolVar(&options.tls, "tls", true, "enable TLS")

	varPathDefault := path.Join(os.TempDir(), "serviced")
	if len(os.Getenv("SERVICED_HOME")) > 0 {
		varPathDefault = path.Join(os.Getenv("SERVICED_HOME"), "var")
	} else {
		if user, err := user.Current(); err == nil {
			varPathDefault = path.Join(os.TempDir(), "serviced-"+user.Username, "var")
		}
	}
	flag.StringVar(&options.varPath, "varPath", varPathDefault, "path to store serviced data")

	flag.StringVar(&options.keyPEMFile, "keyfile", "", "path to private key file (defaults to compiled in private key)")
	flag.StringVar(&options.certPEMFile, "certfile", "", "path to public certificate file (defaults to compiled in public cert)")
	options.zookeepers = make(ListOpts, 0)
	flag.Var(&options.zookeepers, "zk", "Specify a zookeeper instance to connect to (e.g. -zk localhost:2181 )")
	flag.BoolVar(&options.repstats, "reportstats", false, "report container statistics")
	flag.StringVar(&options.statshost, "statshost", "127.0.0.1:8443", "host:port for container statistics")
	flag.IntVar(&options.statsperiod, "statsperiod", 5, "Period (minutes) for container statistics reporting")
	flag.IntVar(&options.resourceperiod, "resourceperiod", 360, "Period (minutes) for for registering host resources")
	flag.StringVar(&options.mcusername, "mcusername", "scott", "Username for the Zenoss metric consumer")
	flag.StringVar(&options.mcpasswd, "mcpasswd", "tiger", "Password for the Zenoss metric consumer")
	options.mount = make(ListOpts, 0)
	flag.Var(&options.mount, "mount", "bind mount: container_image:host_path:container_path (e.g. -mount zenoss/zenoss5x:/home/zenoss/zenhome/zenoss/Products/:/opt/zenoss/Products/)")

	flag.Usage = func() {
		flag.PrintDefaults()
	}
}

func compareVersion(a, b []int) int {
	astr := ""
	for _, s := range a {
		astr += fmt.Sprintf("%12d", s)
	}
	bstr := ""
	for _, s := range b {
		bstr += fmt.Sprintf("%12d", s)
	}
	if astr > bstr {
		return -1
	}
	if astr < bstr {
		return 1
	}
	return 0
}

// Start the agent or master services on this host.
func startServer() {
	l, err := net.Listen("tcp", options.listen)
	if err != nil {
		glog.Fatalf("Could not bind to port %v. Is another instance running", err)
	}

	isvcs.Init()
	isvcs.Mgr.SetVolumesDir(options.varPath + "/isvcs")

	dockerVersion, err := serviced.GetDockerVersion()
	if err != nil {
		glog.Fatalf("Could not determine docker version: %s", err)
	}

	atLeast := []int{0, 7, 5}
	atMost := []int{0, 7, 6}
	if compareVersion(atLeast, dockerVersion.Client) < 0 || compareVersion(atMost, dockerVersion.Client) > 0 {
		glog.Fatal("serviced needs at least docker >= 0.7.5 or <= 0.7.6")
	}

	if options.master {
		var master dao.ControlPlane
		var err error
		master, err = elasticsearch.NewControlSvc("localhost", 9200, options.zookeepers)

		if err != nil {
			glog.Fatalf("Could not start ControlPlane service: %v", err)
		}
		// register the API
		glog.V(0).Infoln("registering ControlPlane service")
		rpc.RegisterName("LoadBalancer", master)
		rpc.RegisterName("ControlPlane", master)

		// TODO: Make bind port for web server optional?
		cpserver := web.NewServiceConfig(":8787", options.port, options.zookeepers, options.repstats)
		go cpserver.Serve()
	}
	if options.agent {
		mux := serviced.TCPMux{}

		mux.CertPEMFile = options.certPEMFile
		mux.KeyPEMFile = options.keyPEMFile
		mux.Enabled = true
		mux.Port = options.muxPort
		mux.UseTLS = options.tls

		agent, err := serviced.NewHostAgent(options.port, options.varPath, options.mount, options.zookeepers, mux)
		if err != nil {
			glog.Fatalf("Could not start ControlPlane agent: %v", err)
		}
		// register the API
		glog.V(0).Infoln("registering ControlPlaneAgent service")
		rpc.RegisterName("ControlPlaneAgent", agent)

		go func() {
			signalChan := make(chan os.Signal, 10)
			signal.Notify(signalChan, os.Interrupt)
			<-signalChan
			glog.V(0).Info("Shutting down due to interrupt")
			err = agent.Shutdown()
			if err != nil {
				glog.V(1).Infof("Agent shutdown with error: %v", err)
			}
			isvcs.Mgr.Stop()
			os.Exit(0)
		}()

		resourceDuration := time.Duration(options.resourceperiod) * time.Minute
		go agent.RegisterIPResources(resourceDuration)

	}

	rpc.HandleHTTP()

	if options.repstats {
		statsdest := fmt.Sprintf("http://%s/api/metrics/store", options.statshost)
		sr := StatsReporter{statsdest, options.mcusername, options.mcpasswd}

		glog.V(1).Infoln("Staring containter statistics reporter")
		statsduration := time.Duration(options.statsperiod) * time.Minute
		go sr.Report(statsduration)
	}

	glog.V(0).Infof("Listening on %s", l.Addr().String())
	http.Serve(l, nil) // start the server
}

// main entry point of the product
func main() {

	// parse the command line flags
	flag.Parse()

	// are we in server mode
	if (options.master || options.agent) && len(flag.Args()) == 0 {
		startServer()
	} else {
		// we are in command line mode
		if len(flag.Args()) == 0 {
			// no arguments were give, show help
			cli := ServicedCli{}
			cli.CmdHelp(flag.Args()...)
			flag.Usage()
		} else {
			ParseCommands(flag.Args()...)
		}
	}
	glog.Flush()
}
<|endoftext|>


<commit_before>package services

// DBS module
// Copyright (c) 2015-2016 - Valentin Kuznetsov <vkuznet AT gmail dot com>

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/url"

	"github.com/vkuznet/WorkQueue/utils"
)

// helper function to load DBS data stream
func loadDBSData(data []byte) []utils.Record {
	var out []utils.Record

	// to prevent json.Unmarshal behavior to convert all numbers to float
	// we'll use json decode method with instructions to use numbers as is
	buf := bytes.NewBuffer(data)
	dec := json.NewDecoder(buf)
	dec.UseNumber()
	err := dec.Decode(&out)

	// original way to decode data
	// err := json.Unmarshal(data, &out)
	if err != nil {
		msg := fmt.Sprintf("DBS unable to unmarshal the data, data=%s, error=%v", string(data), err)
		fmt.Println(msg)
	}
	return out
}

// dbsUnmarshal unmarshals DBS data stream records
func dbsUnmarshal(data []byte) []utils.Record {
	records := loadDBSData(data)
	var out []utils.Record
	for _, rec := range records {
		out = append(out, rec)
	}
	return out
}

// Blocks function retrieves list of blocks for a given dataset
func Blocks(dataset string) []string {
	rurl := fmt.Sprintf("%s/blocks?dataset=%s", dbsUrl(), url.PathEscape(dataset))
	resp := utils.FetchResponse(rurl, "")
	if resp.Error != nil {
		fmt.Println(resp.Error)
	}
	var out []string
	for _, rec := range dbsUnmarshal(resp.Data) {
		blk := rec["block_name"].(string)
		out = append(out, blk)
	}
	return out
}

// ParentBlocks function retrieves block parents for given list of blocks
func ParentBlocks(blocks []string) []string {
	var requests []Request
	// TODO: DBS provides blockparents GET API and blockparents POST API
	// it is unclear which API to use here
	for _, block := range blocks {
		rurl := fmt.Sprintf("%s/blockparents?block_name=%s", dbsUrl(), url.PathEscape(block))
		req := Request{Name: block, Url: rurl, Args: ""}
		requests = append(requests, req)
	}
	var out []string
	for _, rec := range Process(requests) { // key here is index, rec = {ReqName: []Records}
		for _, row := range rec { // key here is block request.Name
			switch r := row.(type) {
			case []utils.Record:
				for _, vvv := range r {
					switch val := vvv["parent_block_name"].(type) {
					case string:
						out = append(out, val)
					}
				}
			}
		}
	}
	return out
}
<commit_msg>Implemented MaskedBlock structs and function<commit_after>package services

// DBS module
// Copyright (c) 2015-2016 - Valentin Kuznetsov <vkuznet AT gmail dot com>

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/url"

	"github.com/vkuznet/WorkQueue/utils"
)

// helper function to load DBS data stream
func loadDBSData(data []byte) []utils.Record {
	var out []utils.Record

	// to prevent json.Unmarshal behavior to convert all numbers to float
	// we'll use json decode method with instructions to use numbers as is
	buf := bytes.NewBuffer(data)
	dec := json.NewDecoder(buf)
	dec.UseNumber()
	err := dec.Decode(&out)

	// original way to decode data
	// err := json.Unmarshal(data, &out)
	if err != nil {
		msg := fmt.Sprintf("DBS unable to unmarshal the data, data=%s, error=%v", string(data), err)
		fmt.Println(msg)
	}
	return out
}

// dbsUnmarshal unmarshals DBS data stream records
func dbsUnmarshal(data []byte) []utils.Record {
	records := loadDBSData(data)
	var out []utils.Record
	for _, rec := range records {
		out = append(out, rec)
	}
	return out
}

// Blocks function retrieves list of blocks for a given dataset
func Blocks(dataset string) []string {
	rurl := fmt.Sprintf("%s/blocks?dataset=%s", dbsUrl(), url.PathEscape(dataset))
	resp := utils.FetchResponse(rurl, "")
	if resp.Error != nil {
		fmt.Println(resp.Error)
	}
	var out []string
	for _, rec := range dbsUnmarshal(resp.Data) {
		blk := rec["block_name"].(string)
		out = append(out, blk)
	}
	return out
}

// RunLumis keep track of run-lumis
type RunLumis struct {
	Run   int64
	Lumis []int64
}

// String implements Stringer interface
func (r *RunLumis) String() string {
	return fmt.Sprintf("<Run: %d, Lumis: %v>", r.Run, r.Lumis)
}

// FileLumis keep track of block content
type FileLumis struct {
	Lfn      string
	RunLumis RunLumis
}

// String implements Stringer interface
func (r *FileLumis) String() string {
	return fmt.Sprintf("<Lfn: %s, RunLumis: %v>", r.Lfn, r.RunLumis)
}

// MaskedBlock represents block record with list of fileLumis
type MaskedBlock struct {
	Block      string
	FilesLumis []FileLumis
}

// String implements Stringer interface
func (r *MaskedBlock) String() string {
	return fmt.Sprintf("<Block: %s, FilesLumis: %v>", r.Block, r.FilesLumis)
}

// MaskedBlocks returns record of block details mased by provided lumis
func MaskedBlocks(blocks []string) []MaskedBlock {
	var requests []Request
	for _, block := range blocks {
		rurl := fmt.Sprintf("%s/filelumis?block_name=%s", dbsUrl(), url.PathEscape(block))
		req := Request{Name: block, Url: rurl, Args: ""}
		requests = append(requests, req)
	}
	var out []MaskedBlock
	for _, rec := range Process(requests) { // key here is index, rec = {ReqName: []Records}
		for block, row := range rec { // key here is block request.Name
			switch r := row.(type) {
			case []utils.Record:
				var filesLumis []FileLumis
				for _, vvv := range r {
					lfn := vvv["logical_file_name"].(string)
					run, _ := vvv["run_num"].(json.Number).Int64()
					var lumis []int64
					for _, v := range vvv["lumi_section_num"].([]interface{}) {
						lumiNumber, _ := v.(json.Number).Int64()
						lumis = append(lumis, lumiNumber)
					}
					runLumis := RunLumis{Run: run, Lumis: lumis}
					fileLumis := FileLumis{Lfn: lfn, RunLumis: runLumis}
					filesLumis = append(filesLumis, fileLumis)
				}
				maskedBlock := MaskedBlock{Block: block, FilesLumis: filesLumis}
				out = append(out, maskedBlock)
			}
		}
	}
	return out
}

// ParentBlocks function retrieves block parents for given list of blocks
func ParentBlocks(blocks []string) []string {
	var requests []Request
	// TODO: DBS provides blockparents GET API and blockparents POST API
	// it is unclear which API to use here
	for _, block := range blocks {
		rurl := fmt.Sprintf("%s/blockparents?block_name=%s", dbsUrl(), url.PathEscape(block))
		req := Request{Name: block, Url: rurl, Args: ""}
		requests = append(requests, req)
	}
	var out []string
	for _, rec := range Process(requests) { // key here is index, rec = {ReqName: []Records}
		for _, row := range rec { // key here is block request.Name
			switch r := row.(type) {
			case []utils.Record:
				for _, vvv := range r {
					switch val := vvv["parent_block_name"].(type) {
					case string:
						out = append(out, val)
					}
				}
			}
		}
	}
	return out
}
<|endoftext|>


<commit_before>package buffalo

import (
	"fmt"
	"net/http"
	"strings"
	"testing"

	"github.com/gobuffalo/buffalo/render"
	"github.com/gobuffalo/httptest"

	"github.com/stretchr/testify/require"
)

func Test_Session_SingleCookie(t *testing.T) {
	r := require.New(t)

	sessionName := "_test_session"
	a := New(Options{SessionName: sessionName})
	rr := render.New(render.Options{})

	a.GET("/", func(c Context) error {
		return c.Render(http.StatusCreated, rr.String(""))
	})

	w := httptest.New(a)
	res := w.HTML("/").Get()

	var sessionCookies []string
	for _, c := range res.Header().Values("Set-Cookie") {
		if strings.HasPrefix(c, sessionName) {
			sessionCookies = append(sessionCookies, c)
		}
	}

	r.Equal(1, len(sessionCookies))
}

func Test_Session_CustomValue(t *testing.T) {
	r := require.New(t)

	a := New(Options{})
	rr := render.New(render.Options{})

	// Root path sets a custom session value
	a.GET("/", func(c Context) error {
		c.Session().Set("example", "test")
		return c.Render(http.StatusCreated, rr.String(""))
	})
	// /session path prints custom session value as response
	a.GET("/session", func(c Context) error {
		sessionValue := c.Session().Get("example")
		return c.Render(http.StatusCreated, rr.String(fmt.Sprintf("%s", sessionValue)))
	})

	w := httptest.New(a)

	resSetSession := w.HTML("/").Get()

	// Create second request and set the cookie from the first response
	reqGetSession := w.HTML("/session")
	reqGetSession.Headers["Set-Cookie"] = resSetSession.Header().Values("Set-Cookie")[0]
	resGetSession := reqGetSession.Get()

	r.Equal(resGetSession.Body.String(), "test")
}
<commit_msg>Session test remove unneeded cookie header set<commit_after>package buffalo

import (
	"fmt"
	"net/http"
	"strings"
	"testing"

	"github.com/gobuffalo/buffalo/render"
	"github.com/gobuffalo/httptest"

	"github.com/stretchr/testify/require"
)

func Test_Session_SingleCookie(t *testing.T) {
	r := require.New(t)

	sessionName := "_test_session"
	a := New(Options{SessionName: sessionName})
	rr := render.New(render.Options{})

	a.GET("/", func(c Context) error {
		return c.Render(http.StatusCreated, rr.String(""))
	})

	w := httptest.New(a)
	res := w.HTML("/").Get()

	var sessionCookies []string
	for _, c := range res.Header().Values("Set-Cookie") {
		if strings.HasPrefix(c, sessionName) {
			sessionCookies = append(sessionCookies, c)
		}
	}

	r.Equal(1, len(sessionCookies))
}

func Test_Session_CustomValue(t *testing.T) {
	r := require.New(t)

	a := New(Options{})
	rr := render.New(render.Options{})

	// Root path sets a custom session value
	a.GET("/", func(c Context) error {
		c.Session().Set("example", "test")
		return c.Render(http.StatusCreated, rr.String(""))
	})
	// /session path prints custom session value as response
	a.GET("/session", func(c Context) error {
		sessionValue := c.Session().Get("example")
		return c.Render(http.StatusCreated, rr.String(fmt.Sprintf("%s", sessionValue)))
	})

	w := httptest.New(a)
	_ = w.HTML("/").Get()

	// Create second request that should contain the cookie from the first response
	reqGetSession := w.HTML("/session")
	resGetSession := reqGetSession.Get()

	r.Equal(resGetSession.Body.String(), "test")
}
<|endoftext|>