<commit_before>package gnat

import (
	"bytes"
	"errors"
	"math"
	"math/big"
	"math/rand"
	"net"
	"sort"
	"strconv"
	"sync"
	"time"
)

const (
	iterateFindNode = iota
)

const (
	// a small number representing the degree of parallelism in network calls
	alpha = 3

	// the size in bits of the keys used to identify nodes and store and
	// retrieve data; this is 256, the length of a SHA256
	b = 256

	// the maximum number of contacts stored in a bucket
	k = 32
)

// hashTable represents the hashtable state
type hashTable struct {
	// The ID of the local node
	Self *NetworkNode

	// Routing table a list of all known nodes in the network
	// Nodes within buckets are sorted by least recently seen e.g.
	// [ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ]
	//  ^                                                           ^
	//  └ Least recently seen                    Most recently seen ┘
	RoutingTable [][]*node // 256x32

	mutex *sync.Mutex

	refreshMap [b]time.Time
}

func newHashTable(options *Options) (*hashTable, error) {
	ht := &hashTable{}

	rand.Seed(time.Now().UnixNano())

	ht.mutex = &sync.Mutex{}
	ht.Self = &NetworkNode{}

	if options.ID != nil {
		ht.Self.ID = options.ID
	} else {
		id, err := newID()
		if err != nil {
			return nil, err
		}
		ht.Self.ID = id
	}

	if options.IP == "" || options.Port == "" {
		return nil, errors.New("Port and IP required")
	}

	err := ht.setSelfAddr(options.IP, options.Port)
	if err != nil {
		return nil, err
	}

	for i := 0; i < b; i++ {
		ht.resetRefreshTimeForBucket(i)
	}

	for i := 0; i < b; i++ {
		ht.RoutingTable = append(ht.RoutingTable, []*node{})
	}

	return ht, nil
}

func (ht *hashTable) setSelfAddr(ip string, port string) error {
	ht.Self.IP = net.ParseIP(ip)
	p, err := strconv.Atoi(port)
	if err != nil {
		return err
	}
	ht.Self.Port = p
	return nil
}

func (ht *hashTable) resetRefreshTimeForBucket(bucket int) {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()
	ht.refreshMap[bucket] = time.Now()
}

func (ht *hashTable) getRefreshTimeForBucket(bucket int) time.Time {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()
	return ht.refreshMap[bucket]
}

func (ht *hashTable) markNodeAsSeen(node []byte) {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()
	index := getBucketIndexFromDifferingBit(ht.Self.ID, node)
	bucket := ht.RoutingTable[index]
	nodeIndex := -1
	for i, v := range bucket {
		if bytes.Compare(v.ID, node) == 0 {
			nodeIndex = i
			break
		}
	}
	if nodeIndex == -1 {
		panic(errors.New("Tried to mark nonexistent node as seen"))
	}

	n := bucket[nodeIndex]
	bucket = append(bucket[:nodeIndex], bucket[nodeIndex+1:]...)
	bucket = append(bucket, n)
	ht.RoutingTable[index] = bucket
}

func (ht *hashTable) doesNodeExistInBucket(bucket int, node []byte) bool {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()
	for _, v := range ht.RoutingTable[bucket] {
		if bytes.Compare(v.ID, node) == 0 {
			return true
		}
	}
	return false
}

func (ht *hashTable) getClosestContacts(num int, target []byte, ignoredNodes []*NetworkNode) *shortList {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()
	// First we need to build the list of adjacent indices to our target
	// in order
	index := getBucketIndexFromDifferingBit(ht.Self.ID, target)
	indexList := []int{index}
	i := index - 1
	j := index + 1
	for len(indexList) < b {
		if j < b {
			indexList = append(indexList, j)
		}
		if i >= 0 {
			indexList = append(indexList, i)
		}
		i--
		j++
	}

	sl := &shortList{}

	leftToAdd := num

	// Next we select alpha contacts and add them to the short list
	for leftToAdd > 0 && len(indexList) > 0 {
		index, indexList = indexList[0], indexList[1:]
		bucketContacts := len(ht.RoutingTable[index])
		for i := 0; i < bucketContacts; i++ {
			ignored := false
			for j := 0; j < len(ignoredNodes); j++ {
				if bytes.Compare(ht.RoutingTable[index][i].ID, ignoredNodes[j].ID) == 0 {
					ignored = true
				}
			}
			if !ignored {
				sl.AppendUnique([]*node{ht.RoutingTable[index][i]})
				leftToAdd--
				if leftToAdd == 0 {
					break
				}
			}
		}
	}

	sort.Sort(sl)

	return sl
}

func (ht *hashTable) removeNode(ID []byte) {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()

	index := getBucketIndexFromDifferingBit(ht.Self.ID, ID)
	bucket := ht.RoutingTable[index]

	for i, v := range bucket {
		if bytes.Compare(v.ID, ID) == 0 {
			bucket = append(bucket[:i], bucket[i+1:]...)
		}
	}

	ht.RoutingTable[index] = bucket
}

func (ht *hashTable) getAllNodesInBucketCloserThan(bucket int, id []byte) [][]byte {
	b := ht.RoutingTable[bucket]
	var nodes [][]byte
	for _, v := range b {
		d1 := ht.getDistance(id, ht.Self.ID)
		d2 := ht.getDistance(id, v.ID)

		result := big.NewInt(0)
		result.Sub(d1, d2)
		if result.Sign() > -1 {
			nodes = append(nodes, v.ID)
		}
	}

	return nodes
}

func (ht *hashTable) getCloserNode(node1 *NetworkNode, node2 *NetworkNode, id []byte) *NetworkNode {

	d1 := ht.getDistance(id, node1.ID)
	d2 := ht.getDistance(id, node2.ID)
	result := big.NewInt(0)
	result.Sub(d1, d2)
	if result.Sign() <= 0 {
		return node1
	}

	return node2
}

func (ht *hashTable) getTotalNodesInBucket(bucket int) int {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()
	return len(ht.RoutingTable[bucket])
}

func (ht *hashTable) getDistance(id1 []byte, id2 []byte) *big.Int {
	var dst [k]byte
	for i := 0; i < k; i++ {
		dst[i] = id1[i] ^ id2[i]
	}
	ret := big.NewInt(0)
	return ret.SetBytes(dst[:])
}

func (ht *hashTable) getRandomIDFromBucket(bucket int) []byte {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()

	// Set the new ID to to be equal in every byte up to
	// the byte of the first differing bit in the bucket

	byteIndex := bucket / 8
	var id []byte
	for i := 0; i < byteIndex; i++ {
		id = append(id, ht.Self.ID[i])
	}
	differingBitStart := bucket % 8

	var firstByte byte
	// check each bit from left to right in order
	for i := 0; i < 8; i++ {
		// Set the value of the bit to be the same as the ID
		// up to the differing bit. Then begin randomizing
		var bit bool
		if i < differingBitStart {
			bit = hasBit(ht.Self.ID[byteIndex], uint(i))
		} else {
			bit = rand.Intn(2) == 1
		}

		if bit {
			firstByte += byte(math.Pow(2, float64(7-i)))
		}
	}

	id = append(id, firstByte)

	// Randomize each remaining byte
	for i := byteIndex + 1; i < 32; i++ {
		randomByte := byte(rand.Intn(256))
		id = append(id, randomByte)
	}

	return id
}

func getBucketIndexFromDifferingBit(id1 []byte, id2 []byte) int {
	// Look at each byte from left to right
	for j := 0; j < len(id1); j++ {
		// xor the byte
		xor := id1[j] ^ id2[j]

		// check each bit on the xored result from left to right in order
		for i := 0; i < 8; i++ {
			if hasBit(xor, uint(i)) {
				byteIndex := j * 8
				bitIndex := i
				return b - (byteIndex + bitIndex) - 1
			}
		}
	}

	// the ids must be the same
	// this should only happen during bootstrapping
	return 0
}

func (ht *hashTable) totalNodes() int {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()
	var total int
	for _, v := range ht.RoutingTable {
		total += len(v)
	}
	return total
}

// newID generates a new random ID
func newID() ([]byte, error) {
	result := make([]byte, 32)
	_, err := rand.Read(result)
	return result, err
}

// Simple helper function to determine the value of a particular
// bit in a byte by index

// Example:
// number:  1
// bits:    00000001
// pos:     01234567
func hasBit(n byte, pos uint) bool {
	pos = 7 - pos
	val := n & (1 << pos)
	return (val > 0)
}
<commit_msg>check ID size<commit_after>package gnat

import (
	"bytes"
	"errors"
	"math"
	"math/big"
	"math/rand"
	"net"
	"sort"
	"strconv"
	"sync"
	"time"
)

const (
	iterateFindNode = iota
)

const (
	// a small number representing the degree of parallelism in network calls
	alpha = 3

	// the size in bits of the keys used to identify nodes and store and
	// retrieve data; this is 256, the length of a SHA256
	b = 256

	// the maximum number of contacts stored in a bucket
	k = 32
)

// hashTable represents the hashtable state
type hashTable struct {
	// The ID of the local node
	Self *NetworkNode

	// Routing table a list of all known nodes in the network
	// Nodes within buckets are sorted by least recently seen e.g.
	// [ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ][ ]
	//  ^                                                           ^
	//  └ Least recently seen                    Most recently seen ┘
	RoutingTable [][]*node // 256x32

	mutex *sync.Mutex

	refreshMap [b]time.Time
}

func newHashTable(options *Options) (*hashTable, error) {
	ht := &hashTable{}

	rand.Seed(time.Now().UnixNano())

	ht.mutex = &sync.Mutex{}
	ht.Self = &NetworkNode{}

	if options.ID != nil {
		if len(options.ID) != b {
			return nil, errors.New("error: invalid key size " + string(len(options.ID)))
		}
		ht.Self.ID = options.ID
	} else {
		id, err := newID()
		if err != nil {
			return nil, err
		}
		ht.Self.ID = id
	}

	if options.IP == "" || options.Port == "" {
		return nil, errors.New("Port and IP required")
	}

	err := ht.setSelfAddr(options.IP, options.Port)
	if err != nil {
		return nil, err
	}

	for i := 0; i < b; i++ {
		ht.resetRefreshTimeForBucket(i)
	}

	for i := 0; i < b; i++ {
		ht.RoutingTable = append(ht.RoutingTable, []*node{})
	}

	return ht, nil
}

func (ht *hashTable) setSelfAddr(ip string, port string) error {
	ht.Self.IP = net.ParseIP(ip)
	p, err := strconv.Atoi(port)
	if err != nil {
		return err
	}
	ht.Self.Port = p
	return nil
}

func (ht *hashTable) resetRefreshTimeForBucket(bucket int) {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()
	ht.refreshMap[bucket] = time.Now()
}

func (ht *hashTable) getRefreshTimeForBucket(bucket int) time.Time {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()
	return ht.refreshMap[bucket]
}

func (ht *hashTable) markNodeAsSeen(node []byte) {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()
	index := getBucketIndexFromDifferingBit(ht.Self.ID, node)
	bucket := ht.RoutingTable[index]
	nodeIndex := -1
	for i, v := range bucket {
		if bytes.Compare(v.ID, node) == 0 {
			nodeIndex = i
			break
		}
	}
	if nodeIndex == -1 {
		panic(errors.New("Tried to mark nonexistent node as seen"))
	}

	n := bucket[nodeIndex]
	bucket = append(bucket[:nodeIndex], bucket[nodeIndex+1:]...)
	bucket = append(bucket, n)
	ht.RoutingTable[index] = bucket
}

func (ht *hashTable) doesNodeExistInBucket(bucket int, node []byte) bool {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()
	for _, v := range ht.RoutingTable[bucket] {
		if bytes.Compare(v.ID, node) == 0 {
			return true
		}
	}
	return false
}

func (ht *hashTable) getClosestContacts(num int, target []byte, ignoredNodes []*NetworkNode) *shortList {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()
	// First we need to build the list of adjacent indices to our target
	// in order
	index := getBucketIndexFromDifferingBit(ht.Self.ID, target)
	indexList := []int{index}
	i := index - 1
	j := index + 1
	for len(indexList) < b {
		if j < b {
			indexList = append(indexList, j)
		}
		if i >= 0 {
			indexList = append(indexList, i)
		}
		i--
		j++
	}

	sl := &shortList{}

	leftToAdd := num

	// Next we select alpha contacts and add them to the short list
	for leftToAdd > 0 && len(indexList) > 0 {
		index, indexList = indexList[0], indexList[1:]
		bucketContacts := len(ht.RoutingTable[index])
		for i := 0; i < bucketContacts; i++ {
			ignored := false
			for j := 0; j < len(ignoredNodes); j++ {
				if bytes.Compare(ht.RoutingTable[index][i].ID, ignoredNodes[j].ID) == 0 {
					ignored = true
				}
			}
			if !ignored {
				sl.AppendUnique([]*node{ht.RoutingTable[index][i]})
				leftToAdd--
				if leftToAdd == 0 {
					break
				}
			}
		}
	}

	sort.Sort(sl)

	return sl
}

func (ht *hashTable) removeNode(ID []byte) {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()

	index := getBucketIndexFromDifferingBit(ht.Self.ID, ID)
	bucket := ht.RoutingTable[index]

	for i, v := range bucket {
		if bytes.Compare(v.ID, ID) == 0 {
			bucket = append(bucket[:i], bucket[i+1:]...)
		}
	}

	ht.RoutingTable[index] = bucket
}

func (ht *hashTable) getAllNodesInBucketCloserThan(bucket int, id []byte) [][]byte {
	b := ht.RoutingTable[bucket]
	var nodes [][]byte
	for _, v := range b {
		d1 := ht.getDistance(id, ht.Self.ID)
		d2 := ht.getDistance(id, v.ID)

		result := big.NewInt(0)
		result.Sub(d1, d2)
		if result.Sign() > -1 {
			nodes = append(nodes, v.ID)
		}
	}

	return nodes
}

func (ht *hashTable) getCloserNode(node1 *NetworkNode, node2 *NetworkNode, id []byte) *NetworkNode {

	d1 := ht.getDistance(id, node1.ID)
	d2 := ht.getDistance(id, node2.ID)
	result := big.NewInt(0)
	result.Sub(d1, d2)
	if result.Sign() <= 0 {
		return node1
	}

	return node2
}

func (ht *hashTable) getTotalNodesInBucket(bucket int) int {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()
	return len(ht.RoutingTable[bucket])
}

func (ht *hashTable) getDistance(id1 []byte, id2 []byte) *big.Int {
	var dst [k]byte
	for i := 0; i < k; i++ {
		dst[i] = id1[i] ^ id2[i]
	}
	ret := big.NewInt(0)
	return ret.SetBytes(dst[:])
}

func (ht *hashTable) getRandomIDFromBucket(bucket int) []byte {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()

	// Set the new ID to to be equal in every byte up to
	// the byte of the first differing bit in the bucket

	byteIndex := bucket / 8
	var id []byte
	for i := 0; i < byteIndex; i++ {
		id = append(id, ht.Self.ID[i])
	}
	differingBitStart := bucket % 8

	var firstByte byte
	// check each bit from left to right in order
	for i := 0; i < 8; i++ {
		// Set the value of the bit to be the same as the ID
		// up to the differing bit. Then begin randomizing
		var bit bool
		if i < differingBitStart {
			bit = hasBit(ht.Self.ID[byteIndex], uint(i))
		} else {
			bit = rand.Intn(2) == 1
		}

		if bit {
			firstByte += byte(math.Pow(2, float64(7-i)))
		}
	}

	id = append(id, firstByte)

	// Randomize each remaining byte
	for i := byteIndex + 1; i < 32; i++ {
		randomByte := byte(rand.Intn(256))
		id = append(id, randomByte)
	}

	return id
}

func getBucketIndexFromDifferingBit(id1 []byte, id2 []byte) int {
	// Look at each byte from left to right
	for j := 0; j < len(id1); j++ {
		// xor the byte
		xor := id1[j] ^ id2[j]

		// check each bit on the xored result from left to right in order
		for i := 0; i < 8; i++ {
			if hasBit(xor, uint(i)) {
				byteIndex := j * 8
				bitIndex := i
				return b - (byteIndex + bitIndex) - 1
			}
		}
	}

	// the ids must be the same
	// this should only happen during bootstrapping
	return 0
}

func (ht *hashTable) totalNodes() int {
	ht.mutex.Lock()
	defer ht.mutex.Unlock()
	var total int
	for _, v := range ht.RoutingTable {
		total += len(v)
	}
	return total
}

// newID generates a new random ID
func newID() ([]byte, error) {
	result := make([]byte, 32)
	_, err := rand.Read(result)
	return result, err
}

// Simple helper function to determine the value of a particular
// bit in a byte by index

// Example:
// number:  1
// bits:    00000001
// pos:     01234567
func hasBit(n byte, pos uint) bool {
	pos = 7 - pos
	val := n & (1 << pos)
	return (val > 0)
}
<|endoftext|>


<commit_before>package vsock

import (
	"fmt"
	"os"
	"syscall"
	"unsafe"
)

const (
	// ioctlGetLocalCID is an ioctl value that retrieves the local context ID
	// from /dev/vsock.
	// TODO(mdlayher): this is probably linux/amd64 specific, but I'm unsure
	// how to make it portable across architectures.
	ioctlGetLocalCID = 0x7b9

	// devVsock is the location of /dev/vsock.  It is exposed on both the
	// hypervisor and on virtual machines.
	devVsock = "/dev/vsock"
)

// A fs is an interface over the filesystem and ioctl, to enable testing.
type fs interface {
	Open(name string) (*os.File, error)
	Ioctl(fd uintptr, request int, argp uintptr) error
}

// localContextID retrieves the local context ID for this system, using the
// methods from fs.  The context ID is stored in cid for later use.
//
// This method uses this signature to enable easier testing without unsafe
// usage of unsafe.Pointer.
func localContextID(fs fs, cid *uint32) error {
	f, err := fs.Open(devVsock)
	if err != nil {
		return err
	}
	defer f.Close()

	// Retrieve the context ID of this machine from /dev/vsock.
	return fs.Ioctl(f.Fd(), ioctlGetLocalCID, uintptr(unsafe.Pointer(cid)))
}

// A sysFS is the system call implementation of fs.
type sysFS struct{}

func (sysFS) Open(name string) (*os.File, error) { return os.Open(name) }
func (sysFS) Ioctl(fd uintptr, request int, argp uintptr) error {
	_, _, errno := syscall.Syscall(
		syscall.SYS_IOCTL,
		fd,
		uintptr(request),
		argp,
	)
	if errno != 0 {
		return os.NewSyscallError("ioctl", fmt.Errorf("%d", int(errno)))
	}

	return nil
}
<commit_msg>vsock: use x/sys/unix instead of syscall for ioctl<commit_after>package vsock

import (
	"fmt"
	"os"
	"unsafe"

	"golang.org/x/sys/unix"
)

const (
	// ioctlGetLocalCID is an ioctl value that retrieves the local context ID
	// from /dev/vsock.
	// TODO(mdlayher): this is probably linux/amd64 specific, but I'm unsure
	// how to make it portable across architectures.
	ioctlGetLocalCID = 0x7b9

	// devVsock is the location of /dev/vsock.  It is exposed on both the
	// hypervisor and on virtual machines.
	devVsock = "/dev/vsock"
)

// A fs is an interface over the filesystem and ioctl, to enable testing.
type fs interface {
	Open(name string) (*os.File, error)
	Ioctl(fd uintptr, request int, argp uintptr) error
}

// localContextID retrieves the local context ID for this system, using the
// methods from fs.  The context ID is stored in cid for later use.
//
// This method uses this signature to enable easier testing without unsafe
// usage of unsafe.Pointer.
func localContextID(fs fs, cid *uint32) error {
	f, err := fs.Open(devVsock)
	if err != nil {
		return err
	}
	defer f.Close()

	// Retrieve the context ID of this machine from /dev/vsock.
	return fs.Ioctl(f.Fd(), ioctlGetLocalCID, uintptr(unsafe.Pointer(cid)))
}

// A sysFS is the system call implementation of fs.
type sysFS struct{}

func (sysFS) Open(name string) (*os.File, error) { return os.Open(name) }
func (sysFS) Ioctl(fd uintptr, request int, argp uintptr) error {
	_, _, errno := unix.Syscall(
		unix.SYS_IOCTL,
		fd,
		uintptr(request),
		argp,
	)
	if errno != 0 {
		return os.NewSyscallError("ioctl", fmt.Errorf("%d", int(errno)))
	}

	return nil
}
<|endoftext|>


<commit_before>package engine

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/require"
)

func Test_NewArea_InvalidSize(t *testing.T) {
	var area Area
	var err error

	area, err = NewArea(0, 0)
	require.NotNil(t, err)
	require.Equal(t, Area{}, area)

	area, err = NewArea(1, 0)
	require.NotNil(t, err)
	require.Equal(t, Area{}, area)

	area, err = NewArea(0, 1)
	require.NotNil(t, err)
	require.Equal(t, Area{}, area)
}

func Test_NewArea_ValidSize(t *testing.T) {
	var err error

	_, err = NewArea(1, 1)
	require.Nil(t, err)

	_, err = NewArea(100, 100)
	require.Nil(t, err)
}

func Test_Area_Navigate_SquareArea100x100(t *testing.T) {
	tests := []struct {
		inputDot    Dot
		inputDir    Direction
		inputDis    uint8
		expectedDot Dot
		expectedErr error
	}{
		{Dot{0, 0}, DirectionWest, 0, Dot{0, 0}, nil},
		{Dot{0, 0}, DirectionEast, 0, Dot{0, 0}, nil},
		{Dot{0, 0}, DirectionNorth, 0, Dot{0, 0}, nil},
		{Dot{0, 0}, DirectionSouth, 0, Dot{0, 0}, nil},

		{Dot{0, 0}, DirectionWest, 1, Dot{99, 0}, nil},
		{Dot{0, 0}, DirectionEast, 1, Dot{1, 0}, nil},
		{Dot{0, 0}, DirectionNorth, 1, Dot{0, 99}, nil},
		{Dot{0, 0}, DirectionSouth, 1, Dot{0, 1}, nil},

		{Dot{99, 0}, DirectionEast, 1, Dot{0, 0}, nil},
		{Dot{0, 99}, DirectionSouth, 1, Dot{0, 0}, nil},

		{Dot{0, 99}, DirectionSouth, 150, Dot{0, 49}, nil},
		{Dot{0, 99}, DirectionNorth, 150, Dot{0, 49}, nil},
		{Dot{0, 0}, DirectionEast, 150, Dot{50, 0}, nil},
		{Dot{0, 0}, DirectionWest, 150, Dot{50, 0}, nil},

		{Dot{30, 30}, DirectionWest, 5, Dot{25, 30}, nil},

		// Error case: invalid direction
		{Dot{0, 0}, 21, 150, Dot{}, &ErrNavigation{
			Err: &ErrInvalidDirection{
				Direction: 21,
			},
		}},
		// Error case: area doesn't contains dot
		{Dot{250, 25}, DirectionWest, 150, Dot{}, &ErrNavigation{
			Err: &ErrAreaNotContainsDot{
				Dot: Dot{250, 25},
			},
		}},
	}

	area := Area{
		width:  100,
		height: 100,
	}

	for i, test := range tests {
		actualDot, actualErr := area.Navigate(test.inputDot, test.inputDir, test.inputDis)
		require.Equal(t, test.expectedDot, actualDot, fmt.Sprintf("number %d", i))
		require.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("number %d", i))
	}
}

func Test_Area_Navigate_SquareMaxArea255x255(t *testing.T) {
	tests := []struct {
		inputDot    Dot
		inputDir    Direction
		inputDis    uint8
		expectedDot Dot
		expectedErr error
	}{
		{Dot{0, 0}, DirectionWest, 0, Dot{0, 0}, nil},
		{Dot{0, 0}, DirectionEast, 0, Dot{0, 0}, nil},
		{Dot{0, 0}, DirectionNorth, 0, Dot{0, 0}, nil},
		{Dot{0, 0}, DirectionSouth, 0, Dot{0, 0}, nil},

		{Dot{0, 0}, DirectionWest, 1, Dot{254, 0}, nil},
		{Dot{0, 0}, DirectionEast, 1, Dot{1, 0}, nil},
		{Dot{0, 0}, DirectionNorth, 1, Dot{0, 254}, nil},
		{Dot{0, 0}, DirectionSouth, 1, Dot{0, 1}, nil},

		{Dot{254, 0}, DirectionEast, 1, Dot{0, 0}, nil},
		{Dot{0, 254}, DirectionSouth, 1, Dot{0, 0}, nil},
	}

	area := Area{
		width:  255,
		height: 255,
	}

	for i, test := range tests {
		actualDot, actualErr := area.Navigate(test.inputDot, test.inputDir, test.inputDis)
		require.Equal(t, test.expectedDot, actualDot, fmt.Sprintf("number %d", i))
		require.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("number %d", i))
	}
}

func Test_Area_MarshalJSON(t *testing.T) {
	tests := []struct {
		area Area
		json []byte
	}{
		{
			Area{10, 10},
			[]byte("[10,10]"),
		},
		{
			Area{255, 255},
			[]byte("[255,255]"),
		},
		{
			Area{0, 0},
			[]byte("[0,0]"),
		},
		{
			Area{0, 1},
			[]byte("[0,1]"),
		},
		{
			Area{2, 1},
			[]byte("[2,1]"),
		},
		{
			Area{255, 1},
			[]byte("[255,1]"),
		},
		{
			Area{255, 100},
			[]byte("[255,100]"),
		},
		{
			Area{0, 255},
			[]byte("[0,255]"),
		},
	}

	for i, test := range tests {
		actualJSON, err := test.area.MarshalJSON()
		require.Nil(t, err, "test %d", i)
		require.Equal(t, test.json, actualJSON, "test %d", i)
	}
}
<commit_msg>Create tests for engine.Area.ContainsDot n ContainsRect<commit_after>package engine

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/require"
)

func Test_NewArea_InvalidSize(t *testing.T) {
	var area Area
	var err error

	area, err = NewArea(0, 0)
	require.NotNil(t, err)
	require.Equal(t, Area{}, area)

	area, err = NewArea(1, 0)
	require.NotNil(t, err)
	require.Equal(t, Area{}, area)

	area, err = NewArea(0, 1)
	require.NotNil(t, err)
	require.Equal(t, Area{}, area)
}

func Test_NewArea_ValidSize(t *testing.T) {
	var err error

	_, err = NewArea(1, 1)
	require.Nil(t, err)

	_, err = NewArea(100, 100)
	require.Nil(t, err)
}

func Test_Area_Navigate_SquareArea100x100(t *testing.T) {
	tests := []struct {
		inputDot    Dot
		inputDir    Direction
		inputDis    uint8
		expectedDot Dot
		expectedErr error
	}{
		{Dot{0, 0}, DirectionWest, 0, Dot{0, 0}, nil},
		{Dot{0, 0}, DirectionEast, 0, Dot{0, 0}, nil},
		{Dot{0, 0}, DirectionNorth, 0, Dot{0, 0}, nil},
		{Dot{0, 0}, DirectionSouth, 0, Dot{0, 0}, nil},

		{Dot{0, 0}, DirectionWest, 1, Dot{99, 0}, nil},
		{Dot{0, 0}, DirectionEast, 1, Dot{1, 0}, nil},
		{Dot{0, 0}, DirectionNorth, 1, Dot{0, 99}, nil},
		{Dot{0, 0}, DirectionSouth, 1, Dot{0, 1}, nil},

		{Dot{99, 0}, DirectionEast, 1, Dot{0, 0}, nil},
		{Dot{0, 99}, DirectionSouth, 1, Dot{0, 0}, nil},

		{Dot{0, 99}, DirectionSouth, 150, Dot{0, 49}, nil},
		{Dot{0, 99}, DirectionNorth, 150, Dot{0, 49}, nil},
		{Dot{0, 0}, DirectionEast, 150, Dot{50, 0}, nil},
		{Dot{0, 0}, DirectionWest, 150, Dot{50, 0}, nil},

		{Dot{30, 30}, DirectionWest, 5, Dot{25, 30}, nil},

		// Error case: invalid direction
		{Dot{0, 0}, 21, 150, Dot{}, &ErrNavigation{
			Err: &ErrInvalidDirection{
				Direction: 21,
			},
		}},
		// Error case: area doesn't contains dot
		{Dot{250, 25}, DirectionWest, 150, Dot{}, &ErrNavigation{
			Err: &ErrAreaNotContainsDot{
				Dot: Dot{250, 25},
			},
		}},
	}

	area := Area{
		width:  100,
		height: 100,
	}

	for i, test := range tests {
		actualDot, actualErr := area.Navigate(test.inputDot, test.inputDir, test.inputDis)
		require.Equal(t, test.expectedDot, actualDot, fmt.Sprintf("number %d", i))
		require.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("number %d", i))
	}
}

func Test_Area_Navigate_SquareMaxArea255x255(t *testing.T) {
	tests := []struct {
		inputDot    Dot
		inputDir    Direction
		inputDis    uint8
		expectedDot Dot
		expectedErr error
	}{
		{Dot{0, 0}, DirectionWest, 0, Dot{0, 0}, nil},
		{Dot{0, 0}, DirectionEast, 0, Dot{0, 0}, nil},
		{Dot{0, 0}, DirectionNorth, 0, Dot{0, 0}, nil},
		{Dot{0, 0}, DirectionSouth, 0, Dot{0, 0}, nil},

		{Dot{0, 0}, DirectionWest, 1, Dot{254, 0}, nil},
		{Dot{0, 0}, DirectionEast, 1, Dot{1, 0}, nil},
		{Dot{0, 0}, DirectionNorth, 1, Dot{0, 254}, nil},
		{Dot{0, 0}, DirectionSouth, 1, Dot{0, 1}, nil},

		{Dot{254, 0}, DirectionEast, 1, Dot{0, 0}, nil},
		{Dot{0, 254}, DirectionSouth, 1, Dot{0, 0}, nil},
	}

	area := Area{
		width:  255,
		height: 255,
	}

	for i, test := range tests {
		actualDot, actualErr := area.Navigate(test.inputDot, test.inputDir, test.inputDis)
		require.Equal(t, test.expectedDot, actualDot, fmt.Sprintf("number %d", i))
		require.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("number %d", i))
	}
}

func Test_Area_MarshalJSON(t *testing.T) {
	tests := []struct {
		area Area
		json []byte
	}{
		{
			Area{10, 10},
			[]byte("[10,10]"),
		},
		{
			Area{255, 255},
			[]byte("[255,255]"),
		},
		{
			Area{0, 0},
			[]byte("[0,0]"),
		},
		{
			Area{0, 1},
			[]byte("[0,1]"),
		},
		{
			Area{2, 1},
			[]byte("[2,1]"),
		},
		{
			Area{255, 1},
			[]byte("[255,1]"),
		},
		{
			Area{255, 100},
			[]byte("[255,100]"),
		},
		{
			Area{0, 255},
			[]byte("[0,255]"),
		},
	}

	for i, test := range tests {
		actualJSON, err := test.area.MarshalJSON()
		require.Nil(t, err, "test %d", i)
		require.Equal(t, test.json, actualJSON, "test %d", i)
	}
}

func Test_Area_ContainsDot(t *testing.T) {
	tests := []struct {
		area     Area
		dot      Dot
		expected bool
	}{
		{Area{1, 1}, Dot{}, true},
		{Area{1, 1}, Dot{1, 1}, false},
		{Area{50, 100}, Dot{34, 12}, true},
		{Area{100, 100}, Dot{101, 101}, false},
	}

	for i, test := range tests {
		require.Equal(t, test.expected, test.area.ContainsDot(test.dot), fmt.Sprintf("number: %d", i))
	}
}

func Test_Area_ContainsLocation(t *testing.T) {
	tests := []struct {
		area     Area
		location Location
		expected bool
	}{
		{Area{1, 1}, Location{{0, 0}}, true},
		{Area{1, 1}, Location{{1, 1}}, false},
		{Area{50, 100}, Location{{}, {}, {}}, true},
		{Area{100, 100}, Location{{1, 1}, {2, 10}, {100, 100}}, false},
	}

	for i, test := range tests {
		require.Equal(t, test.expected, test.area.ContainsLocation(test.location), fmt.Sprintf("number: %d", i))
	}
}

func Test_Area_ContainsRect(t *testing.T) {
	tests := []struct {
		area     Area
		rect     Rect
		expected bool
	}{
		{Area{1, 1}, Rect{0, 0, 1, 1}, true},
		{Area{1, 1}, Rect{1, 1, 10, 1}, false},
		{Area{50, 100}, Rect{10, 3, 20, 24}, true},
		{Area{100, 100}, Rect{50, 43, 120, 32}, false},
	}

	for i, test := range tests {
		require.Equal(t, test.expected, test.area.ContainsRect(test.rect), fmt.Sprintf("number: %d", i))
	}
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package tasks provides helpers for managing scheduled tasks.
package tasks

import (
	"errors"
	"fmt"
	"strings"

	"github.com/capnspacehook/taskmaster"
)

var (
	// ErrTaskNotFound indicates a failure due to task resolution
	ErrTaskNotFound = errors.New("task not found")
	// ErrNotRegistered indicates that the querired Scheduled Task
	// is not registered in the Windows Task Scheduler
	ErrNotRegistered = errors.New("scheduled task is not registered")
)

func setEnabled(name string, enabled bool) error {
	svc, err := taskmaster.Connect()
	if err != nil {
		return fmt.Errorf("taskmaster.Connect: %w", err)
	}
	defer svc.Disconnect()

	tasks, err := svc.GetRegisteredTasks()
	if err != nil {
		return fmt.Errorf("svc.GetRegisteredTasks: %w", err)
	}
	defer tasks.Release()

	for _, t := range tasks {
		if strings.EqualFold(t.Name, name) {
			t.Definition.Settings.Enabled = enabled
			_, err = svc.UpdateTask(t.Path, t.Definition)
			return err
		}
	}
	return ErrTaskNotFound
}

// Disable disables a scheduled task.
func Disable(name string) error {
	return setEnabled(name, false)
}

// Enable enables a scheduled task.
func Enable(name string) error {
	return setEnabled(name, true)
}

func taskMatcher(name string, tasks taskmaster.RegisteredTaskCollection) bool {
	for _, t := range tasks {
		if strings.EqualFold(t.Name, name) {
			return true
		}
	}
	return false
}

// TaskExists is a helper function that detects whether a scheduled task exists.
func TaskExists(name string) (bool, error) {
	svc, err := taskmaster.Connect()
	if err != nil {
		return false, fmt.Errorf("taskmaster.Connect: %w", err)
	}
	defer svc.Disconnect()

	tasks, err := svc.GetRegisteredTasks()
	if err != nil {
		return false, fmt.Errorf("svc.GetRegisteredTasks: %w", err)
	}
	defer tasks.Release()

	return taskMatcher(name, tasks), nil
}

// Delete attempts to delete a scheduled task.
func Delete(name string) error {
	svc, err := taskmaster.Connect()
	if err != nil {
		return err
	}
	defer svc.Disconnect()
	tasks, err := svc.GetRegisteredTasks()
	if err != nil {
		return err
	}
	defer tasks.Release()
	for _, t := range tasks {
		if strings.EqualFold(t.Name, name) {
			return svc.DeleteTask(t.Path)
		}
	}
	return ErrTaskNotFound
}
<commit_msg>refactor: Open source tasks.Create<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package tasks provides helpers for managing scheduled tasks.
package tasks

import (
	"errors"
	"fmt"
	"strings"

	"github.com/capnspacehook/taskmaster"
)

var (
	// ErrTaskNotFound indicates a failure due to task resolution
	ErrTaskNotFound = errors.New("task not found")
	// ErrNotRegistered indicates that the querired Scheduled Task
	// is not registered in the Windows Task Scheduler
	ErrNotRegistered = errors.New("scheduled task is not registered")
)

func setEnabled(name string, enabled bool) error {
	svc, err := taskmaster.Connect()
	if err != nil {
		return fmt.Errorf("taskmaster.Connect: %w", err)
	}
	defer svc.Disconnect()

	tasks, err := svc.GetRegisteredTasks()
	if err != nil {
		return fmt.Errorf("svc.GetRegisteredTasks: %w", err)
	}
	defer tasks.Release()

	for _, t := range tasks {
		if strings.EqualFold(t.Name, name) {
			t.Definition.Settings.Enabled = enabled
			_, err = svc.UpdateTask(t.Path, t.Definition)
			return err
		}
	}
	return ErrTaskNotFound
}

// Disable disables a scheduled task.
func Disable(name string) error {
	return setEnabled(name, false)
}

// Enable enables a scheduled task.
func Enable(name string) error {
	return setEnabled(name, true)
}

func taskMatcher(name string, tasks taskmaster.RegisteredTaskCollection) bool {
	for _, t := range tasks {
		if strings.EqualFold(t.Name, name) {
			return true
		}
	}
	return false
}

// TaskExists is a helper function that detects whether a scheduled task exists.
func TaskExists(name string) (bool, error) {
	svc, err := taskmaster.Connect()
	if err != nil {
		return false, fmt.Errorf("taskmaster.Connect: %w", err)
	}
	defer svc.Disconnect()

	tasks, err := svc.GetRegisteredTasks()
	if err != nil {
		return false, fmt.Errorf("svc.GetRegisteredTasks: %w", err)
	}
	defer tasks.Release()

	return taskMatcher(name, tasks), nil
}

// Create attempts to create a scheduled task.
//
// Set common defaults for name and user to reduce the amount of required args.
func Create(id, path, args, name, user string, trigger taskmaster.Trigger) error {
	svc, err := taskmaster.Connect()
	if err != nil {
		return fmt.Errorf("taskmaster.Connect: %w", err)
	}
	defer svc.Disconnect()

	def := svc.NewTaskDefinition()
	def.AddAction(taskmaster.ExecAction{
		ID:   id,
		Path: path,
		Args: args,
	})
	def.Principal.RunLevel = taskmaster.TASK_RUNLEVEL_HIGHEST
	if user == "" {
		user = "SYSTEM"
	}
	def.Principal.UserID = user
	def.AddTrigger(trigger)

	if name == "" {
		name = id
	}
	task, ok, err := svc.CreateTask(`\`+name, def, true)
	if err != nil {
		return fmt.Errorf("svc.CreateTask: %w", err)
	}
	if !ok {
		return errors.New("unable to register task")
	}
	task.Release()

	return nil
}

// Delete attempts to delete a scheduled task.
func Delete(name string) error {
	svc, err := taskmaster.Connect()
	if err != nil {
		return err
	}
	defer svc.Disconnect()
	tasks, err := svc.GetRegisteredTasks()
	if err != nil {
		return err
	}
	defer tasks.Release()
	for _, t := range tasks {
		if strings.EqualFold(t.Name, name) {
			return svc.DeleteTask(t.Path)
		}
	}
	return ErrTaskNotFound
}
<|endoftext|>


<commit_before>package storm

import (
	"reflect"

	"github.com/asdine/storm/q"
	"github.com/boltdb/bolt"
)

// Query is the low level query engine used by Storm. It allows to operate searches through an entire bucket.
type Query interface {
	// Skip matching records by the given number
	Skip(int) Query

	// Limit the results by the given number
	Limit(int) Query

	// Find a list of matching records
	Find(interface{}) error
}

func newQuery(n *Node, tree q.Matcher) *query {
	return &query{
		skip:  0,
		limit: -1,
		node:  n,
		tree:  tree,
	}
}

type query struct {
	limit int
	skip  int
	tree  q.Matcher
	node  *Node
}

func (q *query) Skip(nb int) Query {
	q.skip = nb
	return q
}

func (q *query) Limit(nb int) Query {
	q.limit = nb
	return q
}

func (q *query) Find(to interface{}) error {
	ref := reflect.ValueOf(to)

	if ref.Kind() != reflect.Ptr || reflect.Indirect(ref).Kind() != reflect.Slice {
		return ErrSlicePtrNeeded
	}

	elemType := reflect.Indirect(ref).Type().Elem()

	if elemType.Kind() == reflect.Ptr {
		elemType = elemType.Elem()
	}

	if q.node.tx != nil {
		return q.query(q.node.tx, &ref, elemType)
	}

	return q.node.s.Bolt.Update(func(tx *bolt.Tx) error {
		return q.query(tx, &ref, elemType)
	})
}

func (q *query) query(tx *bolt.Tx, ref *reflect.Value, elemType reflect.Type) error {
	results := reflect.MakeSlice(reflect.Indirect(*ref).Type(), 0, 0)
	bucket := q.node.GetBucket(tx, elemType.Name())

	realType := reflect.Indirect(*ref).Type().Elem()

	// we don't change state so queries can be replayed
	skip := q.skip
	limit := q.limit

	if bucket != nil {
		c := bucket.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			if v == nil {
				continue
			}

			if limit == 0 {
				break
			}

			newElem := reflect.New(elemType)
			err := q.node.s.Codec.Decode(v, newElem.Interface())
			if err != nil {
				return err
			}

			if q.tree.Match(newElem.Interface()) {

				if skip > 0 {
					skip--
					continue
				}

				if limit > 0 {
					limit--
				}

				if realType.Kind() == reflect.Ptr {
					results = reflect.Append(results, newElem)
				} else {
					results = reflect.Append(results, reflect.Indirect(newElem))
				}
			}
		}
	}

	reflect.Indirect(*ref).Set(results)
	return nil
}
<commit_msg>Using sink to control the query flow<commit_after>package storm

import (
	"reflect"

	"github.com/asdine/storm/q"
	"github.com/boltdb/bolt"
)

// Query is the low level query engine used by Storm. It allows to operate searches through an entire bucket.
type Query interface {
	// Skip matching records by the given number
	Skip(int) Query

	// Limit the results by the given number
	Limit(int) Query

	// Find a list of matching records
	Find(interface{}) error
}

func newQuery(n *Node, tree q.Matcher) *query {
	return &query{
		skip:  0,
		limit: -1,
		node:  n,
		tree:  tree,
	}
}

type query struct {
	limit int
	skip  int
	tree  q.Matcher
	node  *Node
}

func (q *query) Skip(nb int) Query {
	q.skip = nb
	return q
}

func (q *query) Limit(nb int) Query {
	q.limit = nb
	return q
}

func (q *query) Find(to interface{}) error {
	var err error
	ref := reflect.ValueOf(to)

	if ref.Kind() != reflect.Ptr || reflect.Indirect(ref).Kind() != reflect.Slice {
		return ErrSlicePtrNeeded
	}

	elemType := reflect.Indirect(ref).Type().Elem()

	if elemType.Kind() == reflect.Ptr {
		elemType = elemType.Elem()
	}

	sink := listSink{
		results: reflect.MakeSlice(reflect.Indirect(ref).Type(), 0, 0),
		isPtr:   reflect.Indirect(ref).Type().Elem().Kind() == reflect.Ptr,
		limit:   q.limit,
		skip:    q.skip,
	}

	if q.node.tx != nil {
		err = q.query(q.node.tx, elemType, &sink)
	} else {
		err = q.node.s.Bolt.Update(func(tx *bolt.Tx) error {
			return q.query(tx, elemType, &sink)
		})
	}

	if err != nil {
		return err
	}

	reflect.Indirect(ref).Set(sink.results)
	return nil
}

func (q *query) query(tx *bolt.Tx, elemType reflect.Type, sink sink) error {
	bucket := q.node.GetBucket(tx, elemType.Name())

	if q.limit == 0 {
		return nil
	}

	if bucket != nil {
		c := bucket.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			if v == nil {
				continue
			}

			newElem := reflect.New(elemType)
			err := q.node.s.Codec.Decode(v, newElem.Interface())
			if err != nil {
				return err
			}

			if q.tree.Match(newElem.Interface()) {
				stop, err := sink.add(newElem)
				if err != nil {
					return err
				}
				if stop {
					return nil
				}
			}
		}
	}

	return nil
}

type sink interface {
	add(elem reflect.Value) (bool, error)
}

type listSink struct {
	results reflect.Value
	isPtr   bool
	skip    int
	limit   int
}

func (l *listSink) add(elem reflect.Value) (bool, error) {
	if l.skip > 0 {
		l.skip--
		return false, nil
	}

	if l.limit > 0 {
		l.limit--
	}

	if l.isPtr {
		l.results = reflect.Append(l.results, elem)
	} else {
		l.results = reflect.Append(l.results, reflect.Indirect(elem))
	}

	return l.limit == 0, nil
}
<|endoftext|>


<commit_before>package yorm

import (
	"database/sql"
	"errors"
	"reflect"
)

type querySetter struct {
	table   string
	dests   []interface{}
	columns []column
}

func QueryOne(i interface{}, row *sql.Row) error {
	if row == nil {
		return errors.New("nil row")
	}
	return convertAssignRow(i, row)
}
func QueryList(i interface{}, rows *sql.Rows) error {
	if rows == nil {
		return errors.New("rows nil")
	}
	return convertAssignRows(i, rows)
}

type sqlScanner interface {
	Scan(dest ...interface{}) error
}

var tableMap map[reflect.Kind]*querySetter = make(map[reflect.Kind]*querySetter)

func newQuery(ri reflect.Value) *querySetter {
	if q, ok := tableMap[ri.Kind()]; ok {
		return q
	}
	if ri.Kind() != reflect.Ptr || ri.IsNil() {
		return nil
	}
	q := new(querySetter)
	defer func() {
		tableMap[ri.Kind()] = q
	}()
	table, cs := structToTable(reflect.Indirect(ri).Interface())
	q.table = table
	q.columns = cs
	q.dests = make([]interface{}, len(cs))
	for k, v := range cs {
		q.dests[k] = newPtrInterface(v.typ.Kind())
	}
	return q
}

func newPtrInterface(k reflect.Kind) interface{} {
	var ti interface{}
	switch k {
	case reflect.Int:
		ti = new(int)
	case reflect.Int64:
		ti = new(int64)
	case reflect.String:
		ti = new(string)
	}
	return ti
}

func convertAssignRows(i interface{}, rows *sql.Rows) error {
	typ := reflect.TypeOf(i)
	if typ.Kind() != reflect.Ptr {
		return errors.New("not ptr")
	}
	typ = typ.Elem()
	if typ.Kind() != reflect.Slice {
		return errors.New("need a slice container")
	}
	typ = typ.Elem()
	var q *querySetter
	if typ.Kind() == reflect.Struct {
		q = newQuery(reflect.New(typ))
		if q == nil {
			return errors.New("q is not support")
		}
	}
	size := 0
	v := reflect.Indirect(reflect.ValueOf(i))
	ti := newPtrInterface(typ.Kind())
	for rows.Next() {
		if size >= v.Cap() {
			newCap := v.Cap()
			if newCap == 0 {
				newCap = 1
			} else {
				if newCap < 1024 {
					newCap += newCap
				} else {
					newCap += newCap / 4
				}
			}
			nv := reflect.MakeSlice(v.Type(), v.Len(), newCap)
			reflect.Copy(nv, v)
			v.Set(nv)
		}
		if size >= v.Len() {
			v.SetLen(size + 1)
		}
		st := reflect.New(typ)
		st = st.Elem()
		if q != nil {
			scanValue(rows, q, st)
		} else {
			rows.Scan(ti)
			st.Set(reflect.ValueOf(ti).Elem())
		}
		v.Index(size).Set(st)
		size++
	}
	return nil
}
func convertAssignRow(i interface{}, row *sql.Row) error {
	typ := reflect.TypeOf(i)

	if typ.Kind() == reflect.Ptr && typ.Kind() != reflect.Struct {
		return row.Scan(i)
	}

	q := newQuery(reflect.ValueOf(i))
	if q == nil {
		return errors.New("nil struct")
	}
	st := reflect.ValueOf(i).Elem()
	return scanValue(row, q, st)
}

func scanValue(sc sqlScanner, q *querySetter, st reflect.Value) error {
	err := sc.Scan(q.dests...)
	if err != nil {
		return err
	}
	for idx, c := range q.columns {
		// different assign func here
		switch c.typ.Kind() {
		case reflect.Int:
			st.Field(c.fieldNum).SetInt(int64(*(q.dests[idx].(*int))))
		case reflect.Int64:
			st.Field(c.fieldNum).SetInt(int64(*(q.dests[idx].(*int64))))
		case reflect.String:
			st.Field(c.fieldNum).SetString(string(*(q.dests[idx].(*string))))
		}
	}
	return nil
}
<commit_msg>name refactor<commit_after>package yorm

import (
	"database/sql"
	"errors"
	"reflect"
)

type querySetter struct {
	table   string
	dests   []interface{}
	columns []column
}

var sqlDb *sql.DB
var tableMap map[reflect.Kind]*querySetter = make(map[reflect.Kind]*querySetter)
var stmtMap map[string]*sql.Stmt = make(map[string]*sql.Stmt)

const (
	MYSQL = "mysql"
)

func Register(dbPath string) error {
	var err error
	sqlDb, err = sql.Open(MYSQL, dbPath)
	if sqlDb == nil {
		return err
	}
	return sqlDb.Ping()
}

func QueryOne(i interface{}, row *sql.Row) error {
	if row == nil {
		return errors.New("nil row")
	}
	return convertAssignRow(i, row)
}
func QueryList(i interface{}, rows *sql.Rows) error {
	if rows == nil {
		return errors.New("rows nil")
	}
	return convertAssignRows(i, rows)
}

func Query(i interface{}, query string, args ...interface{}) error {
	typ := reflect.TypeOf(i)
	if typ.Kind() != reflect.Ptr {
		return errors.New("return value must be ptr to modify")
	}
	typ = typ.Elem()
	stmt := stmtMap[query]
	var err error
	if stmt == nil {
		stmt, err = sqlDb.Prepare(query)
		if stmt == nil {
			return err
		}
		stmtMap[query] = stmt
	}
	if typ.Kind() == reflect.Slice {
		rows, err := stmt.Query(args...)
		if rows == nil {
			return err
		}
		return QueryList(i, rows)
	} else {
		return QueryOne(i, stmt.QueryRow(args...))
	}
	return nil
}

type sqlScanner interface {
	Scan(dest ...interface{}) error
}

func newQuery(ri reflect.Value) *querySetter {
	if q, ok := tableMap[ri.Kind()]; ok {
		return q
	}
	if ri.Kind() != reflect.Ptr || ri.IsNil() {
		return nil
	}
	q := new(querySetter)
	defer func() {
		tableMap[ri.Kind()] = q
	}()
	table, cs := structToTable(reflect.Indirect(ri).Interface())
	q.table = table
	q.columns = cs
	q.dests = make([]interface{}, len(cs))
	for k, v := range cs {
		q.dests[k] = newPtrInterface(v.typ.Kind())
	}
	return q
}

func newPtrInterface(k reflect.Kind) interface{} {
	var ti interface{}
	switch k {
	case reflect.Int:
		ti = new(int)
	case reflect.Int64:
		ti = new(int64)
	case reflect.String:
		ti = new(string)
	}
	return ti
}

func convertAssignRows(i interface{}, rows *sql.Rows) error {
	typ := reflect.TypeOf(i)
	if typ.Kind() != reflect.Ptr {
		return errors.New("not ptr")
	}
	typ = typ.Elem()
	if typ.Kind() != reflect.Slice {
		return errors.New("need a slice container")
	}
	typ = typ.Elem()
	var q *querySetter
	if typ.Kind() == reflect.Struct {
		q = newQuery(reflect.New(typ))
		if q == nil {
			return errors.New("q is not support")
		}
	}
	size := 0
	v := reflect.Indirect(reflect.ValueOf(i))
	ti := newPtrInterface(typ.Kind())
	for rows.Next() {
		if size >= v.Cap() {
			newCap := v.Cap()
			if newCap == 0 {
				newCap = 1
			} else {
				if newCap < 1024 {
					newCap += newCap
				} else {
					newCap += newCap / 4
				}
			}
			nv := reflect.MakeSlice(v.Type(), v.Len(), newCap)
			reflect.Copy(nv, v)
			v.Set(nv)
		}
		if size >= v.Len() {
			v.SetLen(size + 1)
		}
		st := reflect.New(typ)
		st = st.Elem()
		if q != nil {
			err := scanValue(rows, q, st)
			if err != nil {
				return err
			}
		} else {
			rows.Scan(ti)
			st.Set(reflect.ValueOf(ti).Elem())
		}
		v.Index(size).Set(st)
		size++
	}
	return nil
}
func convertAssignRow(i interface{}, row *sql.Row) error {
	typ := reflect.TypeOf(i)

	if typ.Kind() == reflect.Ptr && typ.Kind() != reflect.Struct {
		return row.Scan(i)
	}

	q := newQuery(reflect.ValueOf(i))
	if q == nil {
		return errors.New("nil struct")
	}
	st := reflect.ValueOf(i).Elem()
	return scanValue(row, q, st)
}

func scanValue(sc sqlScanner, q *querySetter, st reflect.Value) error {
	err := sc.Scan(q.dests...)
	if err != nil {
		return err
	}
	for idx, c := range q.columns {
		// different assign func here
		switch c.typ.Kind() {
		case reflect.Int:
			st.Field(c.fieldNum).SetInt(int64(*(q.dests[idx].(*int))))
		case reflect.Int64:
			st.Field(c.fieldNum).SetInt(int64(*(q.dests[idx].(*int64))))
		case reflect.String:
			st.Field(c.fieldNum).SetString(string(*(q.dests[idx].(*string))))
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package querybuilder

type QueryBuilder interface {
	AddCondition(Condition)
	AddOrder(Order)
	AddOrderIfNotExist(Order)
	Copy(QueryBuilder)
	SetOffset(int)
	SetLimit(int)

	Remove(string)
	RemoveByPrefix(string)

	Conditions() []Condition
	HasConditions() bool
	HasOrders() bool
	Limit() int
	Offset() int
	Orders() []Order
	Map() map[string]interface{}
	String(exclude ...string) string
	StringifyOrder() string
}

type querybuilder struct {
	conditions    []Condition
	orders        []Order
	offset, limit int
}

func NewQuerybuilder(offset, limit int) QueryBuilder {
	return &querybuilder{offset: offset, limit: limit}
}

func (qb *querybuilder) AddCondition(c Condition) {
	qb.conditions = append(qb.conditions, c)
}

func (qb *querybuilder) AddOrder(o Order) {
	qb.orders = append(qb.orders, o)
}

func (qb *querybuilder) AddOrderIfNotExist(o Order) {
	for _, order := range qb.orders {
		if order.Field() == o.Field() {
			return
		}
	}
	qb.AddOrder(o)
}

func (qb *querybuilder) Copy(another_qb QueryBuilder) {
	for _, order := range another_qb.Orders() {
		qb.AddOrderIfNotExist(order)
	}
	for _, condition := range another_qb.Conditions() {
		qb.AddCondition(condition)
	}
}

func (qb *querybuilder) SetOffset(offset int) {
	qb.offset = offset
}

func (qb *querybuilder) SetLimit(limit int) {
	qb.limit = limit
}

func (qb *querybuilder) Remove(f string) {
	var conditions []Condition
	for _, condition := range qb.Conditions() {
		if condition.Field() != f {
			conditions = append(conditions, condition)
		}
	}
	qb.conditions = conditions
}

func (qb *querybuilder) RemoveByPrefix(p string) {
	length := len(p)
	var conditions []Condition
	for _, condition := range qb.Conditions() {
		switch {
		case len(condition.Field()) < length:
			conditions = append(conditions, condition)
		case condition.Field()[:length] != p:
			conditions = append(conditions, condition)
		}
	}
	qb.conditions = conditions
}

func (qb *querybuilder) Conditions() []Condition {
	return qb.conditions
}

func (qb *querybuilder) Orders() []Order {
	return qb.orders
}

func (qb *querybuilder) HasConditions() bool {
	return len(qb.conditions) > 0
}

func (qb *querybuilder) HasOrders() bool {
	return len(qb.orders) > 0
}

func (qb *querybuilder) Limit() int {
	return qb.limit
}

func (qb *querybuilder) Offset() int {
	return qb.offset
}

func (qb *querybuilder) Map() map[string]interface{} {
	return map[string]interface{}{
		"conditions": qb.conditions,
		"orders":     qb.orders,
		"limit":      qb.limit,
		"offset":     qb.offset,
	}
}

func (qb *querybuilder) String(exclude ...string) string {
	var str string
	for _, condition := range qb.conditions {
		if !isExist(condition.Field(), exclude) {
			str += condition.String((str != ""))
		}
	}
	return str
}

func (qb *querybuilder) StringifyOrder() string {
	var str string
	for _, order := range qb.orders {
		str += order.String((str != ""))
	}
	return str
}

func isExist(f string, v []string) bool {
	for _, s := range v {
		if s == f {
			return true
		}
	}
	return false
}
<commit_msg>add number<commit_after>package querybuilder

type QueryBuilder interface {
	AddCondition(Condition)
	AddOrder(Order)
	AddOrderIfNotExist(Order)
	Copy(QueryBuilder)
	SetOffset(int)
	SetLimit(int)
	SetNumber(int)

	Remove(string)
	RemoveByPrefix(string)

	Conditions() []Condition
	HasConditions() bool
	HasOrders() bool
	Limit() int
	Number() int
	Offset() int
	Orders() []Order
	Map() map[string]interface{}
	String(exclude ...string) string
	StringifyOrder() string
}

type querybuilder struct {
	conditions            []Condition
	orders                []Order
	limit, number, offset int
}

func NewQuerybuilder(offset, limit int) QueryBuilder {
	return &querybuilder{offset: offset, limit: limit}
}

func (qb *querybuilder) AddCondition(c Condition) {
	qb.conditions = append(qb.conditions, c)
}

func (qb *querybuilder) AddOrder(o Order) {
	qb.orders = append(qb.orders, o)
}

func (qb *querybuilder) AddOrderIfNotExist(o Order) {
	for _, order := range qb.orders {
		if order.Field() == o.Field() {
			return
		}
	}
	qb.AddOrder(o)
}

func (qb *querybuilder) Copy(another_qb QueryBuilder) {
	for _, order := range another_qb.Orders() {
		qb.AddOrderIfNotExist(order)
	}
	for _, condition := range another_qb.Conditions() {
		qb.AddCondition(condition)
	}
}

func (qb *querybuilder) SetLimit(limit int) {
	qb.limit = limit
}

func (qb *querybuilder) SetNumber(number int) {
	qb.number = number
}

func (qb *querybuilder) SetOffset(offset int) {
	qb.offset = offset
}

func (qb *querybuilder) Remove(f string) {
	var conditions []Condition
	for _, condition := range qb.Conditions() {
		if condition.Field() != f {
			conditions = append(conditions, condition)
		}
	}
	qb.conditions = conditions
}

func (qb *querybuilder) RemoveByPrefix(p string) {
	length := len(p)
	var conditions []Condition
	for _, condition := range qb.Conditions() {
		switch {
		case len(condition.Field()) < length:
			conditions = append(conditions, condition)
		case condition.Field()[:length] != p:
			conditions = append(conditions, condition)
		}
	}
	qb.conditions = conditions
}

func (qb *querybuilder) Conditions() []Condition {
	return qb.conditions
}

func (qb *querybuilder) Orders() []Order {
	return qb.orders
}

func (qb *querybuilder) HasConditions() bool {
	return len(qb.conditions) > 0
}

func (qb *querybuilder) HasOrders() bool {
	return len(qb.orders) > 0
}

func (qb *querybuilder) Number() int {
	return qb.number
}

func (qb *querybuilder) Offset() int {
	return qb.offset
}

func (qb *querybuilder) Limit() int {
	return qb.limit
}

func (qb *querybuilder) Map() map[string]interface{} {
	return map[string]interface{}{
		"conditions": qb.conditions,
		"orders":     qb.orders,
		"limit":      qb.limit,
		"offset":     qb.offset,
	}
}

func (qb *querybuilder) String(exclude ...string) string {
	var str string
	for _, condition := range qb.conditions {
		if !isExist(condition.Field(), exclude) {
			str += condition.String((str != ""))
		}
	}
	return str
}

func (qb *querybuilder) StringifyOrder() string {
	var str string
	for _, order := range qb.orders {
		str += order.String((str != ""))
	}
	return str
}

func isExist(f string, v []string) bool {
	for _, s := range v {
		if s == f {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package eventhub

import (
	"log"
	"strconv"
	"strings"
	"unicode"
)

type MatchArray [2][]string

// Given two string arrays, returns true if the
// value array contains all of the query array's values
func (m MatchArray) Match() bool {

	qArr := m[0]
	eArr := m[1]

	//both are empty, ignore
	if len(qArr) == 0 && len(eArr) == 0 {
		return true
	}

	if len(qArr) > 0 {
		allMatch := true
		for _, s := range qArr {
			if !stringInSlice(s, eArr) {
				allMatch = false
			}
		}
		if !allMatch {
			return false
		}
	}
	return true

}

// Returns true if a string is in the slice
func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

// Returns true if any of the OR delimited q string matches e:
//     orMatchStringAny("foo OR bar or baz", "baz")  // returns true
//     orMatchStringAny("foo", "foo")  // returns true
//     orMatchStringAny("  foo  ", "foo")  // returns true
//     orMatchStringAny("foo OR bar or baz", "moo")  // returns false
func orMatchStringAny(q, e string) bool {

	for _, s := range strings.Split(q, "OR") {
		s = strings.TrimSpace(s)
		if e == s {
			return true
		}
	}
	return false
}

//TODO: Created/Updated, lt & gt
//TODO: Importance, 1+OR+3 gt1 lt4. single param
//TODO: Sort & direction
type Query struct {
	Origin          string   `url:"origin,omitempty" schema:"origin" json:"origin"`
	Key             string   `url:"key,omitempty" schema:"key" json:"key"`
	Entities        []string `url:"entities,omitempty" schema:"entities" json:"entities"`
	OtherReferences []string `url:"other_references,omitempty" schema:"other_references" json:"other_references"`
	Actors          []string `url:"actors,omitempty" schema:"actors" json:"actors"`
	Tags            []string `url:"tags,omitempty" schema:"tags" json:"tags"`
	Importance      string   `url:"importance,omitempty" schema:"importance" json:"importance"`
}

// Return true if s is a valid array type
func (q *Query) IsValidArrayType(s string) bool {
	return stringInSlice(s, []string{"entities", "other_references", "actors", "tags"})
}

// Returns true if the query values are empty
func (q *Query) IsEmpty() bool {

	for _, s := range []string{q.Origin, q.Key, q.Importance} {
		if s != "" {
			return false
		}
	}
	for _, arr := range [][]string{q.Entities, q.OtherReferences, q.Actors, q.Tags} {
		if len(arr) > 0 {
			return false
		}
	}
	return true

}

func (q *Query) matchImportance(i int) bool {

	if q.Importance == "" {
		return true
	}

	val, err := strconv.Atoi(strings.TrimFunc(q.Importance, unicode.IsLetter))
	if err != nil {
		//maybe don't fatal
		log.Fatal(err)
		return false
	}

	switch strings.TrimFunc(q.Importance, unicode.IsDigit) {
	case "gte":
		return (i >= val)
	case "gt":
		return (i > val)
	case "lte":
		return (i <= val)
	case "lt":
		return (i < val)
	}
	return (val == i)

}

// Determines whether an event matched the query
func (q *Query) Match(e Event) bool {

	// empty query matches everything
	if q.IsEmpty() {
		return true
	}

	orPairs := [][2]string{
		[2]string{q.Origin, e.Origin},
		[2]string{q.Key, e.Key},
	}

	for _, pair := range orPairs {

		//should orMatchStringAny return true if q (pair[0]) is empty?
		if pair[0] != "" && !orMatchStringAny(pair[0], pair[1]) {
			return false
		}

	}

	arrays := []MatchArray{
		{q.Entities, e.Entities},
		{q.OtherReferences, e.OtherReferences},
		{q.Actors, e.Actors},
		{q.Tags, e.Tags},
	}

	for _, ma := range arrays {
		if !ma.Match() {
			return false
		}
	}

	return q.matchImportance(e.Importance)
}
<commit_msg>Added To & From filters<commit_after>package eventhub

import (
	"log"
	"strconv"
	"strings"
	"time"
	"unicode"
)

type MatchArray [2][]string

// Given two string arrays, returns true if the
// value array contains all of the query array's values
func (m MatchArray) Match() bool {

	qArr := m[0]
	eArr := m[1]

	//both are empty, ignore
	if len(qArr) == 0 && len(eArr) == 0 {
		return true
	}

	if len(qArr) > 0 {
		allMatch := true
		for _, s := range qArr {
			if !stringInSlice(s, eArr) {
				allMatch = false
			}
		}
		if !allMatch {
			return false
		}
	}
	return true

}

// Returns true if a string is in the slice
func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

// Returns true if any of the OR delimited q string matches e:
//     orMatchStringAny("foo OR bar or baz", "baz")  // returns true
//     orMatchStringAny("foo", "foo")  // returns true
//     orMatchStringAny("  foo  ", "foo")  // returns true
//     orMatchStringAny("foo OR bar or baz", "moo")  // returns false
func orMatchStringAny(q, e string) bool {

	for _, s := range strings.Split(q, "OR") {
		s = strings.TrimSpace(s)
		if e == s {
			return true
		}
	}
	return false
}

//TODO: Created/Updated, lt & gt
//TODO: Importance, 1+OR+3 gt1 lt4. single param
//TODO: Sort & direction
type Query struct {
	Origin          string    `url:"origin,omitempty" schema:"origin" json:"origin"`
	Key             string    `url:"key,omitempty" schema:"key" json:"key"`
	Entities        []string  `url:"entities,omitempty" schema:"entities" json:"entities"`
	OtherReferences []string  `url:"other_references,omitempty" schema:"other_references" json:"other_references"`
	Actors          []string  `url:"actors,omitempty" schema:"actors" json:"actors"`
	Tags            []string  `url:"tags,omitempty" schema:"tags" json:"tags"`
	Importance      string    `url:"importance,omitempty" schema:"importance" json:"importance"`
	From            time.Time `url:"from,omitempty" schema:"from" json:"from"`
	To              time.Time `url:"to,omitempty" schema:"to" json:"to"`
}

// Return true if s is a valid array type
func (q *Query) IsValidArrayType(s string) bool {
	return stringInSlice(s, []string{"entities", "other_references", "actors", "tags"})
}

// Returns true if the query values are empty
func (q *Query) IsEmpty() bool {

	for _, s := range []string{q.Origin, q.Key, q.Importance} {
		if s != "" {
			return false
		}
	}
	for _, arr := range [][]string{q.Entities, q.OtherReferences, q.Actors, q.Tags} {
		if len(arr) > 0 {
			return false
		}
	}

	if !q.From.IsZero() {
		return false
	}

	if !q.To.IsZero() {
		return false
	}

	return true

}

func (q *Query) matchImportance(i int) bool {

	if q.Importance == "" {
		return true
	}

	val, err := strconv.Atoi(strings.TrimFunc(q.Importance, unicode.IsLetter))
	if err != nil {
		//maybe don't fatal
		log.Fatal(err)
		return false
	}

	switch strings.TrimFunc(q.Importance, unicode.IsDigit) {
	case "gte":
		return (i >= val)
	case "gt":
		return (i > val)
	case "lte":
		return (i <= val)
	case "lt":
		return (i < val)
	}
	return (val == i)

}

// Determines whether an event matched the query
func (q *Query) Match(e Event) bool {

	// empty query matches everything
	if q.IsEmpty() {
		return true
	}

	orPairs := [][2]string{
		[2]string{q.Origin, e.Origin},
		[2]string{q.Key, e.Key},
	}

	for _, pair := range orPairs {

		//should orMatchStringAny return true if q (pair[0]) is empty?
		if pair[0] != "" && !orMatchStringAny(pair[0], pair[1]) {
			return false
		}

	}

	arrays := []MatchArray{
		{q.Entities, e.Entities},
		{q.OtherReferences, e.OtherReferences},
		{q.Actors, e.Actors},
		{q.Tags, e.Tags},
	}

	for _, ma := range arrays {
		if !ma.Match() {
			return false
		}
	}

	if !q.matchImportance(e.Importance) {
		return false
	}

	if !q.From.IsZero() && e.Created.UnixNano() < q.From.UnixNano() {
		return false
	}

	if !q.To.IsZero() && e.Created.UnixNano() > q.To.UnixNano() {
		return false
	}

	return true
}
<|endoftext|>


<commit_before>package zenodb

import (
	"context"
	"errors"
	"fmt"
	"time"

	"github.com/getlantern/bytemap"

	"github.com/getlantern/zenodb/common"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/encoding"
	"github.com/getlantern/zenodb/planner"
)

var (
	ErrOutOfMemory = errors.New("out of memory")
)

func (db *DB) Query(sqlString string, isSubQuery bool, subQueryResults [][]interface{}, includeMemStore bool) (core.FlatRowSource, error) {
	opts := &planner.Opts{
		GetTable: func(table string, outFields func(tableFields core.Fields) (core.Fields, error)) (planner.Table, error) {
			return db.getQueryable(table, outFields, includeMemStore)
		},
		Now:             db.now,
		IsSubQuery:      isSubQuery,
		SubQueryResults: subQueryResults,
	}
	if db.opts.Passthrough {
		opts.QueryCluster = func(ctx context.Context, sqlString string, isSubQuery bool, subQueryResults [][]interface{}, unflat bool, onFields core.OnFields, onRow core.OnRow, onFlatRow core.OnFlatRow) (interface{}, error) {
			return db.queryCluster(ctx, sqlString, isSubQuery, subQueryResults, includeMemStore, unflat, onFields, onRow, onFlatRow)
		}
	}
	plan, err := planner.Plan(sqlString, opts)
	if err != nil {
		return nil, err
	}
	log.Debugf("\n------------ Query Plan ------------\n\n%v\n\n%v\n----------- End Query Plan ----------", sqlString, core.FormatSource(plan))
	return plan, nil
}

func (db *DB) getQueryable(table string, outFields func(tableFields core.Fields) (core.Fields, error), includeMemStore bool) (*queryable, error) {
	t := db.getTable(table)
	if t == nil {
		return nil, fmt.Errorf("Table %v not found", table)
	}
	if t.Virtual {
		return nil, fmt.Errorf("Table %v is virtual and cannot be queried", table)
	}
	until := encoding.RoundTimeUp(db.clock.Now(), t.Resolution)
	asOf := encoding.RoundTimeUp(until.Add(-1*t.RetentionPeriod), t.Resolution)
	fields := t.getFields()
	out, err := outFields(fields)
	if err != nil {
		return nil, err
	}
	if out == nil {
		out = t.getFields()
	}
	return &queryable{db, t, out, asOf, until, includeMemStore}, nil
}

func MetaDataFor(source core.FlatRowSource, fields core.Fields) *common.QueryMetaData {
	return &common.QueryMetaData{
		FieldNames: fields.Names(),
		AsOf:       source.GetAsOf(),
		Until:      source.GetUntil(),
		Resolution: source.GetResolution(),
		Plan:       core.FormatSource(source),
	}
}

type queryable struct {
	db              *DB
	t               *table
	fields          core.Fields
	asOf            time.Time
	until           time.Time
	includeMemStore bool
}

func (q *queryable) GetGroupBy() []core.GroupBy {
	return q.t.GroupBy
}

func (q *queryable) GetResolution() time.Duration {
	return q.t.Resolution
}

func (q *queryable) GetAsOf() time.Time {
	return q.asOf
}

func (q *queryable) GetUntil() time.Time {
	return q.until
}

func (q *queryable) GetPartitionBy() []string {
	return q.t.PartitionBy
}

func (q *queryable) String() string {
	return q.t.Name
}

func (q *queryable) Iterate(ctx context.Context, onFields core.OnFields, onRow core.OnRow) (interface{}, error) {
	// We report all fields from the table
	err := onFields(q.fields)
	if err != nil {
		return nil, err
	}

	i := 1
	// When iterating, as an optimization, we read only the needed fields (not
	// all table fields).
	highWaterMark, err := q.t.iterate(ctx, q.fields, q.includeMemStore, func(key bytemap.ByteMap, vals []encoding.Sequence) (bool, error) {
		if i%1000 == 0 {
			// every 1000 rows, check and cap memory size
			if !q.db.capMemorySize(false) {
				log.Error("Returning ErrOutOfMemory")
				return false, ErrOutOfMemory
			}
		}
		i++
		return onRow(key, vals)
	})
	if err != nil {
		log.Errorf("Error on iterating: %v", err)
	}
	numSuccessfulPartitions := 0
	if err == nil {
		numSuccessfulPartitions = 1
	}
	return &common.QueryStats{
		NumPartitions:           1,
		NumSuccessfulPartitions: numSuccessfulPartitions,
		LowestHighWaterMark:     common.TimeToMillis(highWaterMark),
		HighestHighWaterMark:    common.TimeToMillis(highWaterMark),
	}, err
}
<commit_msg>Reporting error if query didn't select any fields<commit_after>package zenodb

import (
	"context"
	"errors"
	"fmt"
	"time"

	"github.com/getlantern/bytemap"

	"github.com/getlantern/zenodb/common"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/encoding"
	"github.com/getlantern/zenodb/planner"
)

var (
	ErrOutOfMemory = errors.New("out of memory")
)

func (db *DB) Query(sqlString string, isSubQuery bool, subQueryResults [][]interface{}, includeMemStore bool) (core.FlatRowSource, error) {
	opts := &planner.Opts{
		GetTable: func(table string, outFields func(tableFields core.Fields) (core.Fields, error)) (planner.Table, error) {
			return db.getQueryable(table, outFields, includeMemStore)
		},
		Now:             db.now,
		IsSubQuery:      isSubQuery,
		SubQueryResults: subQueryResults,
	}
	if db.opts.Passthrough {
		opts.QueryCluster = func(ctx context.Context, sqlString string, isSubQuery bool, subQueryResults [][]interface{}, unflat bool, onFields core.OnFields, onRow core.OnRow, onFlatRow core.OnFlatRow) (interface{}, error) {
			return db.queryCluster(ctx, sqlString, isSubQuery, subQueryResults, includeMemStore, unflat, onFields, onRow, onFlatRow)
		}
	}
	plan, err := planner.Plan(sqlString, opts)
	if err != nil {
		return nil, err
	}
	log.Debugf("\n------------ Query Plan ------------\n\n%v\n\n%v\n----------- End Query Plan ----------", sqlString, core.FormatSource(plan))
	return plan, nil
}

func (db *DB) getQueryable(table string, outFields func(tableFields core.Fields) (core.Fields, error), includeMemStore bool) (*queryable, error) {
	t := db.getTable(table)
	if t == nil {
		return nil, fmt.Errorf("Table %v not found", table)
	}
	if t.Virtual {
		return nil, fmt.Errorf("Table %v is virtual and cannot be queried", table)
	}
	until := encoding.RoundTimeUp(db.clock.Now(), t.Resolution)
	asOf := encoding.RoundTimeUp(until.Add(-1*t.RetentionPeriod), t.Resolution)
	fields := t.getFields()
	out, err := outFields(fields)
	if err != nil {
		return nil, err
	}
	if out == nil {
		out = t.getFields()
	}
	return &queryable{db, t, out, asOf, until, includeMemStore}, nil
}

func MetaDataFor(source core.FlatRowSource, fields core.Fields) *common.QueryMetaData {
	return &common.QueryMetaData{
		FieldNames: fields.Names(),
		AsOf:       source.GetAsOf(),
		Until:      source.GetUntil(),
		Resolution: source.GetResolution(),
		Plan:       core.FormatSource(source),
	}
}

type queryable struct {
	db              *DB
	t               *table
	fields          core.Fields
	asOf            time.Time
	until           time.Time
	includeMemStore bool
}

func (q *queryable) GetGroupBy() []core.GroupBy {
	return q.t.GroupBy
}

func (q *queryable) GetResolution() time.Duration {
	return q.t.Resolution
}

func (q *queryable) GetAsOf() time.Time {
	return q.asOf
}

func (q *queryable) GetUntil() time.Time {
	return q.until
}

func (q *queryable) GetPartitionBy() []string {
	return q.t.PartitionBy
}

func (q *queryable) String() string {
	return q.t.Name
}

func (q *queryable) Iterate(ctx context.Context, onFields core.OnFields, onRow core.OnRow) (interface{}, error) {
	// We report all fields from the table
	err := onFields(q.fields)
	if err != nil {
		return nil, err
	}

	if len(q.fields) == 0 {
		return nil, errors.New("No fields found!")
	}

	i := 1
	// When iterating, as an optimization, we read only the needed fields (not
	// all table fields).
	highWaterMark, err := q.t.iterate(ctx, q.fields, q.includeMemStore, func(key bytemap.ByteMap, vals []encoding.Sequence) (bool, error) {
		if i%1000 == 0 {
			// every 1000 rows, check and cap memory size
			if !q.db.capMemorySize(false) {
				log.Error("Returning ErrOutOfMemory")
				return false, ErrOutOfMemory
			}
		}
		i++
		return onRow(key, vals)
	})
	if err != nil {
		log.Errorf("Error on iterating: %v", err)
	}
	numSuccessfulPartitions := 0
	if err == nil {
		numSuccessfulPartitions = 1
	}
	return &common.QueryStats{
		NumPartitions:           1,
		NumSuccessfulPartitions: numSuccessfulPartitions,
		LowestHighWaterMark:     common.TimeToMillis(highWaterMark),
		HighestHighWaterMark:    common.TimeToMillis(highWaterMark),
	}, err
}
<|endoftext|>


<commit_before>package rpc

import (
	"fmt"
	"io"
	"net"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
	"golang.org/x/net/context"
)

var testPort int = 8089

type longCallResult struct {
	res interface{}
	err error
}

func prepServer(listener chan error) error {
	server := &server{port: testPort}

	serverReady := make(chan struct{})
	var err error
	go func() {
		err = server.Run(serverReady, listener)
	}()
	<-serverReady
	// TODO: Fix the race here -- when serverReady is closed by
	// server.Run, err is in an indeterminate state.
	return err
}

func prepClient(t *testing.T) (TestClient, net.Conn) {
	c, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", testPort))
	require.Nil(t, err, "a dialer error occurred")

	xp := NewTransport(c, nil, nil)
	return TestClient{GenericClient: NewClient(xp, nil, nil)}, c
}

func prepTest(t *testing.T) (TestClient, chan error, net.Conn) {
	listener := make(chan error)
	prepServer(listener)
	cli, conn := prepClient(t)
	return cli, listener, conn
}

func endTest(t *testing.T, c net.Conn, listener chan error) {
	c.Close()
	err := <-listener
	require.EqualError(t, err, io.EOF.Error(), "expected EOF")
}

func TestCall(t *testing.T) {
	cli, listener, conn := prepTest(t)
	defer endTest(t, conn, listener)

	B := 34
	for A := 10; A < 23; A += 2 {
		res, err := cli.Add(context.Background(), AddArgs{A: A, B: B})
		require.Nil(t, err, "an error occurred while adding parameters")
		require.Equal(t, A+B, res, "Result should be the two parameters added together")
	}
}

func TestBrokenCall(t *testing.T) {
	cli, listener, conn := prepTest(t)
	defer endTest(t, conn, listener)

	err := cli.BrokenMethod()
	require.EqualError(t, err, "method 'broken' not found in protocol 'test.1.testp'")

	err = cli.BrokenProtocol()
	require.EqualError(t, err, "protocol not found: test.2.testp")
}

func TestNotify(t *testing.T) {
	cli, listener, conn := prepTest(t)
	defer endTest(t, conn, listener)

	pi := 31415

	err := cli.UpdateConstants(context.Background(), Constants{Pi: pi})
	require.Nil(t, err, "Unexpected error on notify: %v", err)

	constants, err := cli.GetConstants(context.Background())
	require.Nil(t, err, "Unexpected error on GetConstants: %v", err)
	require.Equal(t, pi, constants.Pi, "we set the constant properly via Notify")
}

func TestLongCall(t *testing.T) {
	cli, listener, conn := prepTest(t)
	defer endTest(t, conn, listener)

	longResult, err := cli.LongCall(context.Background())
	require.Nil(t, err, "call should have succeeded")
	require.Equal(t, longResult, 100, "call should have succeeded")
}

func TestLongCallCancel(t *testing.T) {
	cli, listener, conn := prepTest(t)
	defer endTest(t, conn, listener)

	ctx, cancel := context.WithCancel(context.Background())
	ctx = AddRpcTagsToContext(ctx, CtxRpcTags{"hello": []string{"world"}})

	resultCh := make(chan longCallResult)
	runInBg(func() error {
		var longResult interface{}
		var err error
		longResult, err = cli.LongCall(ctx)
		resultCh <- longCallResult{longResult, err}
		longResult, err = cli.LongCallResult(context.Background())
		resultCh <- longCallResult{longResult, err}
		longResult, err = cli.LongCallDebugTags(context.Background())
		resultCh <- longCallResult{longResult, err}
		return nil
	})
	// TODO figure out a way to avoid this sleep
	time.Sleep(time.Millisecond)
	cancel()
	res := <-resultCh
	require.EqualError(t, res.err, context.Canceled.Error())
	require.Equal(t, 0, res.res, "call should be canceled")

	res = <-resultCh
	require.Nil(t, res.err, "call should have succeeded")
	require.Equal(t, -1, res.res, "canceled call should have set the longCallResult to canceled")

	res = <-resultCh
	require.Nil(t, res.err, "call should have succeeded")
	require.Equal(t, CtxRpcTags{"hello": []interface{}{"world"}}, res.res, "canceled call should have set the debug tags")
}

func TestClosedConnection(t *testing.T) {
	cli, listener, conn := prepTest(t)
	defer endTest(t, conn, listener)

	resultCh := make(chan longCallResult)
	runInBg(func() error {
		var longResult interface{}
		var err error
		longResult, err = cli.LongCall(context.Background())
		resultCh <- longCallResult{longResult, err}
		return nil
	})
	// TODO figure out a way to avoid this sleep
	time.Sleep(time.Millisecond)
	conn.Close()
	res := <-resultCh
	require.EqualError(t, res.err, io.EOF.Error())
	require.Equal(t, 0, res.res)
}
<commit_msg>protocol_test: Make a new test<commit_after>package rpc

import (
	"fmt"
	"io"
	"net"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
	"golang.org/x/net/context"
    telnet "github.com/reiver/go-telnet"
)

var testPort int = 8089
var testHostPort = fmt.Sprintf("127.0.0.1:%d", testPort)

type longCallResult struct {
	res interface{}
	err error
}

func prepServer(listener chan error) error {
	server := &server{port: testPort}

	serverReady := make(chan struct{})
	var err error
	go func() {
		err = server.Run(serverReady, listener)
	}()
	<-serverReady
	// TODO: Fix the race here -- when serverReady is closed by
	// server.Run, err is in an indeterminate state.
	return err
}

func prepClient(t *testing.T) (TestClient, net.Conn) {
	c, err := net.Dial("tcp", testHostPort)
	require.Nil(t, err, "a dialer error occurred")

	xp := NewTransport(c, nil, nil)
	return TestClient{GenericClient: NewClient(xp, nil, nil)}, c
}

func prepTest(t *testing.T) (TestClient, chan error, net.Conn) {
	listener := make(chan error)
	prepServer(listener)
	cli, conn := prepClient(t)
	return cli, listener, conn
}

func endTest(t *testing.T, c net.Conn, listener chan error) {
	c.Close()
	err := <-listener
	require.EqualError(t, err, io.EOF.Error(), "expected EOF")
}

func TestCall(t *testing.T) {
	cli, listener, conn := prepTest(t)
	defer endTest(t, conn, listener)

	B := 34
	for A := 10; A < 23; A += 2 {
		res, err := cli.Add(context.Background(), AddArgs{A: A, B: B})
		require.Nil(t, err, "an error occurred while adding parameters")
		require.Equal(t, A+B, res, "Result should be the two parameters added together")
	}
}

func TestBrokenCall(t *testing.T) {
	cli, listener, conn := prepTest(t)
	defer endTest(t, conn, listener)

	err := cli.BrokenMethod()
	require.EqualError(t, err, "method 'broken' not found in protocol 'test.1.testp'")

	err = cli.BrokenProtocol()
	require.EqualError(t, err, "protocol not found: test.2.testp")
}

func TestNotify(t *testing.T) {
	cli, listener, conn := prepTest(t)
	defer endTest(t, conn, listener)

	pi := 31415

	err := cli.UpdateConstants(context.Background(), Constants{Pi: pi})
	require.Nil(t, err, "Unexpected error on notify: %v", err)

	constants, err := cli.GetConstants(context.Background())
	require.Nil(t, err, "Unexpected error on GetConstants: %v", err)
	require.Equal(t, pi, constants.Pi, "we set the constant properly via Notify")
}

func TestLongCall(t *testing.T) {
	cli, listener, conn := prepTest(t)
	defer endTest(t, conn, listener)

	longResult, err := cli.LongCall(context.Background())
	require.Nil(t, err, "call should have succeeded")
	require.Equal(t, longResult, 100, "call should have succeeded")
}

func TestLongCallCancel(t *testing.T) {
	cli, listener, conn := prepTest(t)
	defer endTest(t, conn, listener)

	ctx, cancel := context.WithCancel(context.Background())
	ctx = AddRpcTagsToContext(ctx, CtxRpcTags{"hello": []string{"world"}})

	resultCh := make(chan longCallResult)
	runInBg(func() error {
		var longResult interface{}
		var err error
		longResult, err = cli.LongCall(ctx)
		resultCh <- longCallResult{longResult, err}
		longResult, err = cli.LongCallResult(context.Background())
		resultCh <- longCallResult{longResult, err}
		longResult, err = cli.LongCallDebugTags(context.Background())
		resultCh <- longCallResult{longResult, err}
		return nil
	})
	// TODO figure out a way to avoid this sleep
	time.Sleep(time.Millisecond)
	cancel()
	res := <-resultCh
	require.EqualError(t, res.err, context.Canceled.Error())
	require.Equal(t, 0, res.res, "call should be canceled")

	res = <-resultCh
	require.Nil(t, res.err, "call should have succeeded")
	require.Equal(t, -1, res.res, "canceled call should have set the longCallResult to canceled")

	res = <-resultCh
	require.Nil(t, res.err, "call should have succeeded")
	require.Equal(t, CtxRpcTags{"hello": []interface{}{"world"}}, res.res, "canceled call should have set the debug tags")
}

func TestClosedConnection(t *testing.T) {
	cli, listener, conn := prepTest(t)
	defer endTest(t, conn, listener)

	resultCh := make(chan longCallResult)
	runInBg(func() error {
		var longResult interface{}
		var err error
		longResult, err = cli.LongCall(context.Background())
		resultCh <- longCallResult{longResult, err}
		return nil
	})
	// TODO figure out a way to avoid this sleep
	time.Sleep(time.Millisecond)
	conn.Close()
	res := <-resultCh
	require.EqualError(t, res.err, io.EOF.Error())
	require.Equal(t, 0, res.res)
}

func TestKillClient(t *testing.T) {
	listener := make(chan error)
	prepServer(listener)
    defer func() {
        err := <-listener
        require.EqualError(t, err, io.EOF.Error(), "expected EOF")
    }()

    var cli telnet.Caller = telnet.StandardCaller
    err := telnet.DialToAndCall(testHostPort, caller)
    require.NoError(t, err)
}

<|endoftext|>


<commit_before>package discovery

import (
	"context"
	"testing"
	"time"

	bhost "github.com/libp2p/go-libp2p/p2p/host/basic"

	host "github.com/libp2p/go-libp2p-host"
	netutil "github.com/libp2p/go-libp2p-netutil"

	pstore "github.com/libp2p/go-libp2p-peerstore"
)

type DiscoveryNotifee struct {
	h host.Host
}

func (n *DiscoveryNotifee) HandlePeerFound(pi pstore.PeerInfo) {
	n.h.Connect(context.Background(), pi)
}

func TestMdnsDiscovery(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	a := bhost.New(netutil.GenSwarmNetwork(t, ctx))
	b := bhost.New(netutil.GenSwarmNetwork(t, ctx))

	sa, err := NewMdnsService(ctx, a, time.Second, "someTag")
	if err != nil {
		t.Fatal(err)
	}

	sb, err := NewMdnsService(ctx, b, time.Second, "someTag")
	if err != nil {
		t.Fatal(err)
	}

	_ = sb

	n := &DiscoveryNotifee{a}

	sa.RegisterNotifee(n)

	time.Sleep(time.Second * 2)

	err = a.Connect(ctx, pstore.PeerInfo{ID: b.ID()})
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>Skip TestMdnsDiscovery<commit_after>package discovery

import (
	"context"
	"testing"
	"time"

	bhost "github.com/libp2p/go-libp2p/p2p/host/basic"

	host "github.com/libp2p/go-libp2p-host"
	netutil "github.com/libp2p/go-libp2p-netutil"

	pstore "github.com/libp2p/go-libp2p-peerstore"
)

type DiscoveryNotifee struct {
	h host.Host
}

func (n *DiscoveryNotifee) HandlePeerFound(pi pstore.PeerInfo) {
	n.h.Connect(context.Background(), pi)
}

func TestMdnsDiscovery(t *testing.T) {
	//TODO: re-enable when the new lib will get integrated
	t.Skip("TestMdnsDiscovery fails randomly with current lib")

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	a := bhost.New(netutil.GenSwarmNetwork(t, ctx))
	b := bhost.New(netutil.GenSwarmNetwork(t, ctx))

	sa, err := NewMdnsService(ctx, a, time.Second, "someTag")
	if err != nil {
		t.Fatal(err)
	}

	sb, err := NewMdnsService(ctx, b, time.Second, "someTag")
	if err != nil {
		t.Fatal(err)
	}

	_ = sb

	n := &DiscoveryNotifee{a}

	sa.RegisterNotifee(n)

	time.Sleep(time.Second * 2)

	err = a.Connect(ctx, pstore.PeerInfo{ID: b.ID()})
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package devastator

import (
	"github.com/nbusy/cmap"
	"github.com/nbusy/neptulon/jsonrpc"
)

// Queue is a message queue for queueing and sending messages to users.
type Queue struct {
	conns *cmap.CMap                   // user ID -> conn ID
	route *jsonrpc.Router              // route to send messages through
	reqs  map[string]([]queuedRequest) // user ID -> []queuedRequest
}

// NewQueue creates a new queue object.
func NewQueue(r *jsonrpc.Router) Queue {
	return Queue{
		conns: cmap.New(),
		route: r,
		reqs:  make(map[string]([]queuedRequest)),
	}
}

// SetConn associates a user with a connection by ID.
// If there are pending messages for the user, they are started to be send immediately.
func (q *Queue) SetConn(userID, connID string) {
	q.conns.Set(userID, connID)
	// todo: trigger processing
}

// RemoveConn removes a user's associated connection ID.
func (q *Queue) RemoveConn(userID string) {
	q.conns.Delete(userID)
}

// AddRequest queues a request message to be sent to the given user.
func (q *Queue) AddRequest(userID string, method string, params interface{}, resHandler func(ctx *jsonrpc.ResCtx)) error {
	if connID, ok := q.conns.Get(userID); ok {
		if err := q.route.SendRequest(connID.(string), method, params, resHandler); err != nil {
			return err
		}
	} else {
		r := queuedRequest{Method: method, Params: params}
		if rs, ok := q.reqs[userID]; ok {
			q.reqs[userID] = append(rs, r)
		} else {
			q.reqs[userID] = []queuedRequest{{Method: method, Params: params}}
		}
	}

	return nil
}

type queuedRequest struct {
	Method string
	Params interface{}
}
<commit_msg>queue processor concept<commit_after>package devastator

import (
	"github.com/nbusy/cmap"
	"github.com/nbusy/neptulon/jsonrpc"
)

// Queue is a message queue for queueing and sending messages to users.
type Queue struct {
	conns *cmap.CMap                   // user ID -> conn ID
	route *jsonrpc.Router              // route to send messages through
	reqs  map[string]([]queuedRequest) // user ID -> []queuedRequest
}

// NewQueue creates a new queue object.
func NewQueue(r *jsonrpc.Router) Queue {
	return Queue{
		conns: cmap.New(),
		route: r,
		reqs:  make(map[string]([]queuedRequest)),
	}
}

// SetConn associates a user with a connection by ID.
// If there are pending messages for the user, they are started to be send immediately.
func (q *Queue) SetConn(userID, connID string) {
	q.conns.Set(userID, connID)
	go q.processQueue(userID)
}

// RemoveConn removes a user's associated connection ID.
func (q *Queue) RemoveConn(userID string) {
	q.conns.Delete(userID)
}

// AddRequest queues a request message to be sent to the given user.
func (q *Queue) AddRequest(userID string, method string, params interface{}, resHandler func(ctx *jsonrpc.ResCtx)) error {
	if connID, ok := q.conns.Get(userID); ok {
		if err := q.route.SendRequest(connID.(string), method, params, resHandler); err != nil {
			return err
		}
	} else {
		r := queuedRequest{Method: method, Params: params}
		if rs, ok := q.reqs[userID]; ok {
			q.reqs[userID] = append(rs, r)
		} else {
			q.reqs[userID] = []queuedRequest{{Method: method, Params: params}}
		}
	}

	return nil
}

type queuedRequest struct {
	Method string
	Params interface{}
}

func (q *Queue) processQueue(userID string) {
	// todo: register channel before loop so we can capture message even during initialization

}
<|endoftext|>


<commit_before>/*
queue is a simple Queue system written in Go that will use Redis underneath.
Focus of this design is mainly horisontal scalability via concurrency, paritioning and fault-detection
Queues can be partitions in to more than one Redis is necessary.

Number of redis paritions is set by using Urls function and setting slice of Redis URL connections.
Redis paritioning is required in cases that one redis cannot handle the load because of IO, moemory or in rare situations CPU limitations.

In case of crash record of all incomplete tasks will be kepts in redis as keys with this format
	QUEUE::PENDING::ID
ID will indicate the ID of failed tasks.

To use this library you need to use queue struct.

	var q Queue
	q.Urls([]{redis://localhost:6379})

Adding tasks is done by calling AddTask. This function will accept an ID and the task itself that will be as a string.

	q.AddTask(1, "task1")
	q.AddTask(2, "task2")

ID can be used in a special way. If ID of two tasks are the same while processing AnalysePool will send them to the same analyser goroutine if analyzer waits enough.

	q.AddTask(2, "start")
	q.AddTask(1, "load")
	q.AddTask(2, "load")
	q.AddTask(1, "stop")
	q.AddTask(2, "stop")

This feature can be used in analyser needs to process a set of related tasks one after another.

AnalysePool accepts 3 parameters. One analyzerID that will identify which redis pool this AnalysePool will connect to.

	whichRedis=len(q.urls) % analyzerID

AnalysePool need two closued, analyzer and exitOnEmpty. Format of those closure are as follows.

	analyzer := func(id int, task chan string, success chan bool, next chan bool) {
		for {
			select {
			case msg := <-task:
				if id == 2 {
					time.Sleep(20 * time.Millisecond)
				}
				fmt.Println(id, msg)
				if msg == "stop" {
					<-next
					success <- true
					return
				}
			case <-time.After(2 * time.Second):
				fmt.Println("no new event for 2 seconds for ID", id)
				<-next
				success <- false
				return
			}
		}
	}
	exitOnEmpty := func() bool {
		return true
	}
	q.AnalysePool(1, exitOnEmpty, analyzer)

*/
package queue

import (
	"regexp"
	"runtime"
	"strconv"
	"time"

	"github.com/garyburd/redigo/redis"
)

//Queue the strcuture that will ecompass the queue settings and methods.
type Queue struct {
	// AnalyzeBuff will set number of concurrent running anlyzers. It will default to number of cpu if not set.
	AnalyzerBuff int
	// QueueName this will set the name used in udnerlying system for the queue. Default is "QUEUE"
	QueueName string
	paritions int
	urls      []string
	pool      []redis.Conn
}

// Urls will accept a slice of redis connection URLS. This slice will setup the connections and also set how many redis paritions will be used.
// Setting more than one redis is useful in some cases that a single redis can't handle a the queue load either because of IO and memory restrictions or if possible CPU.
func (q *Queue) Urls(urls []string) {
	q.urls = q.urls[:0]
	q.pool = q.pool[:0]
	for _, v := range urls {
		c, e := redis.DialURL(v)
		checkErr(e)
		q.urls = append(q.urls, v)
		q.pool = append(q.pool, c)
	}
	q.paritions = len(q.urls)
}

// AddTask will add a task to the queue. It will accept an ID and a string.
// If more than one task are added with the same ID, queue will make sure they are send
// to the same analyser as long as analyers does not return before next ID is poped from the queue.
func (q *Queue) AddTask(id int, task string) {
	task = strconv.Itoa(id) + ";" + task
	queueName := "QUEUE"
	if q.QueueName != "" {
		queueName = q.QueueName
	}
	_, e := q.pool[id%q.paritions].Do("RPUSH", queueName, task)
	checkErr(e)
}

func (q *Queue) waitforSuccess(id int, success chan bool, pool map[int]chan string, queueName string) {
	redisdb, _ := redis.DialURL(q.urls[id%q.paritions])
	redisdb.Do("SET", queueName+"::PENDING::"+strconv.Itoa(id), 1)
	r := <-success
	if r {
		delete(pool, id)
		redisdb.Do("DEL", queueName+"::PENDING::"+strconv.Itoa(id))
	}
}

func removeTask(redisdb redis.Conn, queueName string) (int, string) {
	r, e := redisdb.Do("LPOP", queueName)
	checkErr(e)
	if r != nil {
		s, _ := redis.String(r, e)
		m := regexp.MustCompile(`(\d+);(.*)$`).FindStringSubmatch(s)
		id, _ := strconv.Atoi(m[1])
		redisdb.Do("SET", queueName+"::PENDING::"+strconv.Itoa(id), 1)
		return id, m[2]
	}
	return 0, ""
}

/*
AnalysePool can be calls to process redis queue(s).
analyzerID will set which redis AnalysePool will connect to (redis:=pool[len(urls)%AnalysePool])

exitOnEmpty is a closure function which will control inner loop of AnalysePool when queue is empty.
	exitOnEmpty := func() bool {
		return true
	}
analyzer is a closure function which will be called for processing the tasks popped from queue.
	analyzer := func(id int, task chan string, success chan bool, next chan bool) {
		for {
			select {
			case msg := <-task:
				if id == 2 {
					time.Sleep(20 * time.Millisecond)
				}
				fmt.Println(id, msg)
				if msg == "stop" {
					<-next
					success <- true
					return
				}
			case <-time.After(2 * time.Second):
				fmt.Println("no new event for 2 seconds for ID", id)
				<-next
				success <- false
				return
			}
		}
	}
Analyser clousre must be able to accept the new Tasks without delay and if needed process them concurrently. Delay in accepting new Task will block AnalysePool.

*/
func (q *Queue) AnalysePool(analyzerID int, exitOnEmpty func() bool, analyzer func(int, chan string, chan bool, chan bool)) {
	redisdb, _ := redis.DialURL(q.urls[q.paritions%analyzerID])
	queueName := "QUEUE"
	if q.QueueName != "" {
		queueName = q.QueueName
	}
	buff := q.AnalyzerBuff
	if buff == 0 {
		buff = runtime.NumCPU()
	}
	next := make(chan bool, buff)
	pool := make(map[int]chan string)
	for {
		id, task := removeTask(redisdb, queueName)
		if task == "" {
			if exitOnEmpty() {
				break
			} else {
				time.Sleep(100 * time.Millisecond)
			}
		} else {
			if pool[id] == nil {
				pool[id] = make(chan string)
				success := make(chan bool)
				go analyzer(id, pool[id], success, next)
				go q.waitforSuccess(id, success, pool, queueName)
				pool[id] <- task
				next <- true
			} else {
				pool[id] <- task
			}
		}
	}

	for i := 0; i < buff; i++ {
		next <- true
	}
}

func checkErr(e error) {
	if e != nil {
		panic(e)
	}
}
<commit_msg>cleaning the doc<commit_after>/*
queue is a simple Queue system written in Go that will use Redis underneath.
Focus of this design is mainly horisontal scalability via concurrency, paritioning and fault-detection
Queues can be partitions in to more than one Redis is necessary.

Number of redis paritions is set by using Urls function and setting slice of Redis URL connections.
Redis paritioning is required in cases that one redis cannot handle the load because of IO, moemory or in rare situations CPU limitations.

In case of crash record of all incomplete tasks will be kepts in redis as keys with this format
	QUEUE::PENDING::ID
ID will indicate the ID of failed tasks.

To use this library you need to use queue struct.

	var q Queue
	q.Urls([]{redis://localhost:6379})

Adding tasks is done by calling AddTask. This function will accept an ID and the task itself that will be as a string.

	q.AddTask(1, "task1")
	q.AddTask(2, "task2")

ID can be used in a special way. If ID of two tasks are the same while processing AnalysePool will send them to the same analyser goroutine if analyzer waits enough.

	q.AddTask(2, "start")
	q.AddTask(1, "load")
	q.AddTask(2, "load")
	q.AddTask(1, "stop")
	q.AddTask(2, "stop")

This feature can be used in analyser needs to process a set of related tasks one after another.

AnalysePool accepts 3 parameters. One analyzerID that will identify which redis pool this AnalysePool will connect to.

	whichRedis=len(q.urls) % analyzerID

AnalysePool need two closures, analyzer and exitOnEmpty. Format of those closure are as follows.

	analyzer := func(id int, task chan string, success chan bool, next chan bool) {
		for {
			select {
			case msg := <-task:
				//process the task
				if msg == "stop_indicator" {
					<-next
					success <- true
					return
				}
			}
		}
	}
	exitOnEmpty := func() bool {
		return true
	}
	q.AnalysePool(1, exitOnEmpty, analyzer)
*/
package queue

import (
	"regexp"
	"runtime"
	"strconv"
	"time"

	"github.com/garyburd/redigo/redis"
)

//Queue the strcuture that will ecompass the queue settings and methods.
type Queue struct {
	// AnalyzeBuff will set number of concurrent running anlyzers. It will default to number of cpu if not set.
	AnalyzerBuff int
	// QueueName this will set the name used in udnerlying system for the queue. Default is "QUEUE"
	QueueName string
	paritions int
	urls      []string
	pool      []redis.Conn
}

// Urls will accept a slice of redis connection URLS. This slice will setup the connections and also set how many redis paritions will be used.
// Setting more than one redis is useful in some cases that a single redis can't handle a the queue load either because of IO and memory restrictions or if possible CPU.
func (q *Queue) Urls(urls []string) {
	q.urls = q.urls[:0]
	q.pool = q.pool[:0]
	for _, v := range urls {
		c, e := redis.DialURL(v)
		checkErr(e)
		q.urls = append(q.urls, v)
		q.pool = append(q.pool, c)
	}
	q.paritions = len(q.urls)
}

// AddTask will add a task to the queue. It will accept an ID and a string.
// If more than one task are added with the same ID, queue will make sure they are send
// to the same analyser as long as analyers does not return before next ID is poped from the queue.
func (q *Queue) AddTask(id int, task string) {
	task = strconv.Itoa(id) + ";" + task
	queueName := "QUEUE"
	if q.QueueName != "" {
		queueName = q.QueueName
	}
	_, e := q.pool[id%q.paritions].Do("RPUSH", queueName, task)
	checkErr(e)
}

func (q *Queue) waitforSuccess(id int, success chan bool, pool map[int]chan string, queueName string) {
	redisdb, _ := redis.DialURL(q.urls[id%q.paritions])
	redisdb.Do("SET", queueName+"::PENDING::"+strconv.Itoa(id), 1)
	r := <-success
	if r {
		delete(pool, id)
		redisdb.Do("DEL", queueName+"::PENDING::"+strconv.Itoa(id))
	}
}

func removeTask(redisdb redis.Conn, queueName string) (int, string) {
	r, e := redisdb.Do("LPOP", queueName)
	checkErr(e)
	if r != nil {
		s, _ := redis.String(r, e)
		m := regexp.MustCompile(`(\d+);(.*)$`).FindStringSubmatch(s)
		id, _ := strconv.Atoi(m[1])
		redisdb.Do("SET", queueName+"::PENDING::"+strconv.Itoa(id), 1)
		return id, m[2]
	}
	return 0, ""
}

/*
AnalysePool can be calls to process redis queue(s).
analyzerID will set which redis AnalysePool will connect to (redis:=pool[len(urls)%AnalysePool])

exitOnEmpty is a closure function which will control inner loop of AnalysePool when queue is empty.
	exitOnEmpty := func() bool {
		return true
	}
analyzer is a closure function which will be called for processing the tasks popped from queue.
	analyzer := func(id int, task chan string, success chan bool, next chan bool) {
		for {
			select {
			case msg := <-task:
				if id == 2 {
					time.Sleep(20 * time.Millisecond)
				}
				fmt.Println(id, msg)
				if msg == "stop" {
					<-next
					success <- true
					return
				}
			case <-time.After(2 * time.Second):
				fmt.Println("no new event for 2 seconds for ID", id)
				<-next
				success <- false
				return
			}
		}
	}
Analyser clousre must be able to accept the new Tasks without delay and if needed process them concurrently. Delay in accepting new Task will block AnalysePool.
*/
func (q *Queue) AnalysePool(analyzerID int, exitOnEmpty func() bool, analyzer func(int, chan string, chan bool, chan bool)) {
	redisdb, _ := redis.DialURL(q.urls[q.paritions%analyzerID])
	queueName := "QUEUE"
	if q.QueueName != "" {
		queueName = q.QueueName
	}
	buff := q.AnalyzerBuff
	if buff == 0 {
		buff = runtime.NumCPU()
	}
	next := make(chan bool, buff)
	pool := make(map[int]chan string)
	for {
		id, task := removeTask(redisdb, queueName)
		if task == "" {
			if exitOnEmpty() {
				break
			} else {
				time.Sleep(100 * time.Millisecond)
			}
		} else {
			if pool[id] == nil {
				pool[id] = make(chan string)
				success := make(chan bool)
				go analyzer(id, pool[id], success, next)
				go q.waitforSuccess(id, success, pool, queueName)
				pool[id] <- task
				next <- true
			} else {
				pool[id] <- task
			}
		}
	}

	for i := 0; i < buff; i++ {
		next <- true
	}
}

func checkErr(e error) {
	if e != nil {
		panic(e)
	}
}
<|endoftext|>