<commit_before>package ftp

import (
	"path"
)

//Walker traverses the directory tree of a remote FTP server
type Walker struct {
	serverConn *ServerConn
	root       string
	cur        *item
	stack      []*item
	descend    bool
}

type item struct {
	path  string
	entry *Entry
	err   error
}

// Next advances the Walker to the next file or directory,
// which will then be available through the Path, Stat, and Err methods.
// It returns false when the walk stops at the end of the tree.
func (w *Walker) Next() bool {
	// check if we need to init cur, maybe this should be inside Walk
	if w.cur == nil {
		w.cur = &item{
			path: w.root,
			entry: &Entry{
				Type: EntryTypeFolder,
			},
		}
	}

	if w.descend && w.cur.entry.Type == EntryTypeFolder {
		entries, err := w.serverConn.List(w.cur.path)

		// an error occurred, drop out and stop walking
		if err != nil {
			w.cur.err = err
			return false
		}

		for _, entry := range entries {
			if entry.Name == "." || entry.Name == ".." {
				continue
			}

			item := &item{
				path:  path.Join(w.cur.path, entry.Name),
				entry: entry,
			}

			w.stack = append(w.stack, item)
		}
	}

	if len(w.stack) == 0 {
		return false
	}

	// update cur
	i := len(w.stack) - 1
	w.cur = w.stack[i]
	w.stack = w.stack[:i]

	// reset SkipDir
	w.descend = true

	return true
}

//SkipDir tells the Next function to skip the currently processed directory
func (w *Walker) SkipDir() {
	w.descend = false
}

//Err returns the error, if any, for the most recent attempt by Next to
//visit a file or a directory. If a directory has an error, the walker
//will not descend in that directory
func (w *Walker) Err() error {
	return w.cur.err
}

// Stat returns info for the most recent file or directory
// visited by a call to Step.
func (w *Walker) Stat() *Entry {
	return w.cur.entry
}

// Path returns the path to the most recent file or directory
// visited by a call to Next. It contains the argument to Walk
// as a prefix; that is, if Walk is called with "dir", which is
// a directory containing the file "a", Path will return "dir/a".
func (w *Walker) Path() string {
	return w.cur.path
}
<commit_msg>Documentation update.<commit_after>package ftp

import (
	"path"
)

//Walker traverses the directory tree of a remote FTP server
type Walker struct {
	serverConn *ServerConn
	root       string
	cur        *item
	stack      []*item
	descend    bool
}

type item struct {
	path  string
	entry *Entry
	err   error
}

// Next advances the Walker to the next file or directory,
// which will then be available through the Path, Stat, and Err methods.
// It returns false when the walk stops at the end of the tree.
func (w *Walker) Next() bool {
	// check if we need to init cur, maybe this should be inside Walk
	if w.cur == nil {
		w.cur = &item{
			path: w.root,
			entry: &Entry{
				Type: EntryTypeFolder,
			},
		}
	}

	if w.descend && w.cur.entry.Type == EntryTypeFolder {
		entries, err := w.serverConn.List(w.cur.path)

		// an error occurred, drop out and stop walking
		if err != nil {
			w.cur.err = err
			return false
		}

		for _, entry := range entries {
			if entry.Name == "." || entry.Name == ".." {
				continue
			}

			item := &item{
				path:  path.Join(w.cur.path, entry.Name),
				entry: entry,
			}

			w.stack = append(w.stack, item)
		}
	}

	if len(w.stack) == 0 {
		return false
	}

	// update cur
	i := len(w.stack) - 1
	w.cur = w.stack[i]
	w.stack = w.stack[:i]

	// reset SkipDir
	w.descend = true

	return true
}

//SkipDir tells the Next function to skip the currently processed directory
func (w *Walker) SkipDir() {
	w.descend = false
}

//Err returns the error, if any, for the most recent attempt by Next to
//visit a file or a directory. If a directory has an error, the walker
//will not descend in that directory
func (w *Walker) Err() error {
	return w.cur.err
}

// Stat returns info for the most recent file or directory
// visited by a call to Next.
func (w *Walker) Stat() *Entry {
	return w.cur.entry
}

// Path returns the path to the most recent file or directory
// visited by a call to Next. It contains the argument to Walk
// as a prefix; that is, if Walk is called with "dir", which is
// a directory containing the file "a", Path will return "dir/a".
func (w *Walker) Path() string {
	return w.cur.path
}
<|endoftext|>


<commit_before>package main

import "bufio"
import "fmt"
import "os"
import "strings"
import "time"

type TimeModeID string

const (
	WorkID TimeModeID = "w"
	BreakID = "b"
)

type TimeMode struct {
	id TimeModeID
	message string
}

var (
	Work = TimeMode{id: WorkID, message: "Work!"}
	Break = TimeMode{id: BreakID, message: "Take a break."}
)

func (mode TimeMode) oppositeMode() TimeMode {
	if mode == Work {
		return Break
	}
	return Work
}

func printInputOptions(currentMode TimeMode) {
	fmt.Println(" Options:")
	if currentMode == Break {
		fmt.Println("  [w]ork")
	} else if currentMode == Work {
		fmt.Println("  [b]reak")
	}
	fmt.Println("  [q]uit")
}

func runTimer(currentMode TimeMode) bool {
	fmt.Println(currentMode.message)

	stdinCh := make(chan string)
	go func(ch chan string) {
		reader := bufio.NewReader(os.Stdin)
		for {
			s, err := reader.ReadString('\n')
			if err != nil {
				// io.EOF and other errors
				close(ch)
				return
			}
			ch <- s
		}
	}(stdinCh)

	ticker := time.NewTicker(time.Second)
	start := time.Now()
	for {
		select {
		case <- ticker.C:
			now := time.Now()
			elapsed := now.Sub(start)
			minutes := int(elapsed.Minutes())
			seconds := int(elapsed.Seconds()) % 60
			fmt.Printf("\r%02d:%02d", minutes, seconds)
		case stdin, ok := <-stdinCh:
			if !ok {
				fmt.Println("stdin was lost")
				return false
			} else {
				stdin = strings.TrimSpace(stdin)
				if stdin == string(currentMode.oppositeMode().id) {
					return true
				} else if stdin == "q" {
					return false
				} else {
					printInputOptions(currentMode)
				}
			}
		}
	}
}

func main() {
	mode := Work
	for {
		keepGoing := runTimer(mode)
		if !keepGoing {
			break
		}
		mode = mode.oppositeMode()
	}
}
<commit_msg>go fmt<commit_after>package main

import "bufio"
import "fmt"
import "os"
import "strings"
import "time"

type TimeModeID string

const (
	WorkID  TimeModeID = "w"
	BreakID            = "b"
)

type TimeMode struct {
	id      TimeModeID
	message string
}

var (
	Work  = TimeMode{id: WorkID, message: "Work!"}
	Break = TimeMode{id: BreakID, message: "Take a break."}
)

func (mode TimeMode) oppositeMode() TimeMode {
	if mode == Work {
		return Break
	}
	return Work
}

func printInputOptions(currentMode TimeMode) {
	fmt.Println(" Options:")
	if currentMode == Break {
		fmt.Println("  [w]ork")
	} else if currentMode == Work {
		fmt.Println("  [b]reak")
	}
	fmt.Println("  [q]uit")
}

func runTimer(currentMode TimeMode) bool {
	fmt.Println(currentMode.message)

	stdinCh := make(chan string)
	go func(ch chan string) {
		reader := bufio.NewReader(os.Stdin)
		for {
			s, err := reader.ReadString('\n')
			if err != nil {
				// io.EOF and other errors
				close(ch)
				return
			}
			ch <- s
		}
	}(stdinCh)

	ticker := time.NewTicker(time.Second)
	start := time.Now()
	for {
		select {
		case <-ticker.C:
			now := time.Now()
			elapsed := now.Sub(start)
			minutes := int(elapsed.Minutes())
			seconds := int(elapsed.Seconds()) % 60
			fmt.Printf("\r%02d:%02d", minutes, seconds)
		case stdin, ok := <-stdinCh:
			if !ok {
				fmt.Println("stdin was lost")
				return false
			} else {
				stdin = strings.TrimSpace(stdin)
				if stdin == string(currentMode.oppositeMode().id) {
					return true
				} else if stdin == "q" {
					return false
				} else {
					printInputOptions(currentMode)
				}
			}
		}
	}
}

func main() {
	mode := Work
	for {
		keepGoing := runTimer(mode)
		if !keepGoing {
			break
		}
		mode = mode.oppositeMode()
	}
}
<|endoftext|>


<commit_before>package worker

import (
	"fmt"
	"sync"
	"time"
)

type Tasker interface {
	Do() bool
}

type Work struct {
	debug         bool          //是否开启debu，如果开启debug，则会在对应的地方执行log函数。
	retryTimes    uint8         //如果任务失败重试的次数，默认为0。
	c             chan bool     //用于通讯的chan，长度根据limit长度在start的时候进行make
	idleSleepTime time.Duration //没有获取到任务再次获取的间隔时间，默认为1s。
	taskSleepTime time.Duration //每个任务之间间隔的时间，默认为0。
	lock          sync.Mutex    //获取task的时候要进行并发处理，所以先将slice锁起来。
	limit         int           //最多同时有几个任务
	taskList      []Tasker      //任务列表
}

// 设置失败重试的次数
func (w *Work) RetryTimes(t uint8) {
	w.retryTimes = t
}

// 设置没有任务的时候的睡眠时间
func (w *Work) IdleSleepTime(t time.Duration) {
	w.idleSleepTime = t
}

// 设置每个任务之间的间隔时间
func (w *Work) TaskSleepTime(t time.Duration) {
	w.taskSleepTime = t
}

// 设置是否为debug
func (w *Work) Debug(debug bool) *Work {
	w.debug = debug
	return w
}

// 设置goroutine速度
func (w *Work) Limit(limit int) {
	w.limit = limit
}

// 增加一个任务到列表里面
func (w *Work) AddTask(t Tasker) {
	w.taskList = append(w.taskList, t)
	if w.debug {
		w.log()
	}
}

// 获取一个任务，以及是否获取成功
func (w *Work) getTask() (Tasker, bool) {
	w.lock.Lock()
	defer w.lock.Unlock()
	if len(w.taskList) == 0 {
		return nil, false
	}
	t := w.taskList[0]
	w.taskList = w.taskList[1:]
	return t, true
}

// 任务开始
func (w *Work) Start() {
	w.c = make(chan bool, w.limit)

	for i := 0; i < w.limit; i++ {
		go w.Do()
	}

	for {
		<-w.c
		go w.Do()
	}

}

// 获取执行任务进行执行
func (w *Work) Do() {
	for {
		t, ok := w.getTask()
		try := w.retryTimes
		ret := false
		if ok {
			for {
				ret = t.Do()
				if w.taskSleepTime != 0 {
					time.Sleep(w.taskSleepTime)
				}
				if ret {
					break
				} else {
					if try <= 0 {
						break
					} else {
						continue
					}
				}
			}
			w.c <- ret
			break
		}
		time.Sleep(w.idleSleepTime)
	}

}

// 返回任务列表现有的个数
func (w *Work) Len() int {
	return len(w.taskList)
}

func (w *Work) log() {
	fmt.Println("task len:", w.Len(), "limit", w.limit, "idleSleepTime:", w.idleSleepTime, "taskSleepTime:", w.taskSleepTime)
}

// 创建一个Work
func NewWork() *Work {
	w := new(Work)
	w.idleSleepTime = time.Second
	w.taskSleepTime = 0
	w.limit = 1
	w.retryTimes = 0
	return w
}
<commit_msg>增加停止任务功能<commit_after>package worker

import (
	"fmt"
	"sync"
	"time"
)

type Tasker interface {
	Do() bool
}

type Work struct {
	debug         bool //是否开启debu，如果开启debug，则会在对应的地方执行log函数。
	isStop        bool
	retryTimes    uint8         //如果任务失败重试的次数，默认为0。
	c             chan bool     //用于通讯的chan，长度根据limit长度在start的时候进行make
	idleSleepTime time.Duration //没有获取到任务再次获取的间隔时间，默认为1s。
	taskSleepTime time.Duration //每个任务之间间隔的时间，默认为0。
	lock          sync.Mutex    //获取task的时候要进行并发处理，所以先将slice锁起来。
	limit         int           //最多同时有几个任务
	taskList      []Tasker      //任务列表
}

// 设置失败重试的次数
func (w *Work) RetryTimes(t uint8) {
	w.retryTimes = t
}

// 设置没有任务的时候的睡眠时间
func (w *Work) IdleSleepTime(t time.Duration) {
	w.idleSleepTime = t
}

// 设置每个任务之间的间隔时间
func (w *Work) TaskSleepTime(t time.Duration) {
	w.taskSleepTime = t
}

// 设置是否为debug
func (w *Work) Debug(debug bool) *Work {
	w.debug = debug
	return w
}

// 设置goroutine速度
func (w *Work) Limit(limit int) {
	w.limit = limit
}

// 增加一个任务到列表里面
func (w *Work) AddTask(t Tasker) {
	w.taskList = append(w.taskList, t)
	if w.debug {
		w.log()
	}
}

// 获取一个任务，以及是否获取成功
func (w *Work) getTask() (Tasker, bool) {
	w.lock.Lock()
	defer w.lock.Unlock()
	if len(w.taskList) == 0 {
		return nil, false
	}
	t := w.taskList[0]
	w.taskList = w.taskList[1:]
	return t, true
}

// 任务开始
func (w *Work) Start() {
	w.c = make(chan bool, w.limit)

	for i := 0; i < w.limit; i++ {
		go w.Do()
	}

	for {
		<-w.c
		if !w.isStop {
			go w.Do()
		} else {
			break
		}
	}

}

// 获取执行任务进行执行
func (w *Work) Do() {
	for {
		t, ok := w.getTask()
		try := w.retryTimes
		ret := false
		if ok {
			for {
				ret = t.Do()
				if w.taskSleepTime != 0 {
					time.Sleep(w.taskSleepTime)
				}
				if ret {
					break
				} else {
					if try <= 0 {
						break
					} else {
						continue
					}
				}
			}
			w.c <- ret
			break
		}
		time.Sleep(w.idleSleepTime)
	}

}

// 停止所有任务，已经执行的任务会等执行完毕之后才会停止。
func (w *Work) Stop() {
	w.isStop = true
}

// 返回任务列表现有的个数
func (w *Work) Len() int {
	return len(w.taskList)
}

// debug的时候会调用这个函数
func (w *Work) log() {
	fmt.Println("task len:", w.Len(), "limit", w.limit, "idleSleepTime:", w.idleSleepTime, "taskSleepTime:", w.taskSleepTime)
}

// 创建一个Work
func NewWork() *Work {
	w := new(Work)
	w.idleSleepTime = time.Second
	w.taskSleepTime = 0
	w.limit = 1
	w.retryTimes = 0
	w.isStop = false
	return w
}
<|endoftext|>


<commit_before>package gdw

import (
	"fmt"
	"math/rand"
	"sync"
	"sync/atomic"
	"time"

	"github.com/eapache/channels"
)

// Worker is used to define a goroutine pool whose results and/or execution are of interest, thus awaitable through WaitGroup.
type WorkerPool struct {
	jobQueue   *channels.InfiniteChannel
	limiter    *channels.ResizableChannel
	queueDepth int64
	wgMap      map[string]*sync.WaitGroup
	mapSync    sync.Mutex
}

// Random string utilities - START
// source - http://stackoverflow.com/a/31832326/1112789

var src = rand.NewSource(time.Now().UnixNano())

const (
	letterIdxBits = 6
	letterIdxMask = 1<<letterIdxBits - 1
	letterIdxMax  = 63 / letterIdxBits
	letterBytes   = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
)

func generateToken() string {
	b := make([]byte, 20)
	for i, cache, remain := 20-1, src.Int63(), letterIdxMax; i >= 0; {
		if remain == 0 {
			cache, remain = src.Int63(), letterIdxMax
		}
		if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
			b[i] = letterBytes[idx]
			i--
		}
		cache >>= letterIdxBits
		remain--
	}
	return string(b)
}

// Random string utilities - END

func NewWorkerPool(size int) *WorkerPool {
	jobQueue := channels.NewInfiniteChannel()
	limiter := channels.NewResizableChannel()
	limiter.Resize(channels.BufferCap(size))
	worker := &WorkerPool{
		jobQueue:   jobQueue,
		limiter:    limiter,
		queueDepth: 0,
		wgMap:      make(map[string]*sync.WaitGroup),
	}
	worker.wgMap["gdw_main_pool"] = &sync.WaitGroup{}

	go func() {
		jobQueueOut := jobQueue.Out()
		limiterIn := limiter.In()
		limiterOut := limiter.Out()
		for jobs := range jobQueueOut {
			switch jt := jobs.(type) {

			case Job:
				limiterIn <- true
				atomic.AddInt64(&worker.queueDepth, -1)
				go func(j Job) {
					j.DoWork()
					<-limiterOut
					worker.wgMap["gdw_main_pool"].Done()
				}(jt)

			case []Job:
				for _, job := range jt {
					limiterIn <- true
					atomic.AddInt64(&worker.queueDepth, -1)
					go func(j Job) {
						j.DoWork()
						<-limiterOut
						worker.wgMap["gdw_main_pool"].Done()
					}(job)
				}

			case *batchedJob:
				limiterIn <- true
				atomic.AddInt64(&worker.queueDepth, -1)
				go func(bj *batchedJob) {
					bj.batched.DoWork()
					<-limiterOut
					worker.wgMap[bj.name].Done()
					worker.wgMap["gdw_main_pool"].Done()
				}(jt)

			case []*batchedJob:
				for _, job := range jt {
					limiterIn <- true
					atomic.AddInt64(&worker.queueDepth, -1)
					go func(bj *batchedJob) {
						bj.batched.DoWork()
						<-limiterOut
						worker.wgMap[bj.name].Done()
						worker.wgMap["gdw_main_pool"].Done()
					}(job)
				}

			}
		}
	}()

	return worker
}

func (w *WorkerPool) NewBatch(name string) (*Batch, error) {
	if _, ok := w.wgMap[name]; ok {
		return nil, fmt.Errorf("Batch named %s already exists.", name)
	}
	w.mapSync.Lock()
	w.wgMap[name] = &sync.WaitGroup{}
	w.mapSync.Unlock()
	return &Batch{
		worker: w,
		name:   name,
	}, nil
}

func (w *WorkerPool) NewTempBatch() *Batch {
	token := generateToken()
	w.mapSync.Lock()
	w.wgMap[token] = &sync.WaitGroup{}
	w.mapSync.Unlock()
	return &Batch{
		worker: w,
		name:   token,
	}
}

func (w *WorkerPool) LoadBatch(name string) (*Batch, error) {
	if _, ok := w.wgMap[name]; !ok {
		return nil, fmt.Errorf("No batch named %s exists.", name)
	}
	return &Batch{
		worker: w,
		name:   name,
	}, nil
}

func (w *WorkerPool) SetPoolSize(size int) {
	w.limiter.Resize(channels.BufferCap(size))
}

func (w *WorkerPool) GetPoolSize() int {
	return int(w.limiter.Cap())
}

func (w *WorkerPool) GetQueueDepth() int {
	return int(atomic.LoadInt64(&w.queueDepth))
}

func (w *WorkerPool) Add(job Job, amount int) {
	w.add(job, amount, "gdw_main_pool")
}

func (w *WorkerPool) add(job Job, amount int, batch string) {
	w.wgMap["gdw_main_pool"].Add(amount)
	atomic.AddInt64(&w.queueDepth, int64(amount))
	switch batch {

	case "gdw_main_pool":
		jobs := make([]Job, amount)
		for i := 0; i < amount; i++ {
			jobs[i] = job
		}
		w.jobQueue.In() <- jobs

	default:
		bjs := make([]*batchedJob, amount)
		for i := 0; i < amount; i++ {
			bj := &batchedJob{
				batched: job,
				name:    batch,
			}
			bjs[i] = bj
		}
		w.jobQueue.In() <- bjs

	}
}

func (w *WorkerPool) AddOne(job Job) {
	w.addOne(job, "gdw_main_pool")
}

func (w *WorkerPool) addOne(job Job, batch string) {
	w.wgMap["gdw_main_pool"].Add(1)
	atomic.AddInt64(&w.queueDepth, 1)
	switch batch {

	case "gdw_main_pool":
		w.jobQueue.In() <- job

	default:
		bj := &batchedJob{
			batched: job,
			name:    batch,
		}
		w.jobQueue.In() <- bj

	}
}

func (w *WorkerPool) Wait() {
	w.wgMap["gdw_main_pool"].Wait()
}

func (w *WorkerPool) WaitBatch(batch string) error {
	wg, ok := w.wgMap[batch]
	if !ok {
		return fmt.Errorf("No batch named %s exists.", batch)
	}
	wg.Wait()
	return nil
}

func (w *WorkerPool) CleanBatch(batch string) error {
	if _, ok := w.wgMap[batch]; !ok {
		return fmt.Errorf("No batch named %s exists.", batch)
	}
	w.mapSync.Lock()
	delete(w.wgMap, batch)
	w.mapSync.Unlock()
	return nil
}

func (w *WorkerPool) Close() {
	w.jobQueue.Close()
	w.limiter.Close()
}
<commit_msg>remove unnecessary mutex naming<commit_after>package gdw

import (
	"fmt"
	"math/rand"
	"sync"
	"sync/atomic"
	"time"

	"github.com/eapache/channels"
)

// Worker is used to define a goroutine pool whose results and/or execution are of interest, thus awaitable through WaitGroup.
type WorkerPool struct {
	jobQueue   *channels.InfiniteChannel
	limiter    *channels.ResizableChannel
	queueDepth int64
	wgMap      map[string]*sync.WaitGroup
	sync.Mutex
}

// Random string utilities - START
// source - http://stackoverflow.com/a/31832326/1112789

var src = rand.NewSource(time.Now().UnixNano())

const (
	letterIdxBits = 6
	letterIdxMask = 1<<letterIdxBits - 1
	letterIdxMax  = 63 / letterIdxBits
	letterBytes   = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
)

func generateToken() string {
	b := make([]byte, 20)
	for i, cache, remain := 20-1, src.Int63(), letterIdxMax; i >= 0; {
		if remain == 0 {
			cache, remain = src.Int63(), letterIdxMax
		}
		if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
			b[i] = letterBytes[idx]
			i--
		}
		cache >>= letterIdxBits
		remain--
	}
	return string(b)
}

// Random string utilities - END

func NewWorkerPool(size int) *WorkerPool {
	jobQueue := channels.NewInfiniteChannel()
	limiter := channels.NewResizableChannel()
	limiter.Resize(channels.BufferCap(size))
	worker := &WorkerPool{
		jobQueue:   jobQueue,
		limiter:    limiter,
		queueDepth: 0,
		wgMap:      make(map[string]*sync.WaitGroup),
	}
	worker.wgMap["gdw_main_pool"] = &sync.WaitGroup{}

	go func() {
		jobQueueOut := jobQueue.Out()
		limiterIn := limiter.In()
		limiterOut := limiter.Out()
		for jobs := range jobQueueOut {
			switch jt := jobs.(type) {

			case Job:
				limiterIn <- true
				atomic.AddInt64(&worker.queueDepth, -1)
				go func(j Job) {
					j.DoWork()
					<-limiterOut
					worker.wgMap["gdw_main_pool"].Done()
				}(jt)

			case []Job:
				for _, job := range jt {
					limiterIn <- true
					atomic.AddInt64(&worker.queueDepth, -1)
					go func(j Job) {
						j.DoWork()
						<-limiterOut
						worker.wgMap["gdw_main_pool"].Done()
					}(job)
				}

			case *batchedJob:
				limiterIn <- true
				atomic.AddInt64(&worker.queueDepth, -1)
				go func(bj *batchedJob) {
					bj.batched.DoWork()
					<-limiterOut
					worker.wgMap[bj.name].Done()
					worker.wgMap["gdw_main_pool"].Done()
				}(jt)

			case []*batchedJob:
				for _, job := range jt {
					limiterIn <- true
					atomic.AddInt64(&worker.queueDepth, -1)
					go func(bj *batchedJob) {
						bj.batched.DoWork()
						<-limiterOut
						worker.wgMap[bj.name].Done()
						worker.wgMap["gdw_main_pool"].Done()
					}(job)
				}

			}
		}
	}()

	return worker
}

func (w *WorkerPool) NewBatch(name string) (*Batch, error) {
	if _, ok := w.wgMap[name]; ok {
		return nil, fmt.Errorf("Batch named %s already exists.", name)
	}
	w.Lock()
	w.wgMap[name] = &sync.WaitGroup{}
	w.Unlock()
	return &Batch{
		worker: w,
		name:   name,
	}, nil
}

func (w *WorkerPool) NewTempBatch() *Batch {
	token := generateToken()
	w.Lock()
	w.wgMap[token] = &sync.WaitGroup{}
	w.Unlock()
	return &Batch{
		worker: w,
		name:   token,
	}
}

func (w *WorkerPool) LoadBatch(name string) (*Batch, error) {
	if _, ok := w.wgMap[name]; !ok {
		return nil, fmt.Errorf("No batch named %s exists.", name)
	}
	return &Batch{
		worker: w,
		name:   name,
	}, nil
}

func (w *WorkerPool) SetPoolSize(size int) {
	w.limiter.Resize(channels.BufferCap(size))
}

func (w *WorkerPool) GetPoolSize() int {
	return int(w.limiter.Cap())
}

func (w *WorkerPool) GetQueueDepth() int {
	return int(atomic.LoadInt64(&w.queueDepth))
}

func (w *WorkerPool) Add(job Job, amount int) {
	w.add(job, amount, "gdw_main_pool")
}

func (w *WorkerPool) add(job Job, amount int, batch string) {
	w.wgMap["gdw_main_pool"].Add(amount)
	atomic.AddInt64(&w.queueDepth, int64(amount))
	switch batch {

	case "gdw_main_pool":
		jobs := make([]Job, amount)
		for i := 0; i < amount; i++ {
			jobs[i] = job
		}
		w.jobQueue.In() <- jobs

	default:
		bjs := make([]*batchedJob, amount)
		for i := 0; i < amount; i++ {
			bj := &batchedJob{
				batched: job,
				name:    batch,
			}
			bjs[i] = bj
		}
		w.jobQueue.In() <- bjs

	}
}

func (w *WorkerPool) AddOne(job Job) {
	w.addOne(job, "gdw_main_pool")
}

func (w *WorkerPool) addOne(job Job, batch string) {
	w.wgMap["gdw_main_pool"].Add(1)
	atomic.AddInt64(&w.queueDepth, 1)
	switch batch {

	case "gdw_main_pool":
		w.jobQueue.In() <- job

	default:
		bj := &batchedJob{
			batched: job,
			name:    batch,
		}
		w.jobQueue.In() <- bj

	}
}

func (w *WorkerPool) Wait() {
	w.wgMap["gdw_main_pool"].Wait()
}

func (w *WorkerPool) WaitBatch(batch string) error {
	wg, ok := w.wgMap[batch]
	if !ok {
		return fmt.Errorf("No batch named %s exists.", batch)
	}
	wg.Wait()
	return nil
}

func (w *WorkerPool) CleanBatch(batch string) error {
	if _, ok := w.wgMap[batch]; !ok {
		return fmt.Errorf("No batch named %s exists.", batch)
	}
	w.Lock()
	delete(w.wgMap, batch)
	w.Unlock()
	return nil
}

func (w *WorkerPool) Close() {
	w.jobQueue.Close()
	w.limiter.Close()
}
<|endoftext|>


<commit_before>package worker

import (
        //"os"
        "sync"
        "errors"
)

// Result represents a delivery of a job done.
type Result interface {}

// ReplyJob represents a delivery of a job done and the Action() will
// be called when done.
type ReplyJob interface {
        Action()
}

// Job represents a piece of job to be done.
type Job interface {
        Action() Result
}

// Sentry is used to ensure a sequence of jobs are done at a point.
type Sentry struct {
        worker *Worker
        mutex *sync.Mutex
        results []Result
        waiter *sync.WaitGroup
}

type guard struct {
        sentry *Sentry
        job Job
}
func (m *guard) Action() Result {
        result := m.job.Action()
        m.sentry.mutex.Lock()
        m.sentry.results = append(m.sentry.results, result)
        m.sentry.mutex.Unlock()
        m.sentry.waiter.Done()
        return nil
}

type stop struct {
        waiter *sync.WaitGroup
}
func (m *stop) Action() Result { return nil }

// Worker represents a worker to dispatch jobs being done.
type Worker struct {
        routines int
        i chan Job
        o chan Result
}

// New creates a new worker valid to do user-defined jobs.
func New() *Worker {
        return &Worker{}
}

func (w *Worker) routine(num int) {
        for msg := range w.i {
                if msg != nil {
                        var sw *sync.WaitGroup
                        if stop, ok := msg.(*stop); ok && stop != nil {
                                sw = stop.waiter
                        }
                        w.o <- msg.Action(); go w.reply(sw)
                        if sw != nil { return }
                }
        }
}

func (w *Worker) reply(wg *sync.WaitGroup) {
        if res := <-w.o; res != nil {
                if j, ok := res.(ReplyJob); ok && j != nil {
                        j.Action()
                }
        }
        if wg != nil {
                wg.Done()
        }
}

// Worker.SpawnN starts a number of `num` threads for jobs.
func (w *Worker) SpawnN(num int) error {
        if w.i != nil {
                return errors.New("worker is busy on jobs")
        }
        if w.o != nil {
                return errors.New("worker is busy on job results")
        }
        w.i, w.o, w.routines = make(chan Job, 1), make(chan Result, 1), num
        for i := 0; i < num; i++ {
                go w.routine(i)
        }
        return nil
}

// Worker.KillN stops a number of `num` threads.
func (w *Worker) KillAll() error {
        if w.i == nil {
                return errors.New("worker free")
        }
        if w.o == nil {
                return errors.New("worker don't have job results")
        }

        c := &stop{ new(sync.WaitGroup) }
        c.waiter.Add(w.routines)
        for i := 0; i < w.routines; i++ {
                w.i <- c
        }
        c.waiter.Wait()
        close(w.i)
        close(w.o)
        w.i, w.o = nil, nil
        return nil
}

// Worker.Do perform a job.
func (w *Worker) Do(m Job) {
        if w.i != nil {
                w.i <- m
        }
}

// Set a new sentry for the worker.
func (w *Worker) Sentry() *Sentry {
        sentry := &Sentry{
                worker: w,
                mutex: new(sync.Mutex),
                waiter: new(sync.WaitGroup),
        }
        return sentry
}

// Perform a guarded job by the sentry.
func (s *Sentry) Guard(m Job) {
        s.waiter.Add(1)
        s.worker.Do(&guard{ s, m })
}

// Wait for finish of all guarded jobs.
func (s *Sentry) Wait() (results []Result) {
        s.waiter.Wait()
        return s.results
}
<commit_msg>add global SpawnN<commit_after>package worker

import (
        //"os"
        "sync"
        "errors"
)

// Result represents a delivery of a job done.
type Result interface {}

// ReplyJob represents a delivery of a job done and the Action() will
// be called when done.
type ReplyJob interface {
        Action()
}

// Job represents a piece of job to be done.
type Job interface {
        Action() Result
}

// Sentry is used to ensure a sequence of jobs are done at a point.
type Sentry struct {
        worker *Worker
        mutex *sync.Mutex
        results []Result
        waiter *sync.WaitGroup
}

type guard struct {
        sentry *Sentry
        job Job
}
func (m *guard) Action() Result {
        result := m.job.Action()
        m.sentry.mutex.Lock()
        m.sentry.results = append(m.sentry.results, result)
        m.sentry.mutex.Unlock()
        m.sentry.waiter.Done()
        return nil
}

type stop struct {
        waiter *sync.WaitGroup
}
func (m *stop) Action() Result { return nil }

// Worker represents a worker to dispatch jobs being done.
type Worker struct {
        routines int
        i chan Job
        o chan Result
}

// New creates a new worker valid to do user-defined jobs.
func New() *Worker {
        return &Worker{}
}

// New creates a new worker and spawn N threads to do user-defined jobs.
func SpawnN(num int) *Worker {
        w := New()
        w.SpawnN(num)
        return w
}

func (w *Worker) routine(num int) {
        for msg := range w.i {
                if msg != nil {
                        var sw *sync.WaitGroup
                        if stop, ok := msg.(*stop); ok && stop != nil {
                                sw = stop.waiter
                        }
                        w.o <- msg.Action(); go w.reply(sw)
                        if sw != nil { return }
                }
        }
}

func (w *Worker) reply(wg *sync.WaitGroup) {
        if res := <-w.o; res != nil {
                if j, ok := res.(ReplyJob); ok && j != nil {
                        j.Action()
                }
        }
        if wg != nil {
                wg.Done()
        }
}

// Worker.SpawnN starts a number of `num` threads for jobs.
func (w *Worker) SpawnN(num int) error {
        if w.i != nil {
                return errors.New("worker is busy on jobs")
        }
        if w.o != nil {
                return errors.New("worker is busy on job results")
        }
        w.i, w.o, w.routines = make(chan Job, 1), make(chan Result, 1), num
        for i := 0; i < num; i++ {
                go w.routine(i)
        }
        return nil
}

// Worker.KillN stops a number of `num` threads.
func (w *Worker) KillAll() error {
        if w.i == nil {
                return errors.New("worker free")
        }
        if w.o == nil {
                return errors.New("worker don't have job results")
        }

        c := &stop{ new(sync.WaitGroup) }
        c.waiter.Add(w.routines)
        for i := 0; i < w.routines; i++ {
                w.i <- c
        }
        c.waiter.Wait()
        close(w.i)
        close(w.o)
        w.i, w.o = nil, nil
        return nil
}

// Worker.Do perform a job.
func (w *Worker) Do(m Job) {
        if w.i != nil {
                w.i <- m
        }
}

// Set a new sentry for the worker.
func (w *Worker) Sentry() *Sentry {
        sentry := &Sentry{
                worker: w,
                mutex: new(sync.Mutex),
                waiter: new(sync.WaitGroup),
        }
        return sentry
}

// Perform a guarded job by the sentry.
func (s *Sentry) Guard(m Job) {
        s.waiter.Add(1)
        s.worker.Do(&guard{ s, m })
}

// Wait for finish of all guarded jobs.
func (s *Sentry) Wait() (results []Result) {
        s.waiter.Wait()
        return s.results
}
<|endoftext|>


<commit_before>// Copyright 2013 Benoît Amiaux. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package iobit

import (
	"encoding/binary"
	"errors"
)

// Writer wraps a raw byte array and provides multiple methoods to write data bit-by-bit
// Its methods don't return the usual error as it is too expensive.
// Instead, write errors can be checked with the Flush() method.
type Writer struct {
	dst   []byte
	cache uint64
	fill  uint
	idx   int
}

var (
	ErrOverflow  = errors.New("bit overflow")
	ErrUnderflow = errors.New("bit underflow")
)

// NewWriter returns a new writer writing to <dst> byte array.
func NewWriter(dst []byte) *Writer {
	return &Writer{dst: dst}
}

// PutUint32 writes up to 32 <bits> from <val> in big-endian order.
func (w *Writer) PutUint32(bits uint, val uint32) {
	u := uint64(val)
	// manually inlined until compiler improves
	if w.fill+bits > 64 {
		if w.idx+4 <= len(w.dst) {
			binary.BigEndian.PutUint32(w.dst[w.idx:], uint32(w.cache>>32))
		}
		w.idx += 4
		w.cache <<= 32
		w.fill -= 32
	}
	u &= ^(^uint64(0) << bits)
	u <<= 64 - w.fill - bits
	w.fill += bits
	w.cache |= u
}

// PutUint64 writes up to 64 <bits> from <val> in big-endian order.
func (w *Writer) PutUint64(bits uint, val uint64) {
	if bits > 32 {
		w.PutUint32(bits-32, uint32(val>>32))
		bits = 32
		val &= 0xFFFFFFFF
	}
	w.PutUint32(bits, uint32(val))
}

// PutBit writes one bit to output.
func (w *Writer) PutBit(val bool) {
	v := uint32(0)
	if val {
		v = 1
	}
	w.PutUint32(1, v)
}

// PutByte writes one byte.
func (w *Writer) PutByte(val byte) {
	w.PutUint32(8, uint32(val))
}

// PutLe16 writes 16 bits in little-endian order.
func (w *Writer) PutLe16(val uint16) {
	w.PutUint32(16, uint32(bswap16(val)))
}

// PutBe16 writes 16 bits in big-endian order.
func (w *Writer) PutBe16(val uint16) {
	w.PutUint32(16, uint32(val))
}

// PutLe32 writes 32 bits in little-endian order.
func (w *Writer) PutLe32(val uint32) {
	w.PutUint32(32, bswap32(val))
}

// PutBe32 writes 32 bits in big-endian order.
func (w *Writer) PutBe32(val uint32) {
	w.PutUint32(32, val)
}

// PutLe64 writes 64 bits in little-endian order.
func (w *Writer) PutLe64(val uint64) {
	w.PutLe32(uint32(val))
	w.PutLe32(uint32(val >> 32))
}

// PutBe64 writes 64 bits in big-endian order.
func (w *Writer) PutBe64(val uint64) {
	w.PutUint64(64, val)
}

// PutUint8 writes up to 8 bits.
func (w *Writer) PutUint8(bits uint, val byte) {
	w.PutUint32(bits, uint32(val))
}

// PutInt8 writes up to 8 signed bits.
func (w *Writer) PutInt8(bits uint, val int8) {
	w.PutUint32(bits, uint32(val))
}

// PutUint16 writes up to 16 bits in big-endian order.
func (w *Writer) PutUint16(bits uint, val uint16) {
	w.PutUint32(bits, uint32(val))
}

// PutInt16 writes up to 16 signed bits in big-endian order.
func (w *Writer) PutInt16(bits uint, val int16) {
	w.PutUint32(bits, uint32(val))
}

// PutInt32 writes up to 32 signed bits in big-endian order.
func (w *Writer) PutInt32(bits uint, val int32) {
	w.PutUint32(bits, uint32(val))
}

// PutInt64 writes up to 64 signed bits in big-endian order.
func (w *Writer) PutInt64(bits uint, val int64) {
	w.PutUint64(bits, uint64(val))
}

// Flush flushes the writer to its underlying buffer.
// Returns ErrUnderflow if the output is not byte-aligned.
// Returns ErrOverflow if the output array is too small.
func (w *Writer) Flush() error {
	for w.fill >= 8 && w.idx < len(w.dst) {
		w.dst[w.idx] = byte(w.cache >> 56)
		w.idx += 1
		w.cache <<= 8
		w.fill -= 8
	}
	if w.idx+int(w.fill) > len(w.dst) {
		return ErrOverflow
	}
	if w.fill != 0 {
		return ErrUnderflow
	}
	return nil
}

// Write writes a whole slice <p> at once.
// Returns an error if the writer is not byte-aligned.
func (w *Writer) Write(p []byte) (int, error) {
	err := w.Flush()
	if err != nil {
		return 0, err
	}
	n := 0
	if w.idx < len(w.dst) {
		n = copy(w.dst[w.idx:], p)
	}
	w.idx += len(p)
	if n != len(p) {
		return n, ErrOverflow
	}
	return n, nil
}

// Index returns the current writer position in bits.
func (w *Writer) Index() int {
	return w.idx<<3 + int(w.fill)
}

func imin(a, b int) int {
	if a > b {
		return b
	}
	return a
}

// Bits returns the number of bits available to write.
func (w *Writer) Bits() int {
	size := len(w.dst)
	return size<<3 - imin(w.idx<<3+int(w.fill), size<<3)
}

// Bytes returns a byte array of what's left to write.
// Note that this array is 8-bit aligned even if the writer is not.
func (w *Writer) Bytes() []byte {
	skip := w.idx + int(w.fill>>3)
	if skip >= len(w.dst) {
		return w.dst[:0]
	}
	return w.dst[skip:len(w.dst)]
}

// Reset resets the writer to its initial position.
func (w *Writer) Reset() {
	w.fill = 0
	w.idx = 0
}
<commit_msg>simplify PutUint64<commit_after>// Copyright 2013 Benoît Amiaux. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package iobit

import (
	"encoding/binary"
	"errors"
)

// Writer wraps a raw byte array and provides multiple methoods to write data bit-by-bit
// Its methods don't return the usual error as it is too expensive.
// Instead, write errors can be checked with the Flush() method.
type Writer struct {
	dst   []byte
	cache uint64
	fill  uint
	idx   int
}

var (
	ErrOverflow  = errors.New("bit overflow")
	ErrUnderflow = errors.New("bit underflow")
)

// NewWriter returns a new writer writing to <dst> byte array.
func NewWriter(dst []byte) *Writer {
	return &Writer{dst: dst}
}

// PutUint32 writes up to 32 <bits> from <val> in big-endian order.
func (w *Writer) PutUint32(bits uint, val uint32) {
	u := uint64(val)
	// manually inlined until compiler improves
	if w.fill+bits > 64 {
		if w.idx+4 <= len(w.dst) {
			binary.BigEndian.PutUint32(w.dst[w.idx:], uint32(w.cache>>32))
		}
		w.idx += 4
		w.cache <<= 32
		w.fill -= 32
	}
	u &= ^(^uint64(0) << bits)
	u <<= 64 - w.fill - bits
	w.fill += bits
	w.cache |= u
}

// PutUint64 writes up to 64 <bits> from <val> in big-endian order.
func (w *Writer) PutUint64(bits uint, val uint64) {
	if bits <= 32 {
		w.PutUint32(bits, uint32(val))
	} else {
		w.PutUint32(bits-32, uint32(val>>32))
		w.PutBe32(uint32(val))
	}
}

// PutBit writes one bit to output.
func (w *Writer) PutBit(val bool) {
	v := uint32(0)
	if val {
		v = 1
	}
	w.PutUint32(1, v)
}

// PutByte writes one byte.
func (w *Writer) PutByte(val byte) {
	w.PutUint32(8, uint32(val))
}

// PutLe16 writes 16 bits in little-endian order.
func (w *Writer) PutLe16(val uint16) {
	w.PutUint32(16, uint32(bswap16(val)))
}

// PutBe16 writes 16 bits in big-endian order.
func (w *Writer) PutBe16(val uint16) {
	w.PutUint32(16, uint32(val))
}

// PutLe32 writes 32 bits in little-endian order.
func (w *Writer) PutLe32(val uint32) {
	w.PutUint32(32, bswap32(val))
}

// PutBe32 writes 32 bits in big-endian order.
func (w *Writer) PutBe32(val uint32) {
	w.PutUint32(32, val)
}

// PutLe64 writes 64 bits in little-endian order.
func (w *Writer) PutLe64(val uint64) {
	w.PutLe32(uint32(val))
	w.PutLe32(uint32(val >> 32))
}

// PutBe64 writes 64 bits in big-endian order.
func (w *Writer) PutBe64(val uint64) {
	w.PutUint64(64, val)
}

// PutUint8 writes up to 8 bits.
func (w *Writer) PutUint8(bits uint, val byte) {
	w.PutUint32(bits, uint32(val))
}

// PutInt8 writes up to 8 signed bits.
func (w *Writer) PutInt8(bits uint, val int8) {
	w.PutUint32(bits, uint32(val))
}

// PutUint16 writes up to 16 bits in big-endian order.
func (w *Writer) PutUint16(bits uint, val uint16) {
	w.PutUint32(bits, uint32(val))
}

// PutInt16 writes up to 16 signed bits in big-endian order.
func (w *Writer) PutInt16(bits uint, val int16) {
	w.PutUint32(bits, uint32(val))
}

// PutInt32 writes up to 32 signed bits in big-endian order.
func (w *Writer) PutInt32(bits uint, val int32) {
	w.PutUint32(bits, uint32(val))
}

// PutInt64 writes up to 64 signed bits in big-endian order.
func (w *Writer) PutInt64(bits uint, val int64) {
	w.PutUint64(bits, uint64(val))
}

// Flush flushes the writer to its underlying buffer.
// Returns ErrUnderflow if the output is not byte-aligned.
// Returns ErrOverflow if the output array is too small.
func (w *Writer) Flush() error {
	for w.fill >= 8 && w.idx < len(w.dst) {
		w.dst[w.idx] = byte(w.cache >> 56)
		w.idx += 1
		w.cache <<= 8
		w.fill -= 8
	}
	if w.idx+int(w.fill) > len(w.dst) {
		return ErrOverflow
	}
	if w.fill != 0 {
		return ErrUnderflow
	}
	return nil
}

// Write writes a whole slice <p> at once.
// Returns an error if the writer is not byte-aligned.
func (w *Writer) Write(p []byte) (int, error) {
	err := w.Flush()
	if err != nil {
		return 0, err
	}
	n := 0
	if w.idx < len(w.dst) {
		n = copy(w.dst[w.idx:], p)
	}
	w.idx += len(p)
	if n != len(p) {
		return n, ErrOverflow
	}
	return n, nil
}

// Index returns the current writer position in bits.
func (w *Writer) Index() int {
	return w.idx<<3 + int(w.fill)
}

func imin(a, b int) int {
	if a > b {
		return b
	}
	return a
}

// Bits returns the number of bits available to write.
func (w *Writer) Bits() int {
	size := len(w.dst)
	return size<<3 - imin(w.idx<<3+int(w.fill), size<<3)
}

// Bytes returns a byte array of what's left to write.
// Note that this array is 8-bit aligned even if the writer is not.
func (w *Writer) Bytes() []byte {
	skip := w.idx + int(w.fill>>3)
	if skip >= len(w.dst) {
		return w.dst[:0]
	}
	return w.dst[skip:len(w.dst)]
}

// Reset resets the writer to its initial position.
func (w *Writer) Reset() {
	w.fill = 0
	w.idx = 0
}
<|endoftext|>


<commit_before>package wx

import (
	"fmt"
	"math"

	"github.com/hillbig/fixvec"
	"github.com/hillbig/rsdic"
	"github.com/hillbig/vecstring"

	"github.com/ugorji/go/codec"
)

type wxImpl struct {
	branches            vecstring.VecStringForWX
	terminals           rsdic.RSDic
	leadingIDs          fixvec.FixVec
	leadings            vecstring.VecString
	leadingIsZeroOrOnes rsdic.RSDic
	leadingIsOnes       rsdic.RSDic
	leadingOnes         []byte
	num                 uint64
}

func (wx wxImpl) ExactMatch(str string) (id uint64, found bool) {
	ret, found := wx.LongestPrefixMatch(str)
	if found == true && int(ret.Len) == len(str) {
		return ret.ID, found
	} else {
		return 0, false
	}
}

func (wx wxImpl) LongestPrefixMatch(str string) (WXString, bool) {
	rets := wx.CommonPrefixMatch(str)
	if len(rets) > 0 {
		return rets[len(rets)-1], true
	} else {
		return WXString{}, false
	}
}

func (wx wxImpl) CommonPrefixMatch(str string) []WXString {
	return wx.CommonPrefixMatchWithLimit(str, math.MaxUint64)
}

func (wx wxImpl) CommonPrefixMatchWithLimit(str string, limit uint64) (rets []WXString) {
	rets = make([]WXString, 0)
	if limit == 0 {
		return
	}
	nodeind := uint64(0)
	strind := uint64(0)
	strlen := uint64(len(str))
	for {
		l, ok := wx.prefixMatchLeading(nodeind, str[strind:])
		strind += l
		if !ok {
			return
		}
		ok, id := wx.terminals.BitAndRank(nodeind)
		if ok {
			rets = append(rets, WXString{id, strind})
			if uint64(len(rets)) == limit {
				return
			}
		}
		if strind == strlen {
			return
		}
		nodeind, ok = wx.branches.FindZeroRank(nodeind, str[strind])
		if !ok {
			return
		}
		nodeind++
		strind++
	}
	// don't come here
}

func (wx wxImpl) getLeading(nodeind uint64) string {
	isZeroOrOne, rank := wx.leadingIsZeroOrOnes.BitAndRank(nodeind)
	if isZeroOrOne {
		isOne, rank := wx.leadingIsOnes.BitAndRank(rank)
		if isOne {
			s := make([]byte, 1)
			s[0] = wx.leadingOnes[rank]
			return string(s)
		} else {
			return ""
		}
	} else {
		id := wx.leadingIDs.Get(rank)
		return wx.leadings.Get(id)
	}
}

func (wx wxImpl) prefixMatchLeading(nodeind uint64, str string) (uint64, bool) {
	isZeroOrOne, rank := wx.leadingIsZeroOrOnes.BitAndRank(nodeind)
	if isZeroOrOne {
		isOne, rank := wx.leadingIsOnes.BitAndRank(rank)
		if isOne {
			if len(str) > 0 && wx.leadingOnes[rank] == str[0] {
				return 1, true
			} else {
				return 0, false
			}
		} else {
			return 0, true
		}
	} else {
		id := wx.leadingIDs.Get(rank)
		return wx.leadings.PrefixMatch(id, str)
	}
}

func (wx wxImpl) PredictiveMatch(str string) (ids []uint64) {
	return wx.PredictiveMatchWithLimit(str, math.MaxUint64)
}

func (wx wxImpl) PredictiveMatchWithLimit(str string, limit uint64) (ids []uint64) {
	ids = make([]uint64, 0)
	if limit == 0 {
		return
	}
	nodeind := uint64(0)
	strind := uint64(0)
	strlen := uint64(len(str))
	for {
		l, ok := wx.prefixMatchLeading(nodeind, str[strind:])
		strind += l
		if !ok {
			if strind != strlen {
				return
			} else {
				break
			}
		}
		if strind == strlen {
			break
		}
		nodeind, ok = wx.branches.FindZeroRank(nodeind, str[strind])
		if !ok {
			return
		}
		nodeind++
		strind++
	}
	// strind == strlen
	wx.enumerateDescendant(nodeind, limit, &ids)
	return
}

func (wx wxImpl) enumerateDescendant(nodeind uint64, limit uint64, ids *[]uint64) {
	ok, id := wx.terminals.BitAndRank(nodeind)
	if ok {
		*ids = append(*ids, id)
		if uint64(len(*ids)) == limit {
			return
		}
	}
	offset, l := wx.branches.OffsetAndLen(nodeind)
	for i := uint64(0); i < l; i++ {
		wx.enumerateDescendant(offset+i+1, limit, ids)
		if uint64(len(*ids)) == limit {
			break
		}
	}
}

func (wx wxImpl) Get(id uint64) string {
	nodeind := wx.terminals.Select(id, true)
	strs := make([]string, 0)
	for {
		str := wx.getLeading(nodeind)
		strs = append(strs, str)
		if nodeind == 0 {
			break
		}
		s := make([]byte, 1)
		s[0] = wx.branches.GetByte(nodeind - 1)
		strs = append(strs, string(s))
		nodeind = wx.branches.IthCharInd(nodeind)
	}

	ret := make([]byte, 0)
	for i := len(strs) - 1; i >= 0; i-- {
		ret = append(ret, []byte(strs[i])...)
	}
	return string(ret)
}

func (wx wxImpl) Num() uint64 {
	return wx.num
}

func (wx wxImpl) MarshalBinary() (out []byte, err error) {
	var bh codec.MsgpackHandle
	enc := codec.NewEncoderBytes(&out, &bh)
	err = enc.Encode(wx.branches)
	if err != nil {
		return
	}
	err = enc.Encode(wx.terminals)
	if err != nil {
		return
	}
	err = enc.Encode(wx.leadingIDs)
	if err != nil {
		return
	}
	err = enc.Encode(wx.leadings)
	if err != nil {
		return
	}
	err = enc.Encode(wx.leadingIsZeroOrOnes)
	if err != nil {
		return
	}
	err = enc.Encode(wx.leadingIsOnes)
	if err != nil {
		return
	}
	err = enc.Encode(wx.leadingOnes)
	if err != nil {
		return
	}
	err = enc.Encode(wx.num)
	if err != nil {
		return
	}
	return
}

func (wx *wxImpl) UnmarshalBinary(in []byte) (err error) {
	var bh codec.MsgpackHandle
	dec := codec.NewDecoderBytes(in, &bh)
	err = dec.Decode(&wx.branches)
	if err != nil {
		return
	}
	err = dec.Decode(&wx.terminals)
	if err != nil {
		return
	}
	err = dec.Decode(&wx.leadingIDs)
	if err != nil {
		return
	}
	err = dec.Decode(&wx.leadings)
	if err != nil {
		return
	}
	err = dec.Decode(&wx.leadingIsZeroOrOnes)
	if err != nil {
		return
	}
	err = dec.Decode(&wx.leadingIsOnes)
	if err != nil {
		return
	}
	err = dec.Decode(&wx.leadingOnes)
	if err != nil {
		return
	}
	err = dec.Decode(&wx.num)
	if err != nil {
		return
	}
	return nil
}

func debugPrintVecString(name string, vs vecstring.VecString) {
	fmt.Printf("%s num=%d\n", name, vs.Num())
	num := vs.Num()
	for i := uint64(0); i < num; i++ {
		s := vs.Get(i)
		fmt.Printf("%d:%s\n", i, s)
	}
}

func debugPrintRSDic(name string, rs rsdic.RSDic) {
	fmt.Printf("%s num=%d\n", name, rs.Num())
	num := rs.Num()
	for i := uint64(0); i < num; i++ {
		if rs.Bit(i) {
			fmt.Printf("1")
		} else {
			fmt.Printf("0")
		}
	}
	fmt.Printf("\n")
}

func debugPrintFixVec(name string, fv fixvec.FixVec) {
	fmt.Printf("%s num=%d blen=%d\n", name, fv.Num(), fv.Blen())
	num := fv.Num()
	for i := uint64(0); i < num; i++ {
		fmt.Printf("%d ", fv.Get(i))
	}
	fmt.Printf("\n")
}

func debugPrintBytes(name string, bytes []byte) {
	fmt.Printf("%s num=%d\n", name, len(bytes))
	for _, v := range bytes {
		fmt.Printf("%c ", v)
	}
	fmt.Printf("\n")
}

func (wx wxImpl) debugPrint() {
	fmt.Printf("num=%d\n", wx.num)
	debugPrintVecString("branches", wx.branches)
	debugPrintRSDic("terminals", wx.terminals)
	debugPrintFixVec("leadingIDs", wx.leadingIDs)
	debugPrintVecString("leadings", wx.leadings)
	debugPrintRSDic("leadingIsZeroOrOnes", wx.leadingIsZeroOrOnes)
	debugPrintRSDic("leadingIsOnes", wx.leadingIsOnes)
	debugPrintBytes("leadingOnes", wx.leadingOnes)
}
<commit_msg>follow the update of vecstring interface<commit_after>package wx

import (
	"fmt"
	"math"

	"github.com/hillbig/fixvec"
	"github.com/hillbig/rsdic"
	"github.com/hillbig/vecstring"

	"github.com/ugorji/go/codec"
)

type wxImpl struct {
	branches            vecstring.VecStringForWX
	terminals           rsdic.RSDic
	leadingIDs          fixvec.FixVec
	leadings            vecstring.VecString
	leadingIsZeroOrOnes rsdic.RSDic
	leadingIsOnes       rsdic.RSDic
	leadingOnes         []byte
	num                 uint64
}

func (wx wxImpl) ExactMatch(str string) (id uint64, found bool) {
	ret, found := wx.LongestPrefixMatch(str)
	if found == true && int(ret.Len) == len(str) {
		return ret.ID, found
	} else {
		return 0, false
	}
}

func (wx wxImpl) LongestPrefixMatch(str string) (WXString, bool) {
	rets := wx.CommonPrefixMatch(str)
	if len(rets) > 0 {
		return rets[len(rets)-1], true
	} else {
		return WXString{}, false
	}
}

func (wx wxImpl) CommonPrefixMatch(str string) []WXString {
	return wx.CommonPrefixMatchWithLimit(str, math.MaxUint64)
}

func (wx wxImpl) CommonPrefixMatchWithLimit(str string, limit uint64) (rets []WXString) {
	rets = make([]WXString, 0)
	if limit == 0 {
		return
	}
	nodeind := uint64(0)
	strind := uint64(0)
	strlen := uint64(len(str))
	for {
		l, ok := wx.prefixMatchLeading(nodeind, str[strind:])
		strind += l
		if !ok {
			return
		}
		ok, id := wx.terminals.BitAndRank(nodeind)
		if ok {
			rets = append(rets, WXString{id, strind})
			if uint64(len(rets)) == limit {
				return
			}
		}
		if strind == strlen {
			return
		}
		nodeind, ok = wx.branches.FindZeroRank(nodeind, str[strind])
		if !ok {
			return
		}
		nodeind++
		strind++
	}
	// don't come here
}

func (wx wxImpl) getLeading(nodeind uint64) string {
	isZeroOrOne, rank := wx.leadingIsZeroOrOnes.BitAndRank(nodeind)
	if isZeroOrOne {
		isOne, rank := wx.leadingIsOnes.BitAndRank(rank)
		if isOne {
			s := make([]byte, 1)
			s[0] = wx.leadingOnes[rank]
			return string(s)
		} else {
			return ""
		}
	} else {
		id := wx.leadingIDs.Get(rank)
		return wx.leadings.Get(id)
	}
}

func (wx wxImpl) prefixMatchLeading(nodeind uint64, str string) (uint64, bool) {
	isZeroOrOne, rank := wx.leadingIsZeroOrOnes.BitAndRank(nodeind)
	if isZeroOrOne {
		isOne, rank := wx.leadingIsOnes.BitAndRank(rank)
		if isOne {
			if len(str) > 0 && wx.leadingOnes[rank] == str[0] {
				return 1, true
			} else {
				return 0, false
			}
		} else {
			return 0, true
		}
	} else {
		id := wx.leadingIDs.Get(rank)
		return wx.leadings.PrefixMatch(id, str)
	}
}

func (wx wxImpl) PredictiveMatch(str string) (ids []uint64) {
	return wx.PredictiveMatchWithLimit(str, math.MaxUint64)
}

func (wx wxImpl) PredictiveMatchWithLimit(str string, limit uint64) (ids []uint64) {
	ids = make([]uint64, 0)
	if limit == 0 {
		return
	}
	nodeind := uint64(0)
	strind := uint64(0)
	strlen := uint64(len(str))
	for {
		l, ok := wx.prefixMatchLeading(nodeind, str[strind:])
		strind += l
		if !ok {
			if strind != strlen {
				return
			} else {
				break
			}
		}
		if strind == strlen {
			break
		}
		nodeind, ok = wx.branches.FindZeroRank(nodeind, str[strind])
		if !ok {
			return
		}
		nodeind++
		strind++
	}
	// strind == strlen
	wx.enumerateDescendant(nodeind, limit, &ids)
	return
}

func (wx wxImpl) enumerateDescendant(nodeind uint64, limit uint64, ids *[]uint64) {
	ok, id := wx.terminals.BitAndRank(nodeind)
	if ok {
		*ids = append(*ids, id)
		if uint64(len(*ids)) == limit {
			return
		}
	}
	l, offset := wx.branches.LenAndOffset(nodeind)
	for i := uint64(0); i < l; i++ {
		wx.enumerateDescendant(offset+i+1, limit, ids)
		if uint64(len(*ids)) == limit {
			break
		}
	}
}

func (wx wxImpl) Get(id uint64) string {
	nodeind := wx.terminals.Select(id, true)
	strs := make([]string, 0)
	for {
		str := wx.getLeading(nodeind)
		strs = append(strs, str)
		if nodeind == 0 {
			break
		}
		s := make([]byte, 1)
		s[0] = wx.branches.GetByte(nodeind - 1)
		strs = append(strs, string(s))
		nodeind = wx.branches.IthCharInd(nodeind - 1)
	}

	ret := make([]byte, 0)
	for i := len(strs) - 1; i >= 0; i-- {
		ret = append(ret, []byte(strs[i])...)
	}
	return string(ret)
}

func (wx wxImpl) Num() uint64 {
	return wx.num
}

func (wx wxImpl) MarshalBinary() (out []byte, err error) {
	var bh codec.MsgpackHandle
	enc := codec.NewEncoderBytes(&out, &bh)
	err = enc.Encode(wx.branches)
	if err != nil {
		return
	}
	err = enc.Encode(wx.terminals)
	if err != nil {
		return
	}
	err = enc.Encode(wx.leadingIDs)
	if err != nil {
		return
	}
	err = enc.Encode(wx.leadings)
	if err != nil {
		return
	}
	err = enc.Encode(wx.leadingIsZeroOrOnes)
	if err != nil {
		return
	}
	err = enc.Encode(wx.leadingIsOnes)
	if err != nil {
		return
	}
	err = enc.Encode(wx.leadingOnes)
	if err != nil {
		return
	}
	err = enc.Encode(wx.num)
	if err != nil {
		return
	}
	return
}

func (wx *wxImpl) UnmarshalBinary(in []byte) (err error) {
	var bh codec.MsgpackHandle
	dec := codec.NewDecoderBytes(in, &bh)
	err = dec.Decode(&wx.branches)
	if err != nil {
		return
	}
	err = dec.Decode(&wx.terminals)
	if err != nil {
		return
	}
	err = dec.Decode(&wx.leadingIDs)
	if err != nil {
		return
	}
	err = dec.Decode(&wx.leadings)
	if err != nil {
		return
	}
	err = dec.Decode(&wx.leadingIsZeroOrOnes)
	if err != nil {
		return
	}
	err = dec.Decode(&wx.leadingIsOnes)
	if err != nil {
		return
	}
	err = dec.Decode(&wx.leadingOnes)
	if err != nil {
		return
	}
	err = dec.Decode(&wx.num)
	if err != nil {
		return
	}
	return nil
}

func debugPrintVecString(name string, vs vecstring.VecString) {
	fmt.Printf("%s num=%d\n", name, vs.Num())
	num := vs.Num()
	for i := uint64(0); i < num; i++ {
		s := vs.Get(i)
		fmt.Printf("%d:%s\n", i, s)
	}
}

func debugPrintRSDic(name string, rs rsdic.RSDic) {
	fmt.Printf("%s num=%d\n", name, rs.Num())
	num := rs.Num()
	for i := uint64(0); i < num; i++ {
		if rs.Bit(i) {
			fmt.Printf("1")
		} else {
			fmt.Printf("0")
		}
	}
	fmt.Printf("\n")
}

func debugPrintFixVec(name string, fv fixvec.FixVec) {
	fmt.Printf("%s num=%d blen=%d\n", name, fv.Num(), fv.Blen())
	num := fv.Num()
	for i := uint64(0); i < num; i++ {
		fmt.Printf("%d ", fv.Get(i))
	}
	fmt.Printf("\n")
}

func debugPrintBytes(name string, bytes []byte) {
	fmt.Printf("%s num=%d\n", name, len(bytes))
	for _, v := range bytes {
		fmt.Printf("%c ", v)
	}
	fmt.Printf("\n")
}

func (wx wxImpl) debugPrint() {
	fmt.Printf("num=%d\n", wx.num)
	debugPrintVecString("branches", wx.branches)
	debugPrintRSDic("terminals", wx.terminals)
	debugPrintFixVec("leadingIDs", wx.leadingIDs)
	debugPrintVecString("leadings", wx.leadings)
	debugPrintRSDic("leadingIsZeroOrOnes", wx.leadingIsZeroOrOnes)
	debugPrintRSDic("leadingIsOnes", wx.leadingIsOnes)
	debugPrintBytes("leadingOnes", wx.leadingOnes)
}
<|endoftext|>


<commit_before>package sputter

import (
	"regexp"
	"testing"
)

func TestLiteral(t *testing.T) {
	testSputHundredEmoji(t, `abc`)
}

func TestCharClassSingle(t *testing.T) {
	testSputHundredEmoji(t, `[A]`)
}

func TestCharClassSingleRange(t *testing.T) {
	testSputHundredEmoji(t, `[0-9]`)
}

func TestCharClassMultipleRange(t *testing.T) {
	testSputHundredEmoji(t, `[A-Z0-9]`)
}

func TestConcat(t *testing.T) {
	testSputHundredEmoji(t, `A[0-9]`)
}

func TestRange(t *testing.T) {
	testSputHundredEmoji(t, `A{1,5}`)
}

func TestAlternate(t *testing.T) {
	testSputHundredEmoji(t, `A|B`)
}

func TestPlus(t *testing.T) {
	testSputHundredEmoji(t, `A+`)
}

func TestStar(t *testing.T) {
	testSputHundredEmoji(t, `A*`)
}

func TestQuest(t *testing.T) {
	testSputHundredEmoji(t, `A+`)
}

func TestAnyCharNotNL(t *testing.T) {
	testSputHundredEmoji(t, ".")
}

func TestBeginLine(t *testing.T) {
	// testSputHundredEmoji(t, "a^a")
}

func TestEndLine(t *testing.T) {
	// testSputHundredEmoji(t, "a$a")
}

func testSputHundredEmoji(t *testing.T, exp string) {
	for i := 0; i < 100; i++ {
		s, err := Gen(exp)
		if err != nil {
			t.Error(`error from Gen:`, err)
		}

		match, err := regexp.Match(exp, []byte(s))
		if !match {
			t.Errorf(`output string "%s" does not match expression "%s"`, s, exp)
		}

		if err != nil {
			t.Error(err)
		}
	}
}
<commit_msg>adding test for exact repeat<commit_after>package sputter

import (
	"regexp"
	"testing"
)

func TestLiteral(t *testing.T) {
	testSputHundredEmoji(t, `abc`)
}

func TestCharClassSingle(t *testing.T) {
	testSputHundredEmoji(t, `[A]`)
}

func TestCharClassSingleRange(t *testing.T) {
	testSputHundredEmoji(t, `[0-9]`)
}

func TestCharClassMultipleRange(t *testing.T) {
	testSputHundredEmoji(t, `[A-Z0-9]`)
}

func TestConcat(t *testing.T) {
	testSputHundredEmoji(t, `A[0-9]`)
}

func TestExactRepeat(t *testing.T) {
	testSputHundredEmoji(t, `A{5}`)
}

func TestRange(t *testing.T) {
	testSputHundredEmoji(t, `A{1,5}`)
}

func TestAlternate(t *testing.T) {
	testSputHundredEmoji(t, `A|B`)
}

func TestPlus(t *testing.T) {
	testSputHundredEmoji(t, `A+`)
}

func TestStar(t *testing.T) {
	testSputHundredEmoji(t, `A*`)
}

func TestQuest(t *testing.T) {
	testSputHundredEmoji(t, `A+`)
}

func TestAnyCharNotNL(t *testing.T) {
	testSputHundredEmoji(t, ".")
}

func TestBeginLine(t *testing.T) {
	// testSputHundredEmoji(t, "a^a")
}

func TestEndLine(t *testing.T) {
	// testSputHundredEmoji(t, "a$a")
}

func testSputHundredEmoji(t *testing.T, exp string) {
	for i := 0; i < 100; i++ {
		s, err := Gen(exp)
		if err != nil {
			t.Error(`error from Gen:`, err)
		}

		match, err := regexp.Match(exp, []byte(s))
		if !match {
			t.Errorf(`output string "%s" does not match expression "%s"`, s, exp)
		}

		if err != nil {
			t.Error(err)
		}
	}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"io/ioutil"
	"os"
	"testing"
)

func TestCfgMem(t *testing.T) {
	testCfg(t, NewCfgMem())
}

func TestCfgSimple(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	testCfg(t, NewCfgSimple(emptyDir+string(os.PathSeparator)+"test.cfg"))
}

func testCfg(t *testing.T, c Cfg) {
	v, cas, err := c.Get("nope", 0)
	if err != nil || v != nil || cas != 0 {
		t.Errorf("expected Get() to miss on brand new CfgMem")
	}
	v, cas, err = c.Get("nope", 100)
	if err != nil || v != nil || cas != 0 {
		t.Errorf("expected Get() to miss on brand new CfgMem with wrong CAS")
	}
	cas, err = c.Set("a", []byte("A"), 100)
	if err == nil || cas != 0 {
		t.Errorf("expected creation Set() to fail when no entry and wrong CAS")
	}

	cas1, err := c.Set("a", []byte("A"), 0)
	if err != nil || cas1 != 1 {
		t.Errorf("expected creation Set() to ok CAS 0")
	}
	cas, err = c.Set("a", []byte("A"), 0)
	if err == nil || cas != 0 {
		t.Errorf("expected re-creation Set() to fail with CAS 0")
	}
	cas, err = c.Set("a", []byte("A"), 100)
	if err == nil || cas != 0 {
		t.Errorf("expected update Set() to fail when entry and wrong CAS")
	}
	v, cas, err = c.Get("a", 100)
	if err == nil || v != nil || cas != 0 {
		t.Errorf("expected Get() to fail on wrong CAS")
	}
	v, cas, err = c.Get("a", 0)
	if err != nil || string(v) != "A" || cas != cas1 {
		t.Errorf("expected Get() to succeed on 0 CAS")
	}
	v, cas, err = c.Get("a", cas1)
	if err != nil || string(v) != "A" || cas != cas1 {
		t.Errorf("expected Get() to succeed on right CAS")
	}

	cas2, err := c.Set("a", []byte("AA"), cas1)
	if err != nil || cas2 != 2 {
		t.Errorf("expected update Set() to succeed when right CAS")
	}
	cas, err = c.Set("a", []byte("AA-should-fail"), 0)
	if err == nil || cas != 0 {
		t.Errorf("expected re-creation Set() to fail with CAS 0")
	}
	cas, err = c.Set("a", []byte("AA"), cas1)
	if err == nil || cas != 0 {
		t.Errorf("expected update Set() to fail when retried after success")
	}
	v, cas, err = c.Get("a", 100)
	if err == nil || v != nil || cas != 0 {
		t.Errorf("expected Get() to fail on wrong CAS")
	}
	v, cas, err = c.Get("a", 0)
	if err != nil || string(v) != "AA" || cas != cas2 {
		t.Errorf("expected Get() to succeed on 0 CAS")
	}
	v, cas, err = c.Get("a", cas2)
	if err != nil || string(v) != "AA" || cas != cas2 {
		t.Errorf("expected Get() to succeed on right CAS")
	}

	err = c.Del("nope", 0)
	if err != nil {
		t.Errorf("expected Del() to succeed on missing item when 0 CAS")
	}
	err = c.Del("nope", 100)
	if err == nil {
		t.Errorf("expected Del() to fail on missing item when non-zero CAS")
	}
	v, cas, err = c.Get("a", cas2)
	if err != nil || string(v) != "AA" || cas != cas2 {
		t.Errorf("expected Get() to succeed on right CAS")
	}
	err = c.Del("a", 100)
	if err == nil {
		t.Errorf("expected Del() to fail when wrong CAS")
	}
	v, cas, err = c.Get("a", cas2)
	if err != nil || string(v) != "AA" || cas != cas2 {
		t.Errorf("expected Get() to succeed on right CAS")
	}
	err = c.Del("a", cas2)
	if err != nil {
		t.Errorf("expected Del() to succeed when right CAS")
	}
	v, cas, err = c.Get("a", cas2)
	if err != nil || v != nil || cas != 0 {
		t.Errorf("expected Get() with CAS to miss after Del(): "+
			" %v, %v, %v", err, v, cas)
	}
	v, cas, err = c.Get("a", 0)
	if err != nil || v != nil || cas != 0 {
		t.Errorf("expected Get() with 0 CAS to miss after Del(): "+
			" %v, %v, %v", err, v, cas)
	}
}
<commit_msg>added TestCfgSimpleLoad()<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"io/ioutil"
	"os"
	"testing"
)

func TestCfgMem(t *testing.T) {
	testCfg(t, NewCfgMem())
}

func TestCfgSimple(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := NewCfgSimple(emptyDir + string(os.PathSeparator) + "test.cfg")
	testCfg(t, cfg)
}

func testCfg(t *testing.T, c Cfg) {
	v, cas, err := c.Get("nope", 0)
	if err != nil || v != nil || cas != 0 {
		t.Errorf("expected Get() to miss on brand new CfgMem")
	}
	v, cas, err = c.Get("nope", 100)
	if err != nil || v != nil || cas != 0 {
		t.Errorf("expected Get() to miss on brand new CfgMem with wrong CAS")
	}
	cas, err = c.Set("a", []byte("A"), 100)
	if err == nil || cas != 0 {
		t.Errorf("expected creation Set() to fail when no entry and wrong CAS")
	}

	cas1, err := c.Set("a", []byte("A"), 0)
	if err != nil || cas1 != 1 {
		t.Errorf("expected creation Set() to ok CAS 0")
	}
	cas, err = c.Set("a", []byte("A"), 0)
	if err == nil || cas != 0 {
		t.Errorf("expected re-creation Set() to fail with CAS 0")
	}
	cas, err = c.Set("a", []byte("A"), 100)
	if err == nil || cas != 0 {
		t.Errorf("expected update Set() to fail when entry and wrong CAS")
	}
	v, cas, err = c.Get("a", 100)
	if err == nil || v != nil || cas != 0 {
		t.Errorf("expected Get() to fail on wrong CAS")
	}
	v, cas, err = c.Get("a", 0)
	if err != nil || string(v) != "A" || cas != cas1 {
		t.Errorf("expected Get() to succeed on 0 CAS")
	}
	v, cas, err = c.Get("a", cas1)
	if err != nil || string(v) != "A" || cas != cas1 {
		t.Errorf("expected Get() to succeed on right CAS")
	}

	cas2, err := c.Set("a", []byte("AA"), cas1)
	if err != nil || cas2 != 2 {
		t.Errorf("expected update Set() to succeed when right CAS")
	}
	cas, err = c.Set("a", []byte("AA-should-fail"), 0)
	if err == nil || cas != 0 {
		t.Errorf("expected re-creation Set() to fail with CAS 0")
	}
	cas, err = c.Set("a", []byte("AA"), cas1)
	if err == nil || cas != 0 {
		t.Errorf("expected update Set() to fail when retried after success")
	}
	v, cas, err = c.Get("a", 100)
	if err == nil || v != nil || cas != 0 {
		t.Errorf("expected Get() to fail on wrong CAS")
	}
	v, cas, err = c.Get("a", 0)
	if err != nil || string(v) != "AA" || cas != cas2 {
		t.Errorf("expected Get() to succeed on 0 CAS")
	}
	v, cas, err = c.Get("a", cas2)
	if err != nil || string(v) != "AA" || cas != cas2 {
		t.Errorf("expected Get() to succeed on right CAS")
	}

	err = c.Del("nope", 0)
	if err != nil {
		t.Errorf("expected Del() to succeed on missing item when 0 CAS")
	}
	err = c.Del("nope", 100)
	if err == nil {
		t.Errorf("expected Del() to fail on missing item when non-zero CAS")
	}
	v, cas, err = c.Get("a", cas2)
	if err != nil || string(v) != "AA" || cas != cas2 {
		t.Errorf("expected Get() to succeed on right CAS")
	}
	err = c.Del("a", 100)
	if err == nil {
		t.Errorf("expected Del() to fail when wrong CAS")
	}
	v, cas, err = c.Get("a", cas2)
	if err != nil || string(v) != "AA" || cas != cas2 {
		t.Errorf("expected Get() to succeed on right CAS")
	}
	err = c.Del("a", cas2)
	if err != nil {
		t.Errorf("expected Del() to succeed when right CAS")
	}
	v, cas, err = c.Get("a", cas2)
	if err != nil || v != nil || cas != 0 {
		t.Errorf("expected Get() with CAS to miss after Del(): "+
			" %v, %v, %v", err, v, cas)
	}
	v, cas, err = c.Get("a", 0)
	if err != nil || v != nil || cas != 0 {
		t.Errorf("expected Get() with 0 CAS to miss after Del(): "+
			" %v, %v, %v", err, v, cas)
	}
}

func TestCfgSimpleLoad(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	path := emptyDir + string(os.PathSeparator) + "test.cfg"

	c := NewCfgSimple(path)
	cas1, err := c.Set("a", []byte("A"), 0)
	if err != nil || cas1 != 1 {
		t.Errorf("expected Set() on initial cfg simple")
	}

	c2 := NewCfgSimple(path)
	if err := c2.(*CfgSimple).Load(); err != nil {
		t.Errorf("expected Load() to work")
	}
	v, cas, err := c2.Get("a", 0)
	if err != nil || v == nil || cas != cas1 {
		t.Errorf("expected Get() to succeed")
	}
	if string(v) != "A" {
		t.Errorf("exepcted to read what we wrote")
	}
}
<|endoftext|>


<commit_before>// Loading PNGs with SDL_image
// Adapted from http://lazyfoo.net/tutorials/SDL/06_extension_libraries_and_loading_other_image_formats/index2.php

package main

import (
	"log"

	"github.com/veandco/go-sdl2/sdl"
	"github.com/veandco/go-sdl2/sdl_image"
)

const (
	screenWidth  = 640
	screenHeight = 480
)

var (
	err              error
	window           *sdl.Window
	screenSurface    *sdl.Surface
	stretchedSurface *sdl.Surface
	quit             bool
	event            sdl.Event
)

func initSDL() error {
	err = sdl.Init(sdl.INIT_VIDEO)

	if err != nil {
		return err
	}

	window, err = sdl.CreateWindow("SDL Tutorial", sdl.WINDOWPOS_UNDEFINED, sdl.WINDOWPOS_UNDEFINED, screenWidth, screenHeight, sdl.WINDOW_SHOWN)
	if err != nil {
		return err
	}

	imgFlags := img.INIT_PNG
	imgInitResult := img.Init(imgFlags)
	if (imgInitResult & imgFlags) != imgFlags {
		return img.GetError()
	}

	screenSurface, err = window.GetSurface()
	if err != nil {
		return err
	}

	return nil
}

func loadSurface(path string) (*sdl.Surface, error) {

	loadedSurface, err := img.Load(path)
	if err != nil {
		return nil, err
	}

	optimizedSurface, err := loadedSurface.Convert(screenSurface.Format, 0)
	if err != nil {
		return nil, err
	}

	// Get rid of old loaded surface
	loadedSurface.Free()

	return optimizedSurface, nil
}

func loadMedia() error {
	stretchedSurface, err = loadSurface("stretch.png")
	if err != nil {
		return err
	}

	return nil
}

func close() {

	window.Destroy()
	sdl.Quit()
}

func main() {
	err = initSDL()
	if err != nil {
		log.Fatal("Error initializing SDL:", err)
	}

	err = loadMedia()
	if err != nil {
		log.Fatal("Error loading Media:", err)
	}

	var stretchRect sdl.Rect
	stretchRect.X = 0
	stretchRect.Y = 0
	stretchRect.W = screenWidth
	stretchRect.H = screenHeight

	stretchedSurface.BlitScaled(nil, screenSurface, &stretchRect)
	window.UpdateSurface()

	quit = false
	for !quit {
		for event = sdl.PollEvent(); event != nil; event = sdl.PollEvent() {
			switch event.(type) {
			case *sdl.QuitEvent:
				quit = true
			}
		}
	}

	close()
}
<commit_msg>Add call to img.Quit<commit_after>// Loading PNGs with SDL_image
// Adapted from http://lazyfoo.net/tutorials/SDL/06_extension_libraries_and_loading_other_image_formats/index2.php

package main

import (
	"log"

	"github.com/veandco/go-sdl2/sdl"
	"github.com/veandco/go-sdl2/sdl_image"
)

const (
	screenWidth  = 640
	screenHeight = 480
)

var (
	err              error
	window           *sdl.Window
	screenSurface    *sdl.Surface
	stretchedSurface *sdl.Surface
	quit             bool
	event            sdl.Event
)

func initSDL() error {
	err = sdl.Init(sdl.INIT_VIDEO)

	if err != nil {
		return err
	}

	window, err = sdl.CreateWindow("SDL Tutorial", sdl.WINDOWPOS_UNDEFINED, sdl.WINDOWPOS_UNDEFINED, screenWidth, screenHeight, sdl.WINDOW_SHOWN)
	if err != nil {
		return err
	}

	imgFlags := img.INIT_PNG
	imgInitResult := img.Init(imgFlags)
	if (imgInitResult & imgFlags) != imgFlags {
		return img.GetError()
	}

	screenSurface, err = window.GetSurface()
	if err != nil {
		return err
	}

	return nil
}

func loadSurface(path string) (*sdl.Surface, error) {

	loadedSurface, err := img.Load(path)
	if err != nil {
		return nil, err
	}

	optimizedSurface, err := loadedSurface.Convert(screenSurface.Format, 0)
	if err != nil {
		return nil, err
	}

	// Get rid of old loaded surface
	loadedSurface.Free()

	return optimizedSurface, nil
}

func loadMedia() error {
	stretchedSurface, err = loadSurface("stretch.png")
	if err != nil {
		return err
	}

	return nil
}

func close() {

	window.Destroy()
	img.Quit()
	sdl.Quit()
}

func main() {
	err = initSDL()
	if err != nil {
		log.Fatal("Error initializing SDL:", err)
	}

	err = loadMedia()
	if err != nil {
		log.Fatal("Error loading Media:", err)
	}

	var stretchRect sdl.Rect
	stretchRect.X = 0
	stretchRect.Y = 0
	stretchRect.W = screenWidth
	stretchRect.H = screenHeight

	stretchedSurface.BlitScaled(nil, screenSurface, &stretchRect)
	window.UpdateSurface()

	quit = false
	for !quit {
		for event = sdl.PollEvent(); event != nil; event = sdl.PollEvent() {
			switch event.(type) {
			case *sdl.QuitEvent:
				quit = true
			}
		}
	}

	close()
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccDataSourceAwsApiGatewayVpcLink(t *testing.T) {
	rName := acctest.RandString(8)
	resource.ParallelTest(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccDataSourceAwsApiGatewayVpcLinkConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccDataSourceAwsApiGatewayVpcLinkCheck("data.aws_api_gateway_vpc_link.vpc_link"),
					resource.TestCheckResourceAttrSet("data.aws_api_gateway_vpc_link.vpc_link", "id"),
				),
			},
		},
	})
}

func testAccDataSourceAwsApiGatewayVpcLinkCheck(name string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		resources, ok := s.RootModule().Resources[name]
		if !ok {
			return fmt.Errorf("root module has no resource called %s", name)
		}

		apiGatewayVpcLinkResources, ok := s.RootModule().Resources["aws_api_gateway_vpc_link.vpc_link"]
		if !ok {
			return fmt.Errorf("can't find aws_api_gateway_vpc_link.vpc_link in state")
		}

		attr := resources.Primary.Attributes
		if attr["name"] != apiGatewayVpcLinkResources.Primary.Attributes["name"] {
			return fmt.Errorf(
				"name is %s; want %s",
				attr["name"],
				apiGatewayVpcLinkResources.Primary.Attributes["name"],
			)
		}

		return nil
	}
}

func testAccDataSourceAwsApiGatewayVpcLinkConfig(r string) string {
	return fmt.Sprintf(`
	resource "aws_vpc" "apigateway_vpclink_test" {
		cidr_block = "10.0.0.0/16"

		tags = {
		  Name = "terraform-testacc-lb-apigateway-vpc-link"
		}
	  }

	  resource "aws_lb" "apigateway_vpclink_test" {
		name = "%s"

		subnets = [
		  "${aws_subnet.apigateway_vpclink_test_subnet1.id}"
		]

		load_balancer_type               = "network"
		internal                         = true
		idle_timeout                     = 60
		enable_deletion_protection       = false
		enable_cross_zone_load_balancing = false

		tags = {
		  Name = "testAccDataSourceAwsApiGatewayVpcLinkConfig_networkLoadbalancer"
		}
	  }

	  resource "aws_lb" "apigateway_vpclink_test2" {
		name = "%s-wrong"

		subnets = [
		  "${aws_subnet.apigateway_vpclink_test_subnet1.id}"
		]

		load_balancer_type               = "network"
		internal                         = true
		idle_timeout                     = 60
		enable_deletion_protection       = false
		enable_cross_zone_load_balancing = false

		tags = {
		  Name = "testAccDataSourceAwsApiGatewayVpcLinkConfig_networkLoadbalancer"
		}
	  }

	  resource "aws_subnet" "apigateway_vpclink_test_subnet1" {
		vpc_id            = "${aws_vpc.apigateway_vpclink_test.id}"
		cidr_block        = "10.0.1.0/24"
		availability_zone = "us-west-2a"

		tags = {
		  Name = "tf-acc-lb-apigateway-vpclink"
		}
	  }

	  resource "aws_api_gateway_vpc_link" "vpc_link" {
		name = "%s"
		target_arns = ["${aws_lb.apigateway_vpclink_test.arn}"]

	  }

	  resource "aws_api_gateway_vpc_link" "vpc_link2" {
		name = "%s-wrong"
		target_arns = ["${aws_lb.apigateway_vpclink_test2.arn}"]

	  }

	  data "aws_api_gateway_vpc_link" "vpc_link" {
		  name = "${aws_api_gateway_vpc_link.vpc_link.name}"
	  }

`, r, r, r, r)
}
<commit_msg>Update aws/data_source_aws_api_gateway_vpc_link_test.go<commit_after>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccDataSourceAwsApiGatewayVpcLink(t *testing.T) {
	rName := acctest.RandString(8)
	resource.ParallelTest(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccDataSourceAwsApiGatewayVpcLinkConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccDataSourceAwsApiGatewayVpcLinkCheck("data.aws_api_gateway_vpc_link.vpc_link"),
					resource.TestCheckResourceAttrSet("data.aws_api_gateway_vpc_link.vpc_link", "id", "aws_api_gateway_vpc_link.vpc_link", "id"),
				),
			},
		},
	})
}

func testAccDataSourceAwsApiGatewayVpcLinkCheck(name string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		resources, ok := s.RootModule().Resources[name]
		if !ok {
			return fmt.Errorf("root module has no resource called %s", name)
		}

		apiGatewayVpcLinkResources, ok := s.RootModule().Resources["aws_api_gateway_vpc_link.vpc_link"]
		if !ok {
			return fmt.Errorf("can't find aws_api_gateway_vpc_link.vpc_link in state")
		}

		attr := resources.Primary.Attributes
		if attr["name"] != apiGatewayVpcLinkResources.Primary.Attributes["name"] {
			return fmt.Errorf(
				"name is %s; want %s",
				attr["name"],
				apiGatewayVpcLinkResources.Primary.Attributes["name"],
			)
		}

		return nil
	}
}

func testAccDataSourceAwsApiGatewayVpcLinkConfig(r string) string {
	return fmt.Sprintf(`
	resource "aws_vpc" "apigateway_vpclink_test" {
		cidr_block = "10.0.0.0/16"

		tags = {
		  Name = "terraform-testacc-lb-apigateway-vpc-link"
		}
	  }

	  resource "aws_lb" "apigateway_vpclink_test" {
		name = "%s"

		subnets = [
		  "${aws_subnet.apigateway_vpclink_test_subnet1.id}"
		]

		load_balancer_type               = "network"
		internal                         = true
		idle_timeout                     = 60
		enable_deletion_protection       = false
		enable_cross_zone_load_balancing = false

		tags = {
		  Name = "testAccDataSourceAwsApiGatewayVpcLinkConfig_networkLoadbalancer"
		}
	  }

	  resource "aws_lb" "apigateway_vpclink_test2" {
		name = "%s-wrong"

		subnets = [
		  "${aws_subnet.apigateway_vpclink_test_subnet1.id}"
		]

		load_balancer_type               = "network"
		internal                         = true
		idle_timeout                     = 60
		enable_deletion_protection       = false
		enable_cross_zone_load_balancing = false

		tags = {
		  Name = "testAccDataSourceAwsApiGatewayVpcLinkConfig_networkLoadbalancer"
		}
	  }

	  resource "aws_subnet" "apigateway_vpclink_test_subnet1" {
		vpc_id            = "${aws_vpc.apigateway_vpclink_test.id}"
		cidr_block        = "10.0.1.0/24"
		availability_zone = "us-west-2a"

		tags = {
		  Name = "tf-acc-lb-apigateway-vpclink"
		}
	  }

	  resource "aws_api_gateway_vpc_link" "vpc_link" {
		name = "%s"
		target_arns = ["${aws_lb.apigateway_vpclink_test.arn}"]

	  }

	  resource "aws_api_gateway_vpc_link" "vpc_link2" {
		name = "%s-wrong"
		target_arns = ["${aws_lb.apigateway_vpclink_test2.arn}"]

	  }

	  data "aws_api_gateway_vpc_link" "vpc_link" {
		  name = "${aws_api_gateway_vpc_link.vpc_link.name}"
	  }

`, r, r, r, r)
}
<|endoftext|>


<commit_before>package text

import (
	"github.com/opsidian/parsley/data"
	"github.com/opsidian/parsley/parser"
	"github.com/opsidian/parsley/parsley"
)

// LeftTrim skips the whitespaces before it tries to match the given parser
func LeftTrim(p parsley.Parser, wsMode WsMode) parser.Func {
	return parser.Func(func(h parsley.History, leftRecCtx data.IntMap, r parsley.Reader, pos int) (data.IntSet, parsley.Node, parsley.Error) {
		return p.Parse(h, leftRecCtx, r, r.(*Reader).SkipWhitespaces(pos, wsMode))
	})
}

// RightTrim reads and skips the whitespaces after any parser matches and updates the reader position
func RightTrim(p parsley.Parser, wsMode WsMode) parser.Func {
	return parser.Func(func(h parsley.History, leftRecCtx data.IntMap, r parsley.Reader, pos int) (data.IntSet, parsley.Node, parsley.Error) {
		tr := r.(*Reader)
		cp, res, err := p.Parse(h, leftRecCtx, r, pos)
		if res != nil {
			res.SetReaderPos(func(pos int) int { return tr.SkipWhitespaces(pos, wsMode) })
		}
		return cp, res, err
	})
}
<commit_msg>Use ast.SetReaderPos set amend the reader position<commit_after>package text

import (
	"github.com/opsidian/parsley/ast"
	"github.com/opsidian/parsley/data"
	"github.com/opsidian/parsley/parser"
	"github.com/opsidian/parsley/parsley"
)

// LeftTrim skips the whitespaces before it tries to match the given parser
func LeftTrim(p parsley.Parser, wsMode WsMode) parser.Func {
	return parser.Func(func(h parsley.History, leftRecCtx data.IntMap, r parsley.Reader, pos int) (data.IntSet, parsley.Node, parsley.Error) {
		return p.Parse(h, leftRecCtx, r, r.(*Reader).SkipWhitespaces(pos, wsMode))
	})
}

// RightTrim reads and skips the whitespaces after any parser matches and updates the reader position
func RightTrim(p parsley.Parser, wsMode WsMode) parser.Func {
	return parser.Func(func(h parsley.History, leftRecCtx data.IntMap, r parsley.Reader, pos int) (data.IntSet, parsley.Node, parsley.Error) {
		tr := r.(*Reader)
		cp, res, err := p.Parse(h, leftRecCtx, r, pos)
		if res != nil {
			res = ast.SetReaderPos(res, func(pos int) int { return tr.SkipWhitespaces(pos, wsMode) })
		}
		return cp, res, err
	})
}
<|endoftext|>


<commit_before>//
// Use and distribution licensed under the Apache license version 2.
//
// See the COPYING file in the root project directory for full text.
//

package ghw

import (
	"testing"
)

// nolint: gocyclo
func TestTopology(t *testing.T) {
	info, err := Topology()

	if err != nil {
		t.Fatalf("Expected nil err, but got %v", err)
	}
	if info == nil {
		t.Fatalf("Expected non-nil TopologyInfo, but got nil")
	}

	if len(info.Nodes) == 0 {
		t.Fatalf("Expected >0 nodes but got 0.")
	}

	if info.Architecture == NUMA && len(info.Nodes) == 1 {
		t.Fatalf("Got NUMA architecture but only 1 node.")
	}

	for _, n := range info.Nodes {
		if len(n.Cores) == 0 {
			t.Fatalf("Expected >0 cores but got 0.")
		}
		for _, c := range n.Cores {
			if len(c.LogicalProcessors) == 0 {
				t.Fatalf("Expected >0 logical processors but got 0.")
			}
			if uint32(len(c.LogicalProcessors)) != c.NumThreads {
				t.Fatalf(
					"Expected NumThreads == len(logical procs) but %d != %d",
					c.NumThreads,
					len(c.LogicalProcessors),
				)
			}
		}
		if len(n.Caches) == 0 {
			t.Fatalf("Expected >0 caches but got 0.")
		}
	}
}
<commit_msg>fix topology_test.go to refer to new ARCHITECTURE_NUMA<commit_after>//
// Use and distribution licensed under the Apache license version 2.
//
// See the COPYING file in the root project directory for full text.
//

package ghw

import (
	"testing"
)

// nolint: gocyclo
func TestTopology(t *testing.T) {
	info, err := Topology()

	if err != nil {
		t.Fatalf("Expected nil err, but got %v", err)
	}
	if info == nil {
		t.Fatalf("Expected non-nil TopologyInfo, but got nil")
	}

	if len(info.Nodes) == 0 {
		t.Fatalf("Expected >0 nodes but got 0.")
	}

	if info.Architecture == ARCHITECTURE_NUMA && len(info.Nodes) == 1 {
		t.Fatalf("Got NUMA architecture but only 1 node.")
	}

	for _, n := range info.Nodes {
		if len(n.Cores) == 0 {
			t.Fatalf("Expected >0 cores but got 0.")
		}
		for _, c := range n.Cores {
			if len(c.LogicalProcessors) == 0 {
				t.Fatalf("Expected >0 logical processors but got 0.")
			}
			if uint32(len(c.LogicalProcessors)) != c.NumThreads {
				t.Fatalf(
					"Expected NumThreads == len(logical procs) but %d != %d",
					c.NumThreads,
					len(c.LogicalProcessors),
				)
			}
		}
		if len(n.Caches) == 0 {
			t.Fatalf("Expected >0 caches but got 0.")
		}
	}
}
<|endoftext|>


<commit_before>package toystore

import (
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"sync"
	"syscall"
	"testing"
	"time"
)

var numTests int = 100
var cmds []*exec.Cmd = []*exec.Cmd{}
var m *sync.Mutex = &sync.Mutex{}
var hosts []string = []string{
	"127.0.0.2",
	"127.0.0.3",
	"127.0.0.4",
	"127.0.0.5",
	"127.0.0.6",
}

func host() string {
	return fmt.Sprintf("http://%s:3000", hosts[rand.Intn(len(hosts))])
}

func startNode(host string) {
	log.Println("Starting node", host)
	args := []string{"run", "api/http.go", host}
	cmd := exec.Command("go", args...)
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	go cmd.Run()

	m.Lock()
	cmds = append(cmds, cmd)
	m.Unlock()
}

func startCluster() {
	log.Println("Starting cluster")

	startNode(hosts[0])

	time.Sleep(time.Second)

	for _, host := range hosts[1:] {
		startNode(host)
	}

	log.Println("Waiting for cluster")
	time.Sleep(time.Second * 30)
	log.Println("Cluster running")
}

func stopCluster() {
	for _, cmd := range cmds {
		log.Println("Killing", cmd.Process.Pid)
		pgid, err := syscall.Getpgid(cmd.Process.Pid)
		if err == nil {
			syscall.Kill(-pgid, 15) // note the minus sign
		} else {
			log.Println("Failed to kill", cmd.Process.Pid)
		}
	}

}

func TestBasicData(t *testing.T) {
	startCluster()
	defer stopCluster()

	for i := 0; i < numTests; i++ {
		key := fmt.Sprintf("basic-%d", i)
		value := fmt.Sprintf("basic-value-%d", i)
		h := host()
		data := url.Values{"key": {key}, "value": {value}}

		log.Printf("Put %s/%s on %s", key, value, h)
		_, err := http.PostForm(h, data)
		log.Printf("POST Request complete %s/%s", h, key)

		if err != nil {
			t.Error(err)
		}
	}

	time.Sleep(time.Second)

	for i := 0; i < numTests; i++ {
		h := host()
		key := fmt.Sprintf("basic-%d", i)
		value := fmt.Sprintf("basic-value-%d", i)
		log.Printf("Get %s on %s", key, h)
		resp, err := http.Get(h + "/" + key)
		log.Printf("GET Request complete %s/%s", h, key)

		if err != nil {
			t.Fatal("Error", err)
		}

		defer resp.Body.Close()

		body, err := ioutil.ReadAll(resp.Body)

		if string(body) != value {
			t.Errorf("%s/%s %s != %s", h, key, string(body), value)
		}
	}
}
<commit_msg>Reduce test timeout<commit_after>package toystore

import (
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"sync"
	"syscall"
	"testing"
	"time"
)

var numTests int = 100
var cmds []*exec.Cmd = []*exec.Cmd{}
var m *sync.Mutex = &sync.Mutex{}
var hosts []string = []string{
	"127.0.0.2",
	"127.0.0.3",
	"127.0.0.4",
	"127.0.0.5",
	"127.0.0.6",
}

func host() string {
	return fmt.Sprintf("http://%s:3000", hosts[rand.Intn(len(hosts))])
}

func startNode(host string) {
	log.Println("Starting node", host)
	args := []string{"run", "api/http.go", host}
	cmd := exec.Command("go", args...)
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	go cmd.Run()

	m.Lock()
	cmds = append(cmds, cmd)
	m.Unlock()
}

func startCluster() {
	log.Println("Starting cluster")

	startNode(hosts[0])

	time.Sleep(time.Second)

	for _, host := range hosts[1:] {
		startNode(host)
	}

	log.Println("Waiting for cluster")
	time.Sleep(time.Second * 10)
	log.Println("Cluster running")
}

func stopCluster() {
	for _, cmd := range cmds {
		log.Println("Killing", cmd.Process.Pid)
		pgid, err := syscall.Getpgid(cmd.Process.Pid)
		if err == nil {
			syscall.Kill(-pgid, 15) // note the minus sign
		} else {
			log.Println("Failed to kill", cmd.Process.Pid)
		}
	}

}

func TestBasicData(t *testing.T) {
	startCluster()
	defer stopCluster()

	for i := 0; i < numTests; i++ {
		key := fmt.Sprintf("basic-%d", i)
		value := fmt.Sprintf("basic-value-%d", i)
		h := host()
		data := url.Values{"key": {key}, "value": {value}}

		_, err := http.PostForm(h, data)

		if err != nil {
			t.Error(err)
		}
	}

	time.Sleep(time.Second)

	for i := 0; i < numTests; i++ {
		h := host()
		key := fmt.Sprintf("basic-%d", i)
		value := fmt.Sprintf("basic-value-%d", i)
		resp, err := http.Get(h + "/" + key)

		if err != nil {
			t.Fatal("Error", err)
		}

		defer resp.Body.Close()

		body, err := ioutil.ReadAll(resp.Body)

		if string(body) != value {
			t.Errorf("%s/%s %s != %s", h, key, string(body), value)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package lang provides template functions for content internationalization.
package lang

import (
	"fmt"
	"math"
	"strconv"
	"strings"

	translators "github.com/gohugoio/localescompressed"
	"github.com/gohugoio/locales"
	"github.com/pkg/errors"

	"github.com/gohugoio/hugo/deps"
	"github.com/spf13/cast"
)

// New returns a new instance of the lang-namespaced template functions.
func New(deps *deps.Deps, translator locales.Translator) *Namespace {
	return &Namespace{
		translator: translator,
		deps:       deps,
	}
}

// Namespace provides template functions for the "lang" namespace.
type Namespace struct {
	translator locales.Translator
	deps       *deps.Deps
}

// Translate returns a translated string for id.
func (ns *Namespace) Translate(id interface{}, args ...interface{}) (string, error) {
	var templateData interface{}

	if len(args) > 0 {
		if len(args) > 1 {
			return "", errors.Errorf("wrong number of arguments, expecting at most 2, got %d", len(args)+1)
		}
		templateData = args[0]
	}

	sid, err := cast.ToStringE(id)
	if err != nil {
		return "", nil
	}

	return ns.deps.Translate(sid, templateData), nil
}

// FormatNumber formats number with the given precision for the current language.
func (ns *Namespace) FormatNumber(precision, number interface{}) (string, error) {
	p, n, err := ns.castPrecisionNumber(precision, number)
	if err != nil {
		return "", err
	}
	return ns.translator.FmtNumber(n, p), nil
}

// FormatPercent formats number with the given precision for the current language.
// Note that the number is assumed to be a percentage.
func (ns *Namespace) FormatPercent(precision, number interface{}) (string, error) {
	p, n, err := ns.castPrecisionNumber(precision, number)
	if err != nil {
		return "", err
	}
	return ns.translator.FmtPercent(n, p), nil
}

// FormatCurrency returns the currency reprecentation of number for the given currency and precision
// for the current language.
func (ns *Namespace) FormatCurrency(precision, currency, number interface{}) (string, error) {
	p, n, err := ns.castPrecisionNumber(precision, number)
	if err != nil {
		return "", err
	}
	c := translators.GetCurrency(cast.ToString(currency))
	if c < 0 {
		return "", fmt.Errorf("unknown currency code: %q", currency)
	}
	return ns.translator.FmtCurrency(n, p, c), nil
}

// FormatAccounting returns the currency reprecentation of number for the given currency and precision
// for the current language in accounting notation.
func (ns *Namespace) FormatAccounting(precision, currency, number interface{}) (string, error) {
	p, n, err := ns.castPrecisionNumber(precision, number)
	if err != nil {
		return "", err
	}
	c := translators.GetCurrency(cast.ToString(currency))
	if c < 0 {
		return "", fmt.Errorf("unknown currency code: %q", currency)
	}
	return ns.translator.FmtAccounting(n, p, c), nil
}

func (ns *Namespace) castPrecisionNumber(precision, number interface{}) (uint64, float64, error) {
	p, err := cast.ToUint64E(precision)
	if err != nil {
		return 0, 0, err
	}

	// Sanity check.
	if p > 20 {
		return 0, 0, fmt.Errorf("invalid precision: %d", precision)
	}

	n, err := cast.ToFloat64E(number)
	if err != nil {
		return 0, 0, err
	}
	return p, n, nil
}

// FormatNumberCustom formats a number with the given precision using the
// negative, decimal, and grouping options.  The `options`
// parameter is a string consisting of `<negative> <decimal> <grouping>`.  The
// default `options` value is `- . ,`.
//
// Note that numbers are rounded up at 5 or greater.
// So, with precision set to 0, 1.5 becomes `2`, and 1.4 becomes `1`.
//
// For a simpler function that adapts to the current language, see FormatNumberCustom.
func (ns *Namespace) FormatNumberCustom(precision, number interface{}, options ...interface{}) (string, error) {
	prec, err := cast.ToIntE(precision)
	if err != nil {
		return "", err
	}

	n, err := cast.ToFloat64E(number)
	if err != nil {
		return "", err
	}

	var neg, dec, grp string

	if len(options) == 0 {
		// defaults
		neg, dec, grp = "-", ".", ","
	} else {
		delim := " "

		if len(options) == 2 {
			// custom delimiter
			s, err := cast.ToStringE(options[1])
			if err != nil {
				return "", nil
			}

			delim = s
		}

		s, err := cast.ToStringE(options[0])
		if err != nil {
			return "", nil
		}

		rs := strings.Split(s, delim)
		switch len(rs) {
		case 0:
		case 1:
			neg = rs[0]
		case 2:
			neg, dec = rs[0], rs[1]
		case 3:
			neg, dec, grp = rs[0], rs[1], rs[2]
		default:
			return "", errors.New("too many fields in options parameter to NumFmt")
		}
	}

	exp := math.Pow(10.0, float64(prec))
	r := math.Round(n*exp) / exp

	// Logic from MIT Licensed github.com/gohugoio/locales/
	// Original Copyright (c) 2016 Go Playground

	s := strconv.FormatFloat(math.Abs(r), 'f', prec, 64)
	L := len(s) + 2 + len(s[:len(s)-1-prec])/3

	var count int
	inWhole := prec == 0
	b := make([]byte, 0, L)

	for i := len(s) - 1; i >= 0; i-- {
		if s[i] == '.' {
			for j := len(dec) - 1; j >= 0; j-- {
				b = append(b, dec[j])
			}
			inWhole = true
			continue
		}

		if inWhole {
			if count == 3 {
				for j := len(grp) - 1; j >= 0; j-- {
					b = append(b, grp[j])
				}
				count = 1
			} else {
				count++
			}
		}

		b = append(b, s[i])
	}

	if n < 0 {
		for j := len(neg) - 1; j >= 0; j-- {
			b = append(b, neg[j])
		}
	}

	// reverse
	for i, j := 0, len(b)-1; i < j; i, j = i+1, j-1 {
		b[i], b[j] = b[j], b[i]
	}

	return string(b), nil
}

// NumFmt is deprecated, use FormatNumberCustom.
// We renamed this in Hugo 0.87.
// Deprecated: Use FormatNumberCustom
func (ns *Namespace) NumFmt(precision, number interface{}, options ...interface{}) (string, error) {
	return ns.FormatNumberCustom(precision, number, options...)
}

type pagesLanguageMerger interface {
	MergeByLanguageInterface(other interface{}) (interface{}, error)
}

// Merge creates a union of pages from two languages.
func (ns *Namespace) Merge(p2, p1 interface{}) (interface{}, error) {
	merger, ok := p1.(pagesLanguageMerger)
	if !ok {
		return nil, fmt.Errorf("language merge not supported for %T", p1)
	}
	return merger.MergeByLanguageInterface(p2)
}
<commit_msg>Clarify "precision" in currency format functions<commit_after>// Copyright 2017 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package lang provides template functions for content internationalization.
package lang

import (
	"fmt"
	"math"
	"strconv"
	"strings"

	translators "github.com/gohugoio/localescompressed"
	"github.com/gohugoio/locales"
	"github.com/pkg/errors"

	"github.com/gohugoio/hugo/deps"
	"github.com/spf13/cast"
)

// New returns a new instance of the lang-namespaced template functions.
func New(deps *deps.Deps, translator locales.Translator) *Namespace {
	return &Namespace{
		translator: translator,
		deps:       deps,
	}
}

// Namespace provides template functions for the "lang" namespace.
type Namespace struct {
	translator locales.Translator
	deps       *deps.Deps
}

// Translate returns a translated string for id.
func (ns *Namespace) Translate(id interface{}, args ...interface{}) (string, error) {
	var templateData interface{}

	if len(args) > 0 {
		if len(args) > 1 {
			return "", errors.Errorf("wrong number of arguments, expecting at most 2, got %d", len(args)+1)
		}
		templateData = args[0]
	}

	sid, err := cast.ToStringE(id)
	if err != nil {
		return "", nil
	}

	return ns.deps.Translate(sid, templateData), nil
}

// FormatNumber formats number with the given precision for the current language.
func (ns *Namespace) FormatNumber(precision, number interface{}) (string, error) {
	p, n, err := ns.castPrecisionNumber(precision, number)
	if err != nil {
		return "", err
	}
	return ns.translator.FmtNumber(n, p), nil
}

// FormatPercent formats number with the given precision for the current language.
// Note that the number is assumed to be a percentage.
func (ns *Namespace) FormatPercent(precision, number interface{}) (string, error) {
	p, n, err := ns.castPrecisionNumber(precision, number)
	if err != nil {
		return "", err
	}
	return ns.translator.FmtPercent(n, p), nil
}

// FormatCurrency returns the currency representation of number for the given currency and precision
// for the current language.
//
// The return value is formatted with at least two decimal places.
func (ns *Namespace) FormatCurrency(precision, currency, number interface{}) (string, error) {
	p, n, err := ns.castPrecisionNumber(precision, number)
	if err != nil {
		return "", err
	}
	c := translators.GetCurrency(cast.ToString(currency))
	if c < 0 {
		return "", fmt.Errorf("unknown currency code: %q", currency)
	}
	return ns.translator.FmtCurrency(n, p, c), nil
}

// FormatAccounting returns the currency representation of number for the given currency and precision
// for the current language in accounting notation.
//
// The return value is formatted with at least two decimal places.
func (ns *Namespace) FormatAccounting(precision, currency, number interface{}) (string, error) {
	p, n, err := ns.castPrecisionNumber(precision, number)
	if err != nil {
		return "", err
	}
	c := translators.GetCurrency(cast.ToString(currency))
	if c < 0 {
		return "", fmt.Errorf("unknown currency code: %q", currency)
	}
	return ns.translator.FmtAccounting(n, p, c), nil
}

func (ns *Namespace) castPrecisionNumber(precision, number interface{}) (uint64, float64, error) {
	p, err := cast.ToUint64E(precision)
	if err != nil {
		return 0, 0, err
	}

	// Sanity check.
	if p > 20 {
		return 0, 0, fmt.Errorf("invalid precision: %d", precision)
	}

	n, err := cast.ToFloat64E(number)
	if err != nil {
		return 0, 0, err
	}
	return p, n, nil
}

// FormatNumberCustom formats a number with the given precision using the
// negative, decimal, and grouping options.  The `options`
// parameter is a string consisting of `<negative> <decimal> <grouping>`.  The
// default `options` value is `- . ,`.
//
// Note that numbers are rounded up at 5 or greater.
// So, with precision set to 0, 1.5 becomes `2`, and 1.4 becomes `1`.
//
// For a simpler function that adapts to the current language, see FormatNumberCustom.
func (ns *Namespace) FormatNumberCustom(precision, number interface{}, options ...interface{}) (string, error) {
	prec, err := cast.ToIntE(precision)
	if err != nil {
		return "", err
	}

	n, err := cast.ToFloat64E(number)
	if err != nil {
		return "", err
	}

	var neg, dec, grp string

	if len(options) == 0 {
		// defaults
		neg, dec, grp = "-", ".", ","
	} else {
		delim := " "

		if len(options) == 2 {
			// custom delimiter
			s, err := cast.ToStringE(options[1])
			if err != nil {
				return "", nil
			}

			delim = s
		}

		s, err := cast.ToStringE(options[0])
		if err != nil {
			return "", nil
		}

		rs := strings.Split(s, delim)
		switch len(rs) {
		case 0:
		case 1:
			neg = rs[0]
		case 2:
			neg, dec = rs[0], rs[1]
		case 3:
			neg, dec, grp = rs[0], rs[1], rs[2]
		default:
			return "", errors.New("too many fields in options parameter to NumFmt")
		}
	}

	exp := math.Pow(10.0, float64(prec))
	r := math.Round(n*exp) / exp

	// Logic from MIT Licensed github.com/gohugoio/locales/
	// Original Copyright (c) 2016 Go Playground

	s := strconv.FormatFloat(math.Abs(r), 'f', prec, 64)
	L := len(s) + 2 + len(s[:len(s)-1-prec])/3

	var count int
	inWhole := prec == 0
	b := make([]byte, 0, L)

	for i := len(s) - 1; i >= 0; i-- {
		if s[i] == '.' {
			for j := len(dec) - 1; j >= 0; j-- {
				b = append(b, dec[j])
			}
			inWhole = true
			continue
		}

		if inWhole {
			if count == 3 {
				for j := len(grp) - 1; j >= 0; j-- {
					b = append(b, grp[j])
				}
				count = 1
			} else {
				count++
			}
		}

		b = append(b, s[i])
	}

	if n < 0 {
		for j := len(neg) - 1; j >= 0; j-- {
			b = append(b, neg[j])
		}
	}

	// reverse
	for i, j := 0, len(b)-1; i < j; i, j = i+1, j-1 {
		b[i], b[j] = b[j], b[i]
	}

	return string(b), nil
}

// NumFmt is deprecated, use FormatNumberCustom.
// We renamed this in Hugo 0.87.
// Deprecated: Use FormatNumberCustom
func (ns *Namespace) NumFmt(precision, number interface{}, options ...interface{}) (string, error) {
	return ns.FormatNumberCustom(precision, number, options...)
}

type pagesLanguageMerger interface {
	MergeByLanguageInterface(other interface{}) (interface{}, error)
}

// Merge creates a union of pages from two languages.
func (ns *Namespace) Merge(p2, p1 interface{}) (interface{}, error) {
	merger, ok := p1.(pagesLanguageMerger)
	if !ok {
		return nil, fmt.Errorf("language merge not supported for %T", p1)
	}
	return merger.MergeByLanguageInterface(p2)
}
<|endoftext|>