<commit_before>package main

import (
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"testing"
)

func TestSavingAssignment(t *testing.T) {
	tmpDir, err := ioutil.TempDir("", "")
	assert.NoError(t, err)

	assignment := Assignment{
		Track: "ruby",
		Slug:  "bob",
		Files: map[string]string{
			"bob_test.rb":      "Tests text",
			"README.md":        "Readme text",
			"/path/to/file.rb": "File text",
		},
	}

	err = SaveAssignment(tmpDir, assignment)
	assert.NoError(t, err)

	readme, err := ioutil.ReadFile(tmpDir + "/ruby/bob/README.md")
	assert.NoError(t, err)
	assert.Equal(t, string(readme), "Readme text")

	tests, err := ioutil.ReadFile(tmpDir + "/ruby/bob/bob_test.rb")
	assert.NoError(t, err)
	assert.Equal(t, string(tests), "Tests text")

	fileInDir, err := ioutil.ReadFile(tmpDir + "/ruby/bob/path/to/file.rb")
	assert.NoError(t, err)
	assert.Equal(t, string(fileInDir), "File text")
}
<commit_msg>Use relative path in assignment test<commit_after>package main

import (
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"testing"
)

func TestSavingAssignment(t *testing.T) {
	tmpDir, err := ioutil.TempDir("", "")
	assert.NoError(t, err)

	assignment := Assignment{
		Track: "ruby",
		Slug:  "bob",
		Files: map[string]string{
			"bob_test.rb":     "Tests text",
			"README.md":       "Readme text",
			"path/to/file.rb": "File text",
		},
	}

	err = SaveAssignment(tmpDir, assignment)
	assert.NoError(t, err)

	readme, err := ioutil.ReadFile(tmpDir + "/ruby/bob/README.md")
	assert.NoError(t, err)
	assert.Equal(t, string(readme), "Readme text")

	tests, err := ioutil.ReadFile(tmpDir + "/ruby/bob/bob_test.rb")
	assert.NoError(t, err)
	assert.Equal(t, string(tests), "Tests text")

	fileInDir, err := ioutil.ReadFile(tmpDir + "/ruby/bob/path/to/file.rb")
	assert.NoError(t, err)
	assert.Equal(t, string(fileInDir), "File text")
}
<|endoftext|>


<commit_before>package collectors

import (
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"bosun.org/cmd/scollector/conf"
	"bosun.org/metadata"
	"bosun.org/opentsdb"
)

func AddProcessConfig(params conf.ProcessParams) error {
	p, err := NewWatchedProc(params)
	if err != nil {
		return err
	}
	watchedProcs = append(watchedProcs, p)
	return nil
}

var watchedProcs = []*WatchedProc{}

func WatchProcesses() {
	if len(watchedProcs) == 0 {
		return
	}
	collectors = append(collectors, &IntervalCollector{
		F: func() (opentsdb.MultiDataPoint, error) {
			return c_linux_processes(watchedProcs)
		},
		name: "c_linux_processes",
	})
}

func linuxProcMonitor(w *WatchedProc, md *opentsdb.MultiDataPoint) error {
	var err error
	var processCount int
	for pid, id := range w.Processes {
		file_status, e := os.Stat("/proc/" + pid)
		if e != nil {
			w.Remove(pid)
			continue
		}
		processCount++
		stats_file, e := ioutil.ReadFile("/proc/" + pid + "/stat")
		if e != nil {
			w.Remove(pid)
			continue
		}
		io_file, e := ioutil.ReadFile("/proc/" + pid + "/io")
		if e != nil {
			w.Remove(pid)
			continue
		}
		limits, e := ioutil.ReadFile("/proc/" + pid + "/limits")
		if e != nil {
			w.Remove(pid)
			continue
		}
		fd_dir, e := os.Open("/proc/" + pid + "/fd")
		if e != nil {
			w.Remove(pid)
			continue
		}
		fds, e := fd_dir.Readdirnames(0)
		fd_dir.Close()
		if e != nil {
			w.Remove(pid)
			continue
		}
		stats := strings.Fields(string(stats_file))
		if len(stats) < 24 {
			err = fmt.Errorf("stats too short")
			continue
		}
		var io []string
		for _, line := range strings.Split(string(io_file), "\n") {
			f := strings.Fields(line)
			if len(f) == 2 {
				io = append(io, f[1])
			}
		}
		if len(io) < 6 {
			err = fmt.Errorf("io too short")
			continue
		}
		tags := opentsdb.TagSet{"name": w.Name, "id": strconv.Itoa(id)}
		for _, line := range strings.Split(string(limits), "\n") {
			f := strings.Fields(line)
			if len(f) == 6 && strings.Join(f[0:3], " ") == "Max open files" {
				if f[3] != "unlimited" {
					Add(md, "linux.proc.num_fds_slim", f[3], tags, metadata.Gauge, metadata.Files, descLinuxSoftFileLimit)
					Add(md, "linux.proc.num_fds_hlim", f[4], tags, metadata.Gauge, metadata.Files, descLinuxHardFileLimit)
				}
			}
		}
		start_ts := file_status.ModTime().Unix()
		Add(md, "linux.proc.cpu", stats[13], opentsdb.TagSet{"type": "user"}.Merge(tags), metadata.Counter, metadata.Pct, descLinuxProcCPUUser)
		Add(md, "linux.proc.cpu", stats[14], opentsdb.TagSet{"type": "system"}.Merge(tags), metadata.Counter, metadata.Pct, descLinuxProcCPUSystem)
		Add(md, "linux.proc.mem.fault", stats[9], opentsdb.TagSet{"type": "minflt"}.Merge(tags), metadata.Counter, metadata.Fault, descLinuxProcMemFaultMin)
		Add(md, "linux.proc.mem.fault", stats[11], opentsdb.TagSet{"type": "majflt"}.Merge(tags), metadata.Counter, metadata.Fault, descLinuxProcMemFaultMax)
		Add(md, "linux.proc.mem.virtual", stats[22], tags, metadata.Gauge, metadata.Bytes, descLinuxProcMemVirtual)
		Add(md, "linux.proc.mem.rss", stats[23], tags, metadata.Gauge, metadata.Page, descLinuxProcMemRss)
		Add(md, "linux.proc.char_io", io[0], opentsdb.TagSet{"type": "read"}.Merge(tags), metadata.Counter, metadata.Bytes, descLinuxProcCharIoRead)
		Add(md, "linux.proc.char_io", io[1], opentsdb.TagSet{"type": "write"}.Merge(tags), metadata.Counter, metadata.Bytes, descLinuxProcCharIoWrite)
		Add(md, "linux.proc.syscall", io[2], opentsdb.TagSet{"type": "read"}.Merge(tags), metadata.Counter, metadata.Syscall, descLinuxProcSyscallRead)
		Add(md, "linux.proc.syscall", io[3], opentsdb.TagSet{"type": "write"}.Merge(tags), metadata.Counter, metadata.Syscall, descLinuxProcSyscallWrite)
		Add(md, "linux.proc.io_bytes", io[4], opentsdb.TagSet{"type": "read"}.Merge(tags), metadata.Counter, metadata.Bytes, descLinuxProcIoBytesRead)
		Add(md, "linux.proc.io_bytes", io[5], opentsdb.TagSet{"type": "write"}.Merge(tags), metadata.Counter, metadata.Bytes, descLinuxProcIoBytesWrite)
		Add(md, "linux.proc.num_fds", len(fds), tags, metadata.Gauge, metadata.Files, descLinuxProcFd)
		Add(md, "linux.proc.start_time", start_ts, tags, metadata.Gauge, metadata.Timestamp, descLinuxProcStartTS)
		Add(md, "linux.proc.uptime", now()-start_ts, tags, metadata.Gauge, metadata.Second, descLinuxProcUptime)
	}
	if w.IncludeCount {
		Add(md, "linux.proc.count", processCount, opentsdb.TagSet{"name": w.Name}, metadata.Gauge, metadata.Process, descLinuxProcCount)
	}
	return err
}

const (
	descLinuxProcCPUUser      = "The amount of time that this process has been scheduled in user mode."
	descLinuxProcCPUSystem    = "The amount of time that this process has been scheduled in kernel mode"
	descLinuxProcMemFaultMin  = "The number of minor faults the process has made which have not required loading a memory page from disk."
	descLinuxProcMemFaultMax  = "The number of major faults the process has made which have required loading a memory page from disk."
	descLinuxProcMemVirtual   = "The virtual memory size."
	descLinuxProcMemRss       = "The resident set size (number of pages the process has in real memory."
	descLinuxProcCharIoRead   = "The number of bytes which this task has caused to be read from storage. This is simply the sum of bytes which this process passed to read(2) and similar system calls. It includes things such as terminal I/O and is unaffected by whether or not actual physical disk I/O was required (the read might have been satisfied from pagecache)"
	descLinuxProcCharIoWrite  = "The number of bytes which this task has caused, or shall cause to be written to disk. Similar caveats apply here as with read."
	descLinuxProcSyscallRead  = "An attempt to count the number of read I/O operations—that is, system calls such as read(2) and pread(2)."
	descLinuxProcSyscallWrite = "Attempt to count the number of write I/O operations—that is, system calls such as write(2) and pwrite(2)."
	descLinuxProcIoBytesRead  = "An attempt to count the number of bytes which this process really did cause to be fetched from the storage layer. This is accurate for block-backed filesystems."
	descLinuxProcIoBytesWrite = "An Attempt to count the number of bytes which this process caused to be sent to the storage layer."
	descLinuxProcFd           = "The number of open file descriptors."
	descLinuxSoftFileLimit    = "The soft limit on the number of open file descriptors."
	descLinuxHardFileLimit    = "The hard limit on the number of open file descriptors."
	descLinuxProcUptime       = "The length of time, in seconds, since the process was started."
	descLinuxProcStartTS      = "The timestamp of process start."
	descLinuxProcCount        = "The number of currently running processes."
)

type byModTime []os.FileInfo

func (bmt byModTime) Len() int      { return len(bmt) }
func (bmt byModTime) Swap(i, j int) { bmt[i], bmt[j] = bmt[j], bmt[i] }
func (bmt byModTime) Less(i, j int) bool {
	return bmt[i].ModTime().Unix() < bmt[j].ModTime().Unix()
}

func getLinuxProccesses() ([]*Process, error) {
	files, err := ioutil.ReadDir("/proc")
	if err != nil {
		return nil, err
	}
	sort.Sort(byModTime(files))
	var pids []string
	for _, f := range files {
		if _, err := strconv.Atoi(f.Name()); err == nil && f.IsDir() {
			pids = append(pids, f.Name())
		}
	}
	var lps []*Process
	for _, pid := range pids {
		cl, err := getLinuxCmdline(pid)
		if err != nil || cl == nil {
			//Continue because the pid might not exist any more
			continue
		}
		lp := &Process{
			Pid:     pid,
			Command: cl[0],
		}
		if len(cl) > 1 {
			lp.Arguments = strings.Join(cl[1:], "")
		}
		lps = append(lps, lp)
	}
	return lps, nil
}

func getLinuxCmdline(pid string) ([]string, error) {
	cmdline, err := ioutil.ReadFile("/proc/" + pid + "/cmdline")
	if err != nil {
		return nil, err
	}
	cl := strings.Split(string(cmdline), "\x00")
	if len(cl) < 1 || len(cl[0]) == 0 {
		return nil, nil
	}
	return cl, nil
}

func c_linux_processes(procs []*WatchedProc) (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	lps, err := getLinuxProccesses()
	if err != nil {
		return nil, nil
	}
	for _, w := range procs {
		w.Check(lps)
		if e := linuxProcMonitor(w, &md); e != nil {
			err = e
		}
	}
	return md, err
}

type Process struct {
	Pid       string
	Command   string
	Arguments string
}

// NewWatchedProc takes a configuration block [[Process]] from conf
func NewWatchedProc(params conf.ProcessParams) (*WatchedProc, error) {
	if params.Name == "" {
		params.Name = params.Command
	}
	if !opentsdb.ValidTag(params.Name) {
		return nil, fmt.Errorf("bad process name: %v", params.Name)
	}
	return &WatchedProc{
		Command:      params.Command,
		Name:         params.Name,
		IncludeCount: params.IncludeCount,
		Processes:    make(map[string]int),
		ArgMatch:     regexp.MustCompile(params.Args),
		idPool:       new(idPool),
	}, nil
}

type WatchedProc struct {
	Command      string
	Name         string
	IncludeCount bool
	Processes    map[string]int
	ArgMatch     *regexp.Regexp
	*idPool
}

// Check finds all matching processes and assigns them a new unique id.
func (w *WatchedProc) Check(procs []*Process) {
	for _, l := range procs {
		if _, ok := w.Processes[l.Pid]; ok {
			continue
		}
		if !strings.Contains(l.Command, w.Command) {
			continue
		}
		if !w.ArgMatch.MatchString(l.Arguments) {
			continue
		}
		w.Processes[l.Pid] = w.get()
	}
}

func (w *WatchedProc) Remove(pid string) {
	w.put(w.Processes[pid])
	delete(w.Processes, pid)
}

type idPool struct {
	free []int
	next int
}

func (i *idPool) get() int {
	if len(i.free) == 0 {
		i.next++
		return i.next
	}
	sort.Ints(i.free)
	return i.free[0]
}

func (i *idPool) put(v int) {
	i.free = append(i.free, v)
}
<commit_msg>processes_linux: Add a tiebreaker if the process creation time is identical.<commit_after>package collectors

import (
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"bosun.org/cmd/scollector/conf"
	"bosun.org/metadata"
	"bosun.org/opentsdb"
)

func AddProcessConfig(params conf.ProcessParams) error {
	p, err := NewWatchedProc(params)
	if err != nil {
		return err
	}
	watchedProcs = append(watchedProcs, p)
	return nil
}

var watchedProcs = []*WatchedProc{}

func WatchProcesses() {
	if len(watchedProcs) == 0 {
		return
	}
	collectors = append(collectors, &IntervalCollector{
		F: func() (opentsdb.MultiDataPoint, error) {
			return c_linux_processes(watchedProcs)
		},
		name: "c_linux_processes",
	})
}

func linuxProcMonitor(w *WatchedProc, md *opentsdb.MultiDataPoint) error {
	var err error
	var processCount int
	for pid, id := range w.Processes {
		file_status, e := os.Stat("/proc/" + pid)
		if e != nil {
			w.Remove(pid)
			continue
		}
		processCount++
		stats_file, e := ioutil.ReadFile("/proc/" + pid + "/stat")
		if e != nil {
			w.Remove(pid)
			continue
		}
		io_file, e := ioutil.ReadFile("/proc/" + pid + "/io")
		if e != nil {
			w.Remove(pid)
			continue
		}
		limits, e := ioutil.ReadFile("/proc/" + pid + "/limits")
		if e != nil {
			w.Remove(pid)
			continue
		}
		fd_dir, e := os.Open("/proc/" + pid + "/fd")
		if e != nil {
			w.Remove(pid)
			continue
		}
		fds, e := fd_dir.Readdirnames(0)
		fd_dir.Close()
		if e != nil {
			w.Remove(pid)
			continue
		}
		stats := strings.Fields(string(stats_file))
		if len(stats) < 24 {
			err = fmt.Errorf("stats too short")
			continue
		}
		var io []string
		for _, line := range strings.Split(string(io_file), "\n") {
			f := strings.Fields(line)
			if len(f) == 2 {
				io = append(io, f[1])
			}
		}
		if len(io) < 6 {
			err = fmt.Errorf("io too short")
			continue
		}
		tags := opentsdb.TagSet{"name": w.Name, "id": strconv.Itoa(id)}
		for _, line := range strings.Split(string(limits), "\n") {
			f := strings.Fields(line)
			if len(f) == 6 && strings.Join(f[0:3], " ") == "Max open files" {
				if f[3] != "unlimited" {
					Add(md, "linux.proc.num_fds_slim", f[3], tags, metadata.Gauge, metadata.Files, descLinuxSoftFileLimit)
					Add(md, "linux.proc.num_fds_hlim", f[4], tags, metadata.Gauge, metadata.Files, descLinuxHardFileLimit)
				}
			}
		}
		start_ts := file_status.ModTime().Unix()
		Add(md, "linux.proc.cpu", stats[13], opentsdb.TagSet{"type": "user"}.Merge(tags), metadata.Counter, metadata.Pct, descLinuxProcCPUUser)
		Add(md, "linux.proc.cpu", stats[14], opentsdb.TagSet{"type": "system"}.Merge(tags), metadata.Counter, metadata.Pct, descLinuxProcCPUSystem)
		Add(md, "linux.proc.mem.fault", stats[9], opentsdb.TagSet{"type": "minflt"}.Merge(tags), metadata.Counter, metadata.Fault, descLinuxProcMemFaultMin)
		Add(md, "linux.proc.mem.fault", stats[11], opentsdb.TagSet{"type": "majflt"}.Merge(tags), metadata.Counter, metadata.Fault, descLinuxProcMemFaultMax)
		Add(md, "linux.proc.mem.virtual", stats[22], tags, metadata.Gauge, metadata.Bytes, descLinuxProcMemVirtual)
		Add(md, "linux.proc.mem.rss", stats[23], tags, metadata.Gauge, metadata.Page, descLinuxProcMemRss)
		Add(md, "linux.proc.char_io", io[0], opentsdb.TagSet{"type": "read"}.Merge(tags), metadata.Counter, metadata.Bytes, descLinuxProcCharIoRead)
		Add(md, "linux.proc.char_io", io[1], opentsdb.TagSet{"type": "write"}.Merge(tags), metadata.Counter, metadata.Bytes, descLinuxProcCharIoWrite)
		Add(md, "linux.proc.syscall", io[2], opentsdb.TagSet{"type": "read"}.Merge(tags), metadata.Counter, metadata.Syscall, descLinuxProcSyscallRead)
		Add(md, "linux.proc.syscall", io[3], opentsdb.TagSet{"type": "write"}.Merge(tags), metadata.Counter, metadata.Syscall, descLinuxProcSyscallWrite)
		Add(md, "linux.proc.io_bytes", io[4], opentsdb.TagSet{"type": "read"}.Merge(tags), metadata.Counter, metadata.Bytes, descLinuxProcIoBytesRead)
		Add(md, "linux.proc.io_bytes", io[5], opentsdb.TagSet{"type": "write"}.Merge(tags), metadata.Counter, metadata.Bytes, descLinuxProcIoBytesWrite)
		Add(md, "linux.proc.num_fds", len(fds), tags, metadata.Gauge, metadata.Files, descLinuxProcFd)
		Add(md, "linux.proc.start_time", start_ts, tags, metadata.Gauge, metadata.Timestamp, descLinuxProcStartTS)
		Add(md, "linux.proc.uptime", now()-start_ts, tags, metadata.Gauge, metadata.Second, descLinuxProcUptime)
	}
	if w.IncludeCount {
		Add(md, "linux.proc.count", processCount, opentsdb.TagSet{"name": w.Name}, metadata.Gauge, metadata.Process, descLinuxProcCount)
	}
	return err
}

const (
	descLinuxProcCPUUser      = "The amount of time that this process has been scheduled in user mode."
	descLinuxProcCPUSystem    = "The amount of time that this process has been scheduled in kernel mode"
	descLinuxProcMemFaultMin  = "The number of minor faults the process has made which have not required loading a memory page from disk."
	descLinuxProcMemFaultMax  = "The number of major faults the process has made which have required loading a memory page from disk."
	descLinuxProcMemVirtual   = "The virtual memory size."
	descLinuxProcMemRss       = "The resident set size (number of pages the process has in real memory."
	descLinuxProcCharIoRead   = "The number of bytes which this task has caused to be read from storage. This is simply the sum of bytes which this process passed to read(2) and similar system calls. It includes things such as terminal I/O and is unaffected by whether or not actual physical disk I/O was required (the read might have been satisfied from pagecache)"
	descLinuxProcCharIoWrite  = "The number of bytes which this task has caused, or shall cause to be written to disk. Similar caveats apply here as with read."
	descLinuxProcSyscallRead  = "An attempt to count the number of read I/O operations—that is, system calls such as read(2) and pread(2)."
	descLinuxProcSyscallWrite = "Attempt to count the number of write I/O operations—that is, system calls such as write(2) and pwrite(2)."
	descLinuxProcIoBytesRead  = "An attempt to count the number of bytes which this process really did cause to be fetched from the storage layer. This is accurate for block-backed filesystems."
	descLinuxProcIoBytesWrite = "An Attempt to count the number of bytes which this process caused to be sent to the storage layer."
	descLinuxProcFd           = "The number of open file descriptors."
	descLinuxSoftFileLimit    = "The soft limit on the number of open file descriptors."
	descLinuxHardFileLimit    = "The hard limit on the number of open file descriptors."
	descLinuxProcUptime       = "The length of time, in seconds, since the process was started."
	descLinuxProcStartTS      = "The timestamp of process start."
	descLinuxProcCount        = "The number of currently running processes."
)

type byModTime []os.FileInfo

func (bmt byModTime) Len() int      { return len(bmt) }
func (bmt byModTime) Swap(i, j int) { bmt[i], bmt[j] = bmt[j], bmt[i] }
func (bmt byModTime) Less(i, j int) bool {
	// If the creation times are identical, sort by filename (pid) instead.
	if bmt[i].ModTime() == bmt[j].ModTime() {
		return sort.StringsAreSorted([]string{bmt[i].Name(), bmt[j].Name()})
	}
	return bmt[i].ModTime().UnixNano() < bmt[j].ModTime().UnixNano()
}

func getLinuxProccesses() ([]*Process, error) {
	files, err := ioutil.ReadDir("/proc")
	if err != nil {
		return nil, err
	}
	sort.Sort(byModTime(files))
	var pids []string
	for _, f := range files {
		if _, err := strconv.Atoi(f.Name()); err == nil && f.IsDir() {
			pids = append(pids, f.Name())
		}
	}
	var lps []*Process
	for _, pid := range pids {
		cl, err := getLinuxCmdline(pid)
		if err != nil || cl == nil {
			//Continue because the pid might not exist any more
			continue
		}
		lp := &Process{
			Pid:     pid,
			Command: cl[0],
		}
		if len(cl) > 1 {
			lp.Arguments = strings.Join(cl[1:], "")
		}
		lps = append(lps, lp)
	}
	return lps, nil
}

func getLinuxCmdline(pid string) ([]string, error) {
	cmdline, err := ioutil.ReadFile("/proc/" + pid + "/cmdline")
	if err != nil {
		return nil, err
	}
	cl := strings.Split(string(cmdline), "\x00")
	if len(cl) < 1 || len(cl[0]) == 0 {
		return nil, nil
	}
	return cl, nil
}

func c_linux_processes(procs []*WatchedProc) (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	lps, err := getLinuxProccesses()
	if err != nil {
		return nil, nil
	}
	for _, w := range procs {
		w.Check(lps)
		if e := linuxProcMonitor(w, &md); e != nil {
			err = e
		}
	}
	return md, err
}

type Process struct {
	Pid       string
	Command   string
	Arguments string
}

// NewWatchedProc takes a configuration block [[Process]] from conf
func NewWatchedProc(params conf.ProcessParams) (*WatchedProc, error) {
	if params.Name == "" {
		params.Name = params.Command
	}
	if !opentsdb.ValidTag(params.Name) {
		return nil, fmt.Errorf("bad process name: %v", params.Name)
	}
	return &WatchedProc{
		Command:      params.Command,
		Name:         params.Name,
		IncludeCount: params.IncludeCount,
		Processes:    make(map[string]int),
		ArgMatch:     regexp.MustCompile(params.Args),
		idPool:       new(idPool),
	}, nil
}

type WatchedProc struct {
	Command      string
	Name         string
	IncludeCount bool
	Processes    map[string]int
	ArgMatch     *regexp.Regexp
	*idPool
}

// Check finds all matching processes and assigns them a new unique id.
func (w *WatchedProc) Check(procs []*Process) {
	for _, l := range procs {
		if _, ok := w.Processes[l.Pid]; ok {
			continue
		}
		if !strings.Contains(l.Command, w.Command) {
			continue
		}
		if !w.ArgMatch.MatchString(l.Arguments) {
			continue
		}
		w.Processes[l.Pid] = w.get()
	}
}

func (w *WatchedProc) Remove(pid string) {
	w.put(w.Processes[pid])
	delete(w.Processes, pid)
}

type idPool struct {
	free []int
	next int
}

func (i *idPool) get() int {
	if len(i.free) == 0 {
		i.next++
		return i.next
	}
	sort.Ints(i.free)
	return i.free[0]
}

func (i *idPool) put(v int) {
	i.free = append(i.free, v)
}
<|endoftext|>


<commit_before>package checkerlution

import (
	"fmt"
	"github.com/couchbaselabs/logg"
	cbot "github.com/tleyden/checkers-bot"
	ng "github.com/tleyden/neurgo"
)

type CheckerlutionScape struct {
	team                  cbot.TeamType
	syncGatewayUrl        string
	feedType              cbot.FeedType
	randomDelayBeforeMove int
	fitnessHistory        map[string]float64
}

func (scape *CheckerlutionScape) FitnessAgainst(cortex *ng.Cortex, opponentCortex *ng.Cortex) (fitness float64) {

	if cortex == opponentCortex {
		logg.LogPanic("Cannot calculate fitnesss between cortex %p and itself %p", cortex, opponentCortex)
	}

	savedFitness, isPresent := scape.lookupFitnessHistory(cortex, opponentCortex)
	if isPresent {
		fitness = savedFitness
		logg.LogTo("CHECKERLUTION_SCAPE", "Fitness from history (us %v): %v", cortex.NodeId.UUID, fitness)
		return
	}

	cortex.Init()
	opponentCortex.Init()

	// setup checkers game for cortex
	thinker := &Checkerlution{}
	thinker.SetMode(TRAINING_MODE)
	thinker.StartWithCortex(cortex, scape.team)
	game := cbot.NewGame(scape.team, thinker)
	game.SetServerUrl(scape.syncGatewayUrl)
	game.SetFeedType(scape.feedType)
	game.SetDelayBeforeMove(scape.randomDelayBeforeMove)

	// setup checkers game for opponent
	thinkerOpponent := &Checkerlution{}
	thinkerOpponent.SetMode(TRAINING_MODE)
	var opponentTeam cbot.TeamType // TODO: why can't just use := syntax here?
	opponentTeam = cbot.RED_TEAM
	if scape.team == cbot.RED_TEAM {
		opponentTeam = cbot.BLUE_TEAM
	}
	thinkerOpponent.StartWithCortex(opponentCortex, opponentTeam)
	gameOpponent := cbot.NewGame(opponentTeam, thinkerOpponent)
	gameOpponent.SetServerUrl(scape.syncGatewayUrl)
	gameOpponent.SetFeedType(scape.feedType)
	gameOpponent.SetDelayBeforeMove(scape.randomDelayBeforeMove)

	// run both game loops and wait for both to finish
	logg.LogTo("CHECKERLUTION_SCAPE", "Started game: %v vs %v", cortex.NodeId.UUID, opponentCortex.NodeId.UUID)
	games := []*cbot.Game{game, gameOpponent}
	scape.runGameLoops(games)
	logg.LogTo("CHECKERLUTION_SCAPE", "Game finished after %v turns", game.Turn())

	fitness = thinker.latestFitnessScore
	fitnessOpponent := thinkerOpponent.latestFitnessScore

	logg.LogTo("CHECKERLUTION_SCAPE", "Fitness (us %v): %v", cortex.NodeId.UUID, fitness)
	logg.LogTo("CHECKERLUTION_SCAPE", "Fitness (opponent %v): %v", opponentCortex.NodeId.UUID, fitnessOpponent)

	scape.recordFitness(cortex, fitness, opponentCortex, fitnessOpponent)

	// wait until the game number increments, otherwise on the
	// next callback to this method, we'll jump into a game which
	// is already over.
	logg.LogTo("CHECKERLUTION_SCAPE", "Wait For Next Game ..")
	game.WaitForNextGame()
	gameOpponent.WaitForNextGame()
	logg.LogTo("CHECKERLUTION_SCAPE", "Done waiting For Next Game ..")

	cortex.Shutdown()
	opponentCortex.Shutdown()

	return
}

func (scape *CheckerlutionScape) Fitness(cortex *ng.Cortex) (fitness float64) {

	logg.LogTo("CHECKERLUTION_SCAPE", "CheckerlutionScape Fitness() called, create random opponent")

	// create an opponent
	thinker := &Checkerlution{}
	thinker.CreateNeurgoCortex()
	thinker.Cortex()

	return scape.FitnessAgainst(cortex, thinker.Cortex())

}

func (scape *CheckerlutionScape) runGameLoops(games []*cbot.Game) {

	resultChannel := make(chan bool)
	runGameLoop := func(game *cbot.Game, result chan bool) {
		game.GameLoop()
		result <- true
	}
	for _, game := range games {
		go runGameLoop(game, resultChannel)
	}
	for _, _ = range games {
		<-resultChannel
	}

}

func (scape *CheckerlutionScape) recordFitness(cortex *ng.Cortex, fitness float64, opponentCortex *ng.Cortex, fitnessOpponent float64) {

	if scape.fitnessHistory == nil {
		scape.fitnessHistory = make(map[string]float64)
	}

	// record the fitness score of us vs them
	key := fmt.Sprintf("%v-%v", cortex.NodeId.UUID, opponentCortex.NodeId.UUID)
	scape.fitnessHistory[key] = fitness

}

func (scape *CheckerlutionScape) lookupFitnessHistory(cortex *ng.Cortex, opponentCortex *ng.Cortex) (fitness float64, isPresent bool) {

	key := fmt.Sprintf("%v-%v", cortex.NodeId.UUID, opponentCortex.NodeId.UUID)
	fitness, isPresent = scape.fitnessHistory[key]
	return

}

func (scape *CheckerlutionScape) SetSyncGatewayUrl(syncGatewayUrl string) {
	scape.syncGatewayUrl = syncGatewayUrl
}

func (scape *CheckerlutionScape) SetTeam(team cbot.TeamType) {
	scape.team = team
}

func (scape *CheckerlutionScape) SetFeedType(feedType cbot.FeedType) {
	scape.feedType = feedType
}

func (scape *CheckerlutionScape) SetRandomDelayBeforeMove(delay int) {
	scape.randomDelayBeforeMove = delay
}
<commit_msg>add debug<commit_after>package checkerlution

import (
	"fmt"
	"github.com/couchbaselabs/logg"
	cbot "github.com/tleyden/checkers-bot"
	ng "github.com/tleyden/neurgo"
)

type CheckerlutionScape struct {
	team                  cbot.TeamType
	syncGatewayUrl        string
	feedType              cbot.FeedType
	randomDelayBeforeMove int
	fitnessHistory        map[string]float64
}

func (scape *CheckerlutionScape) FitnessAgainst(cortex *ng.Cortex, opponentCortex *ng.Cortex) (fitness float64) {

	logg.LogTo("CHECKERLUTION_SCAPE", "FitnessAgainst cortex: %v vs opponent: %v", cortex.NodeId.UUID, opponentCortex.NodeId.UUID)

	if cortex == opponentCortex {
		logg.LogPanic("Cannot calculate fitnesss between cortex %p and itself %p", cortex, opponentCortex)
	}

	savedFitness, isPresent := scape.lookupFitnessHistory(cortex, opponentCortex)
	if isPresent {
		fitness = savedFitness
		logg.LogTo("CHECKERLUTION_SCAPE", "Fitness from history (us %v): %v", cortex.NodeId.UUID, fitness)
		return
	}

	cortex.Init()
	opponentCortex.Init()

	// setup checkers game for cortex
	thinker := &Checkerlution{}
	thinker.SetMode(TRAINING_MODE)
	thinker.StartWithCortex(cortex, scape.team)
	game := cbot.NewGame(scape.team, thinker)
	game.SetServerUrl(scape.syncGatewayUrl)
	game.SetFeedType(scape.feedType)
	game.SetDelayBeforeMove(scape.randomDelayBeforeMove)

	// setup checkers game for opponent
	thinkerOpponent := &Checkerlution{}
	thinkerOpponent.SetMode(TRAINING_MODE)
	var opponentTeam cbot.TeamType // TODO: why can't just use := syntax here?
	opponentTeam = cbot.RED_TEAM
	if scape.team == cbot.RED_TEAM {
		opponentTeam = cbot.BLUE_TEAM
	}
	thinkerOpponent.StartWithCortex(opponentCortex, opponentTeam)
	gameOpponent := cbot.NewGame(opponentTeam, thinkerOpponent)
	gameOpponent.SetServerUrl(scape.syncGatewayUrl)
	gameOpponent.SetFeedType(scape.feedType)
	gameOpponent.SetDelayBeforeMove(scape.randomDelayBeforeMove)

	// run both game loops and wait for both to finish
	logg.LogTo("CHECKERLUTION_SCAPE", "Started game: %v vs %v", cortex.NodeId.UUID, opponentCortex.NodeId.UUID)
	games := []*cbot.Game{game, gameOpponent}
	scape.runGameLoops(games)
	logg.LogTo("CHECKERLUTION_SCAPE", "Game finished after %v turns", game.Turn())

	fitness = thinker.latestFitnessScore
	fitnessOpponent := thinkerOpponent.latestFitnessScore

	logg.LogTo("CHECKERLUTION_SCAPE", "Fitness (us %v): %v", cortex.NodeId.UUID, fitness)
	logg.LogTo("CHECKERLUTION_SCAPE", "Fitness (opponent %v): %v", opponentCortex.NodeId.UUID, fitnessOpponent)

	scape.recordFitness(cortex, fitness, opponentCortex, fitnessOpponent)

	// wait until the game number increments, otherwise on the
	// next callback to this method, we'll jump into a game which
	// is already over.
	logg.LogTo("CHECKERLUTION_SCAPE", "Wait For Next Game ..")
	game.WaitForNextGame()
	gameOpponent.WaitForNextGame()
	logg.LogTo("CHECKERLUTION_SCAPE", "Done waiting For Next Game ..")

	cortex.Shutdown()
	opponentCortex.Shutdown()

	return
}

func (scape *CheckerlutionScape) Fitness(cortex *ng.Cortex) (fitness float64) {

	logg.LogTo("CHECKERLUTION_SCAPE", "CheckerlutionScape Fitness() called, create random opponent")

	// create an opponent
	thinker := &Checkerlution{}
	thinker.CreateNeurgoCortex()
	thinker.Cortex()

	return scape.FitnessAgainst(cortex, thinker.Cortex())

}

func (scape *CheckerlutionScape) runGameLoops(games []*cbot.Game) {

	resultChannel := make(chan bool)
	runGameLoop := func(game *cbot.Game, result chan bool) {
		game.GameLoop()
		result <- true
	}
	for _, game := range games {
		go runGameLoop(game, resultChannel)
	}
	for _, _ = range games {
		<-resultChannel
	}

}

func (scape *CheckerlutionScape) recordFitness(cortex *ng.Cortex, fitness float64, opponentCortex *ng.Cortex, fitnessOpponent float64) {

	if scape.fitnessHistory == nil {
		scape.fitnessHistory = make(map[string]float64)
	}

	// record the fitness score of us vs them
	key := fmt.Sprintf("%v-%v", cortex.NodeId.UUID, opponentCortex.NodeId.UUID)
	scape.fitnessHistory[key] = fitness

}

func (scape *CheckerlutionScape) lookupFitnessHistory(cortex *ng.Cortex, opponentCortex *ng.Cortex) (fitness float64, isPresent bool) {

	key := fmt.Sprintf("%v-%v", cortex.NodeId.UUID, opponentCortex.NodeId.UUID)
	fitness, isPresent = scape.fitnessHistory[key]
	return

}

func (scape *CheckerlutionScape) SetSyncGatewayUrl(syncGatewayUrl string) {
	scape.syncGatewayUrl = syncGatewayUrl
}

func (scape *CheckerlutionScape) SetTeam(team cbot.TeamType) {
	scape.team = team
}

func (scape *CheckerlutionScape) SetFeedType(feedType cbot.FeedType) {
	scape.feedType = feedType
}

func (scape *CheckerlutionScape) SetRandomDelayBeforeMove(delay int) {
	scape.randomDelayBeforeMove = delay
}
<|endoftext|>


<commit_before>package blocks

import (
	"strconv"
	"strings"
)

// getKeyValues returns values for all paths, including arrays
// {"foo":"bar"} returns [bar] for string "foo"
// {"foo":["bar","bar","bar"]} returns [bar, bar, bar] for string "foo[]"
// {"foo":[{"type":"bar"},{"type":"baz"}]} returns [bar, baz] for string "foo[].type"
// {"foo":["bar","baz"]} returns [bar] for string "foo[0]"

// this function is obscene :(
func getKeyValues(d interface{}, p string) []interface{} {
	var values []interface{}
	var key string
	var rest string

	keyIdx := strings.Index(p, ".")
	brkIdx := strings.Index(p, "[")

	if keyIdx != -1 {
		key = p[:keyIdx]
		rest = p[keyIdx+1:]
	} else {
		key = p
	}

	if brkIdx != -1 && brkIdx != 0 {
		key = p[:brkIdx]
		rest = p[brkIdx:]
	}

	bStart := strings.Index(key, "[")
	bEnd := strings.Index(key, "]")
	var id int64
	id = -1
	if bStart == 0 && bEnd != 1 {
		id, _ = strconv.ParseInt(key[bStart+1:bEnd], 10, 64)
	}

	switch d := d.(type) {
	case map[string]interface{}:
		if len(rest) > 0 {
			x := getKeyValues(d[key], rest)
			for _, z := range x {
				values = append(values, z)
			}
		} else {
			_, ok := (d[p]).([]interface{})
			if ok == false {
				values = append(values, d[p])
			}
		}
	case []int:
		var ids []int64
		if id != -1 {
			if len(d) == 0 || int(id) >= len(d) || id < 0 {
				break
			}
			ids = append(ids, int64(id))
		} else {
			for i := range d {
				ids = append(ids, int64(i))
			}
		}

		for _, id := range ids {
			values = append(values, d[id])
		}
	case []string:
		var ids []int64
		if id != -1 {
			if len(d) == 0 || int(id) >= len(d) || id < 0 {
				break
			}

			ids = append(ids, int64(id))
		} else {
			for i := range d {
				ids = append(ids, int64(i))
			}
		}

		for _, id := range ids {
			values = append(values, d[id])
		}
	case []bool:
		var ids []int64
		if id != -1 {
			if len(d) == 0 || int(id) >= len(d) || id < 0 {
				break
			}
			ids = append(ids, int64(id))
		} else {
			for i := range d {
				ids = append(ids, int64(i))
			}
		}

		for _, id := range ids {
			values = append(values, d[id])
		}
	case []float64:
		var ids []int64
		if id != -1 {
			if len(d) == 0 || int(id) >= len(d) || id < 0 {
				break
			}
			ids = append(ids, int64(id))
		} else {
			for i := range d {
				ids = append(ids, int64(i))
			}
		}

		for _, id := range ids {
			values = append(values, d[id])
		}
	case []interface{}:
		var ids []int64
		if id != -1 {
			if len(d) == 0 || int(id) >= len(d) || id < 0 {
				break
			}
			ids = append(ids, int64(id))
		} else {
			for i := range d {
				ids = append(ids, int64(i))
			}
		}

		for _, id := range ids {
			if len(rest) > 0 {
				x := getKeyValues(d[id], rest)
				for _, z := range x {
					values = append(values, z)
				}
			} else {
				values = append(values, d[id])
			}
		}

	default:
	}

	return values
}

func equals(value interface{}, comparator interface{}) bool {
	switch value := value.(type) {
	case float64:
		// comparing floats.....?
		c, ok := comparator.(float64)
		if ok == false {
			return false
		}
		return value == c
	case string:
		return value == comparator
	case bool:
		return value == comparator
	default:
		if value == nil && comparator == nil {
			return true
		}
		return false
	}
}

func greaterthan(value interface{}, comparator interface{}) bool {
	switch value := value.(type) {
	case float64:
		c, ok := comparator.(float64)
		if ok == false {
			return false
		}
		return value > c
	default:
		return false
	}
}

func lessthan(value interface{}, comparator interface{}) bool {
	switch value := value.(type) {
	case float64:
		c, ok := comparator.(float64)
		if ok == false {
			return false
		}
		return value < c
	default:
		return false
	}
}

func subsetof(value interface{}, comparator interface{}) bool {
	switch value := value.(type) {
	case string:
		return strings.Contains(value, comparator.(string))
	}
	return false
}
<commit_msg>adding got support<commit_after>package blocks

import (
	"strconv"
	"strings"
)

// getKeyValues returns values for all paths, including arrays
// {"foo":"bar"} returns [bar] for string "foo"
// {"foo":["bar","bar","bar"]} returns [bar, bar, bar] for string "foo[]"
// {"foo":[{"type":"bar"},{"type":"baz"}]} returns [bar, baz] for string "foo[].type"
// {"foo":["bar","baz"]} returns [bar] for string "foo[0]"
//
// getKeyValues also supports bracket access in case your keys include periods.
// {"key.includes.periods":1} returns [1] for string "['key.includes.periods']"
// {"foo":{"bar.bar":{"baz":1}} returns [1] for string 'foo["bar.bar"].baz'
// {"foo.bar"{"baz":{"bing.bong":{"boo":1}}}} returns [1] for string '["foo.bar"].baz["bing.bong"]["boo"]'

// this function is obscene :(
func getKeyValues(d interface{}, p string) []interface{} {
	var values []interface{}
	var key string
	var rest string
	var id int64
	id = -1

	keyIdx := strings.Index(p, ".")
	brkIdx := strings.Index(p, "[")
	escIdx := strings.Index(p, "[\"")

	if escIdx == -1 {
		escIdx = strings.Index(p, "['")
	}

	if escIdx == 0 {
		endescIdx := strings.Index(p, "\"]")
		if endescIdx == -1 {
			endescIdx = strings.Index(p, "']")
		}

		key = p[escIdx + 2:endescIdx]
		rest = p[endescIdx + 2:]

		if len(rest) > 0 && rest[0] == '.'{
			rest = rest[1:]
		}
	} else {

		if keyIdx != -1 {
			key = p[:keyIdx]
			rest = p[keyIdx+1:]
		} else {
			key = p
		}

		if brkIdx != -1 && brkIdx != 0 {
			key = p[:brkIdx]
			rest = p[brkIdx:]
		}

		bStart := strings.Index(key, "[")
		bEnd := strings.Index(key, "]")

		if bStart == 0 && bEnd != 1 {
			id, _ = strconv.ParseInt(key[bStart+1:bEnd], 10, 64)
		}

	}

	switch d := d.(type) {
	case map[string]interface{}:
		if len(rest) > 0 {
			x := getKeyValues(d[key], rest)
			for _, z := range x {
				values = append(values, z)
			}
		} else {
			_, ok := (d[key]).([]interface{})
			if ok == false {
				values = append(values, d[key])
			}
		}
	case []int:
		var ids []int64
		if id != -1 {
			if len(d) == 0 || int(id) >= len(d) || id < 0 {
				break
			}
			ids = append(ids, int64(id))
		} else {
			for i := range d {
				ids = append(ids, int64(i))
			}
		}

		for _, id := range ids {
			values = append(values, d[id])
		}
	case []string:
		var ids []int64
		if id != -1 {
			if len(d) == 0 || int(id) >= len(d) || id < 0 {
				break
			}

			ids = append(ids, int64(id))
		} else {
			for i := range d {
				ids = append(ids, int64(i))
			}
		}

		for _, id := range ids {
			values = append(values, d[id])
		}
	case []bool:
		var ids []int64
		if id != -1 {
			if len(d) == 0 || int(id) >= len(d) || id < 0 {
				break
			}
			ids = append(ids, int64(id))
		} else {
			for i := range d {
				ids = append(ids, int64(i))
			}
		}

		for _, id := range ids {
			values = append(values, d[id])
		}
	case []float64:
		var ids []int64
		if id != -1 {
			if len(d) == 0 || int(id) >= len(d) || id < 0 {
				break
			}
			ids = append(ids, int64(id))
		} else {
			for i := range d {
				ids = append(ids, int64(i))
			}
		}

		for _, id := range ids {
			values = append(values, d[id])
		}
	case []interface{}:
		var ids []int64
		if id != -1 {
			if len(d) == 0 || int(id) >= len(d) || id < 0 {
				break
			}
			ids = append(ids, int64(id))
		} else {
			for i := range d {
				ids = append(ids, int64(i))
			}
		}

		for _, id := range ids {
			if len(rest) > 0 {
				x := getKeyValues(d[id], rest)
				for _, z := range x {
					values = append(values, z)
				}
			} else {
				values = append(values, d[id])
			}
		}

	default:
	}

	return values
}

func equals(value interface{}, comparator interface{}) bool {
	switch value := value.(type) {
	case float64:
		// comparing floats.....?
		c, ok := comparator.(float64)
		if ok == false {
			return false
		}
		return value == c
	case string:
		return value == comparator
	case bool:
		return value == comparator
	default:
		if value == nil && comparator == nil {
			return true
		}
		return false
	}
}

func greaterthan(value interface{}, comparator interface{}) bool {
	switch value := value.(type) {
	case float64:
		c, ok := comparator.(float64)
		if ok == false {
			return false
		}
		return value > c
	default:
		return false
	}
}

func lessthan(value interface{}, comparator interface{}) bool {
	switch value := value.(type) {
	case float64:
		c, ok := comparator.(float64)
		if ok == false {
			return false
		}
		return value < c
	default:
		return false
	}
}

func subsetof(value interface{}, comparator interface{}) bool {
	switch value := value.(type) {
	case string:
		return strings.Contains(value, comparator.(string))
	}
	return false
}
<|endoftext|>


<commit_before>package blocks

import (
    "log"
    "encoding/json"
    "labix.org/v2/mgo"
    //"labix.org/v2/mgo/bson"
)

func ToMongoDB(b *Block) {

    type toMongoDBRule struct {
        Host string
        Database string
        Collection string
    }

    var rule *toMongoDBRule
    var collection *mgo.Collection
    //var session  * mgo.Session
    //var db *mgo.Database
    //var db = "test"

    for {
        select {
        case m := <-b.Routes["set_rule"]:
            if rule == nil {
                rule = &toMongoDBRule{}
            }
            unmarshal(m, rule)

            session, err := mgo.Dial(rule.Host)
            if err != nil {
                log.Println("Could not connect to MongoDB", err.Error())
                break
            }
            if len(rule.Database) <=0 {
                log.Println("Database field is empty")
                break
            }
            if len(rule.Collection) <= 0 {
                log.Println("Collection name is empty")
                break
            }
            collection = session.DB(rule.Database).C(rule.Collection)

        case r := <-b.Routes["get_rule"]:
            if rule == nil {
                marshal(r, &toMongoDBRule{})
            } else {
                marshal(r, rule)
            }

        case msg := <-b.InChan:
            if rule == nil {
                break
            }
            msgStr, err := json.Marshal(msg.Msg)
            if err != nil {
                log.Println("wow bad json" , err.Error())
                break
            }
            var m map[string]interface{}
            err = json.Unmarshal(msgStr, &m)
            if err != nil {
                log.Println("wow bad json" , err.Error())
                break
            }
            err = collection.Insert(m)

        case msg := <-b.AddChan:
            updateOutChans(msg, b)
        case <-b.QuitChan:
            quit(b)
            return
        }
    }
}
<commit_msg>removed toMongoDB block<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2016 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Test cases (Policy object e2e):
// Test 1: Pass two fully populated PolicySpecs and expect the series of operations to succeed.
// Test 2: Pass one fully populated PolicySpec and another empty PolicySpec and expect the series of operations to succeed.
// Test 3: Pass one partially populated PolicySpec and another fully populated PolicySpec and expect the series of operations to succeed.

// Series of operations each test goes through:
// Update meta1 - check for failure (because it doesn't exist).
// Create meta1 with spec1.
// Apply meta2 with spec2.
// Get meta1 and meta2, compare spec1 and spec2.
// Update meta1 with spec2.
// Get meta1 compare spec2.
// List (empty Meta) ... Get meta1 and meta2.
// List (using Meta1) ... Get meta1.
// Delete meta1.
// Get meta1 ... fail.
// Delete meta2.
// List (empty Meta) ... Get no entries (should not error).

package client_test

import (
	"errors"
	"log"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"

	"github.com/projectcalico/libcalico-go/lib/api"
	"github.com/projectcalico/libcalico-go/lib/testutils"
)

var order1 = 99.999
var order2 = 22.222

var policySpec1 = api.PolicySpec{
	Order:        &order1,
	IngressRules: []api.Rule{testutils.InRule1, testutils.InRule2},
	EgressRules:  []api.Rule{testutils.EgressRule1, testutils.EgressRule2},
	Selector:     "policy1-selector",
}

var policySpec2 = api.PolicySpec{
	Order:        &order2,
	IngressRules: []api.Rule{testutils.InRule2, testutils.InRule1},
	EgressRules:  []api.Rule{testutils.EgressRule2, testutils.EgressRule1},
	Selector:     "policy2-selector",
}

var _ = Describe("Policy tests", func() {

	DescribeTable("Policy e2e tests",
		func(meta1, meta2 api.PolicyMetadata, spec1, spec2 api.PolicySpec) {

			// Erase etcd clean.
			testutils.CleanEtcd()

			// Create a new client.
			c, err := testutils.NewClient("")
			if err != nil {
				log.Println("Error creating client:", err)
			}
			By("Updating the policy before it is created")
			_, outError := c.Policies().Update(&api.Policy{Metadata: meta1, Spec: spec1})

			// Should return an error.
			Expect(outError.Error()).To(Equal(errors.New("resource does not exist: Policy(name=policy1)").Error()))

			By("Create, Apply, Get and compare")

			// Create a policy with meta1 and spec1.
			_, outError = c.Policies().Create(&api.Policy{Metadata: meta1, Spec: spec1})
			Expect(outError).NotTo(HaveOccurred())

			// Apply a policy with meta2 and spec2.
			_, outError = c.Policies().Apply(&api.Policy{Metadata: meta2, Spec: spec2})
			Expect(outError).NotTo(HaveOccurred())

			// Get policy with meta1.
			outPolicy1, outError1 := c.Policies().Get(meta1)
			log.Println("Out Policy object: ", outPolicy1)

			// Get policy with meta2.
			outPolicy2, outError2 := c.Policies().Get(meta2)
			log.Println("Out Policy object: ", outPolicy2)

			// Should match spec1 & outPolicy1 and outPolicy2 & spec2 and errors to be nil.
			Expect(outError1).NotTo(HaveOccurred())
			Expect(outError2).NotTo(HaveOccurred())
			Expect(outPolicy1.Spec).To(Equal(spec1))
			Expect(outPolicy2.Spec).To(Equal(spec2))

			By("Update, Get and compare")

			// Update meta1 policy with spec2.
			_, outError = c.Policies().Update(&api.Policy{Metadata: meta1, Spec: spec2})
			Expect(outError).NotTo(HaveOccurred())

			// Get policy with meta1.
			outPolicy1, outError1 = c.Policies().Get(meta1)

			// Assert the Spec for policy with meta1 matches spec2 and no error.
			Expect(outError1).NotTo(HaveOccurred())
			Expect(outPolicy1.Spec).To(Equal(spec2))

			By("List all the policies and compare")

			// Get a list of policiess.
			policyList, outError := c.Policies().List(api.PolicyMetadata{})
			Expect(outError).NotTo(HaveOccurred())
			log.Println("Get policy list returns: ", policyList.Items)
			metas := []api.PolicyMetadata{meta1, meta2}
			expectedPolicies := []api.Policy{}
			// Go through meta list and append them to expectedPolicies.
			for _, v := range metas {
				p, outError := c.Policies().Get(v)
				Expect(outError).NotTo(HaveOccurred())
				expectedPolicies = append(expectedPolicies, *p)
			}

			// Assert the returned policyList is has the meta1 and meta2 policies.
			Expect(policyList.Items).To(Equal(expectedPolicies))

			By("List a specific policy and compare")

			// Get a policy list with meta1.
			policyList, outError = c.Policies().List(meta1)
			Expect(outError).NotTo(HaveOccurred())
			log.Println("Get policy list returns: ", policyList.Items)

			// Get a policy with meta1.
			outPolicy1, outError1 = c.Policies().Get(meta1)

			// Assert they are equal and no errors.
			Expect(outError1).NotTo(HaveOccurred())
			Expect(policyList.Items[0].Spec).To(Equal(outPolicy1.Spec))

			By("Delete, Get and assert error")

			// Delete a policy with meta1.
			outError1 = c.Policies().Delete(meta1)
			Expect(outError1).NotTo(HaveOccurred())

			// Get a policy with meta1.
			_, outError = c.Policies().Get(meta1)

			// Expect an error since the policy was deleted.
			Expect(outError.Error()).To(Equal(errors.New("resource does not exist: Policy(name=policy1)").Error()))

			// Delete the second policy with meta2.
			outError1 = c.Policies().Delete(meta2)
			Expect(outError1).NotTo(HaveOccurred())

			By("Delete all the policies, Get policy list and expect empty policy list")

			// Both policies are deleted in the calls above.
			// Get the list of all the policys.
			policyList, outError = c.Policies().List(api.PolicyMetadata{})
			Expect(outError).NotTo(HaveOccurred())
			log.Println("Get policy list returns: ", policyList.Items)

			// Create an empty policy list.
			// Note: you can't use make([]api.Policy, 0) because it creates an empty underlying struct,
			// whereas new([]api.Policy) just returns a pointer without creating an empty struct.
			emptyPolicyList := new([]api.Policy)

			// Expect returned policyList to contain empty policyList.
			Expect(policyList.Items).To(Equal(*emptyPolicyList))

		},

		// Test 1: Pass two fully populated PolicySpecs and expect the series of operations to succeed.
		Entry("Two fully populated PolicySpecs",
			api.PolicyMetadata{Name: "policy1"},
			api.PolicyMetadata{Name: "policy2"},
			policySpec1,
			policySpec2,
		),

		// Test 2: Pass one fully populated PolicySpec and another empty PolicySpec and expect the series of operations to succeed.
		Entry("One fully populated PolicySpec and another empty PolicySpec",
			api.PolicyMetadata{Name: "policy1"},
			api.PolicyMetadata{Name: "policy2"},
			policySpec1,
			api.PolicySpec{},
		),

		// Test 3: Pass one partially populated PolicySpec and another fully populated PolicySpec and expect the series of operations to succeed.
		Entry("One partially populated PolicySpec and another fully populated PolicySpec",
			api.PolicyMetadata{Name: "policy1"},
			api.PolicyMetadata{Name: "policy2"},
			api.PolicySpec{
				Selector: "policy1-selector",
			},
			policySpec2,
		),
	)
})
<commit_msg>Test untracked policy<commit_after>// Copyright (c) 2016 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Test cases (Policy object e2e):
// Test 1: Pass two fully populated PolicySpecs and expect the series of operations to succeed.
// Test 2: Pass one fully populated PolicySpec and another empty PolicySpec and expect the series of operations to succeed.
// Test 3: Pass one partially populated PolicySpec and another fully populated PolicySpec and expect the series of operations to succeed.

// Series of operations each test goes through:
// Update meta1 - check for failure (because it doesn't exist).
// Create meta1 with spec1.
// Apply meta2 with spec2.
// Get meta1 and meta2, compare spec1 and spec2.
// Update meta1 with spec2.
// Get meta1 compare spec2.
// List (empty Meta) ... Get meta1 and meta2.
// List (using Meta1) ... Get meta1.
// Delete meta1.
// Get meta1 ... fail.
// Delete meta2.
// List (empty Meta) ... Get no entries (should not error).

package client_test

import (
	"errors"
	"log"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"

	"github.com/projectcalico/libcalico-go/lib/api"
	"github.com/projectcalico/libcalico-go/lib/testutils"
)

var order1 = 99.999
var order2 = 22.222

var policySpec1 = api.PolicySpec{
	Order:        &order1,
	IngressRules: []api.Rule{testutils.InRule1, testutils.InRule2},
	EgressRules:  []api.Rule{testutils.EgressRule1, testutils.EgressRule2},
	Selector:     "policy1-selector",
}

var policySpec2 = api.PolicySpec{
	Order:        &order2,
	IngressRules: []api.Rule{testutils.InRule2, testutils.InRule1},
	EgressRules:  []api.Rule{testutils.EgressRule2, testutils.EgressRule1},
	Selector:     "policy2-selector",
	Untracked:    true,
}

var _ = Describe("Policy tests", func() {

	DescribeTable("Policy e2e tests",
		func(meta1, meta2 api.PolicyMetadata, spec1, spec2 api.PolicySpec) {

			// Erase etcd clean.
			testutils.CleanEtcd()

			// Create a new client.
			c, err := testutils.NewClient("")
			if err != nil {
				log.Println("Error creating client:", err)
			}
			By("Updating the policy before it is created")
			_, outError := c.Policies().Update(&api.Policy{Metadata: meta1, Spec: spec1})

			// Should return an error.
			Expect(outError.Error()).To(Equal(errors.New("resource does not exist: Policy(name=policy1)").Error()))

			By("Create, Apply, Get and compare")

			// Create a policy with meta1 and spec1.
			_, outError = c.Policies().Create(&api.Policy{Metadata: meta1, Spec: spec1})
			Expect(outError).NotTo(HaveOccurred())

			// Apply a policy with meta2 and spec2.
			_, outError = c.Policies().Apply(&api.Policy{Metadata: meta2, Spec: spec2})
			Expect(outError).NotTo(HaveOccurred())

			// Get policy with meta1.
			outPolicy1, outError1 := c.Policies().Get(meta1)
			log.Println("Out Policy object: ", outPolicy1)

			// Get policy with meta2.
			outPolicy2, outError2 := c.Policies().Get(meta2)
			log.Println("Out Policy object: ", outPolicy2)

			// Should match spec1 & outPolicy1 and outPolicy2 & spec2 and errors to be nil.
			Expect(outError1).NotTo(HaveOccurred())
			Expect(outError2).NotTo(HaveOccurred())
			Expect(outPolicy1.Spec).To(Equal(spec1))
			Expect(outPolicy2.Spec).To(Equal(spec2))

			By("Update, Get and compare")

			// Update meta1 policy with spec2.
			_, outError = c.Policies().Update(&api.Policy{Metadata: meta1, Spec: spec2})
			Expect(outError).NotTo(HaveOccurred())

			// Get policy with meta1.
			outPolicy1, outError1 = c.Policies().Get(meta1)

			// Assert the Spec for policy with meta1 matches spec2 and no error.
			Expect(outError1).NotTo(HaveOccurred())
			Expect(outPolicy1.Spec).To(Equal(spec2))

			By("List all the policies and compare")

			// Get a list of policiess.
			policyList, outError := c.Policies().List(api.PolicyMetadata{})
			Expect(outError).NotTo(HaveOccurred())
			log.Println("Get policy list returns: ", policyList.Items)
			metas := []api.PolicyMetadata{meta1, meta2}
			expectedPolicies := []api.Policy{}
			// Go through meta list and append them to expectedPolicies.
			for _, v := range metas {
				p, outError := c.Policies().Get(v)
				Expect(outError).NotTo(HaveOccurred())
				expectedPolicies = append(expectedPolicies, *p)
			}

			// Assert the returned policyList is has the meta1 and meta2 policies.
			Expect(policyList.Items).To(Equal(expectedPolicies))

			By("List a specific policy and compare")

			// Get a policy list with meta1.
			policyList, outError = c.Policies().List(meta1)
			Expect(outError).NotTo(HaveOccurred())
			log.Println("Get policy list returns: ", policyList.Items)

			// Get a policy with meta1.
			outPolicy1, outError1 = c.Policies().Get(meta1)

			// Assert they are equal and no errors.
			Expect(outError1).NotTo(HaveOccurred())
			Expect(policyList.Items[0].Spec).To(Equal(outPolicy1.Spec))

			By("Delete, Get and assert error")

			// Delete a policy with meta1.
			outError1 = c.Policies().Delete(meta1)
			Expect(outError1).NotTo(HaveOccurred())

			// Get a policy with meta1.
			_, outError = c.Policies().Get(meta1)

			// Expect an error since the policy was deleted.
			Expect(outError.Error()).To(Equal(errors.New("resource does not exist: Policy(name=policy1)").Error()))

			// Delete the second policy with meta2.
			outError1 = c.Policies().Delete(meta2)
			Expect(outError1).NotTo(HaveOccurred())

			By("Delete all the policies, Get policy list and expect empty policy list")

			// Both policies are deleted in the calls above.
			// Get the list of all the policys.
			policyList, outError = c.Policies().List(api.PolicyMetadata{})
			Expect(outError).NotTo(HaveOccurred())
			log.Println("Get policy list returns: ", policyList.Items)

			// Create an empty policy list.
			// Note: you can't use make([]api.Policy, 0) because it creates an empty underlying struct,
			// whereas new([]api.Policy) just returns a pointer without creating an empty struct.
			emptyPolicyList := new([]api.Policy)

			// Expect returned policyList to contain empty policyList.
			Expect(policyList.Items).To(Equal(*emptyPolicyList))

		},

		// Test 1: Pass two fully populated PolicySpecs and expect the series of operations to succeed.
		Entry("Two fully populated PolicySpecs",
			api.PolicyMetadata{Name: "policy1"},
			api.PolicyMetadata{Name: "policy2"},
			policySpec1,
			policySpec2,
		),

		// Test 2: Pass one fully populated PolicySpec and another empty PolicySpec and expect the series of operations to succeed.
		Entry("One fully populated PolicySpec and another empty PolicySpec",
			api.PolicyMetadata{Name: "policy1"},
			api.PolicyMetadata{Name: "policy2"},
			policySpec1,
			api.PolicySpec{},
		),

		// Test 3: Pass one partially populated PolicySpec and another fully populated PolicySpec and expect the series of operations to succeed.
		Entry("One partially populated PolicySpec and another fully populated PolicySpec",
			api.PolicyMetadata{Name: "policy1"},
			api.PolicyMetadata{Name: "policy2"},
			api.PolicySpec{
				Selector: "policy1-selector",
			},
			policySpec2,
		),
	)
})
<|endoftext|>


<commit_before>// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package discovery

import (
	"fmt"
	"sync"
	"time"

	"github.com/prometheus/common/log"
	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/discovery/azure"
	"github.com/prometheus/prometheus/discovery/consul"
	"github.com/prometheus/prometheus/discovery/dns"
	"github.com/prometheus/prometheus/discovery/ec2"
	"github.com/prometheus/prometheus/discovery/file"
	"github.com/prometheus/prometheus/discovery/gce"
	"github.com/prometheus/prometheus/discovery/kubernetes"
	"github.com/prometheus/prometheus/discovery/marathon"
	"github.com/prometheus/prometheus/discovery/triton"
	"github.com/prometheus/prometheus/discovery/zookeeper"
	"golang.org/x/net/context"
)

// A TargetProvider provides information about target groups. It maintains a set
// of sources from which TargetGroups can originate. Whenever a target provider
// detects a potential change, it sends the TargetGroup through its provided channel.
//
// The TargetProvider does not have to guarantee that an actual change happened.
// It does guarantee that it sends the new TargetGroup whenever a change happens.
//
// TargetProviders should initially send a full set of all discoverable TargetGroups.
type TargetProvider interface {
	// Run hands a channel to the target provider through which it can send
	// updated target groups.
	// Must returns if the context gets canceled. It should not close the update
	// channel on returning.
	Run(ctx context.Context, up chan<- []*config.TargetGroup)
}

// ProvidersFromConfig returns all TargetProviders configured in cfg.
func ProvidersFromConfig(cfg config.ServiceDiscoveryConfig) map[string]TargetProvider {
	providers := map[string]TargetProvider{}

	app := func(mech string, i int, tp TargetProvider) {
		providers[fmt.Sprintf("%s/%d", mech, i)] = tp
	}

	for i, c := range cfg.DNSSDConfigs {
		app("dns", i, dns.NewDiscovery(c))
	}
	for i, c := range cfg.FileSDConfigs {
		app("file", i, file.NewDiscovery(c))
	}
	for i, c := range cfg.ConsulSDConfigs {
		k, err := consul.NewDiscovery(c)
		if err != nil {
			log.Errorf("Cannot create Consul discovery: %s", err)
			continue
		}
		app("consul", i, k)
	}
	for i, c := range cfg.MarathonSDConfigs {
		m, err := marathon.NewDiscovery(c)
		if err != nil {
			log.Errorf("Cannot create Marathon discovery: %s", err)
			continue
		}
		app("marathon", i, m)
	}
	for i, c := range cfg.KubernetesSDConfigs {
		k, err := kubernetes.New(log.Base(), c)
		if err != nil {
			log.Errorf("Cannot create Kubernetes discovery: %s", err)
			continue
		}
		app("kubernetes", i, k)
	}
	for i, c := range cfg.ServersetSDConfigs {
		app("serverset", i, zookeeper.NewServersetDiscovery(c))
	}
	for i, c := range cfg.NerveSDConfigs {
		app("nerve", i, zookeeper.NewNerveDiscovery(c))
	}
	for i, c := range cfg.EC2SDConfigs {
		app("ec2", i, ec2.NewDiscovery(c))
	}
	for i, c := range cfg.GCESDConfigs {
		gced, err := gce.NewDiscovery(c)
		if err != nil {
			log.Errorf("Cannot initialize GCE discovery: %s", err)
			continue
		}
		app("gce", i, gced)
	}
	for i, c := range cfg.AzureSDConfigs {
		app("azure", i, azure.NewDiscovery(c))
	}
	for i, c := range cfg.TritonSDConfigs {
		t, err := triton.New(log.With("sd", "triton"), c)
		if err != nil {
			log.Errorf("Cannot create Triton discovery: %s", err)
			continue
		}
		app("triton", i, t)
	}
	if len(cfg.StaticConfigs) > 0 {
		app("static", 0, NewStaticProvider(cfg.StaticConfigs))
	}

	return providers
}

// StaticProvider holds a list of target groups that never change.
type StaticProvider struct {
	TargetGroups []*config.TargetGroup
}

// NewStaticProvider returns a StaticProvider configured with the given
// target groups.
func NewStaticProvider(groups []*config.TargetGroup) *StaticProvider {
	for i, tg := range groups {
		tg.Source = fmt.Sprintf("%d", i)
	}
	return &StaticProvider{groups}
}

// Run implements the TargetProvider interface.
func (sd *StaticProvider) Run(ctx context.Context, ch chan<- []*config.TargetGroup) {
	// We still have to consider that the consumer exits right away in which case
	// the context will be canceled.
	select {
	case ch <- sd.TargetGroups:
	case <-ctx.Done():
	}
	close(ch)
}

// TargetSet handles multiple TargetProviders and sends a full overview of their
// discovered TargetGroups to a Syncer.
type TargetSet struct {
	mtx sync.RWMutex
	// Sets of targets by a source string that is unique across target providers.
	tgroups map[string]*config.TargetGroup

	syncer Syncer

	syncCh          chan struct{}
	providerCh      chan map[string]TargetProvider
	cancelProviders func()
}

// Syncer receives updates complete sets of TargetGroups.
type Syncer interface {
	Sync([]*config.TargetGroup)
}

// NewTargetSet returns a new target sending TargetGroups to the Syncer.
func NewTargetSet(s Syncer) *TargetSet {
	return &TargetSet{
		syncCh:     make(chan struct{}, 1),
		providerCh: make(chan map[string]TargetProvider),
		syncer:     s,
	}
}

// Run starts the processing of target providers and their updates.
// It blocks until the context gets canceled.
func (ts *TargetSet) Run(ctx context.Context) {
Loop:
	for {
		// Throttle syncing to once per five seconds.
		select {
		case <-ctx.Done():
			break Loop
		case p := <-ts.providerCh:
			ts.updateProviders(ctx, p)
		case <-time.After(5 * time.Second):
		}

		select {
		case <-ctx.Done():
			break Loop
		case <-ts.syncCh:
			ts.sync()
		case p := <-ts.providerCh:
			ts.updateProviders(ctx, p)
		}
	}
}

func (ts *TargetSet) sync() {
	ts.mtx.RLock()
	var all []*config.TargetGroup
	for _, tg := range ts.tgroups {
		all = append(all, tg)
	}
	ts.mtx.RUnlock()

	ts.syncer.Sync(all)
}

// UpdateProviders sets new target providers for the target set.
func (ts *TargetSet) UpdateProviders(p map[string]TargetProvider) {
	ts.providerCh <- p
}

func (ts *TargetSet) updateProviders(ctx context.Context, providers map[string]TargetProvider) {
	// Lock for the entire time. This may mean up to 5 seconds until the full initial set
	// is retrieved and applied.
	// We could release earlier with some tweaks, but this is easier to reason about.
	ts.mtx.Lock()
	defer ts.mtx.Unlock()

	// Stop all previous target providers of the target set.
	if ts.cancelProviders != nil {
		ts.cancelProviders()
	}
	ctx, ts.cancelProviders = context.WithCancel(ctx)

	var wg sync.WaitGroup
	// (Re-)create a fresh tgroups map to not keep stale targets around. We
	// will retrieve all targets below anyway, so cleaning up everything is
	// safe and doesn't inflict any additional cost.
	ts.tgroups = map[string]*config.TargetGroup{}

	for name, prov := range providers {
		wg.Add(1)

		updates := make(chan []*config.TargetGroup)
		go prov.Run(ctx, updates)

		go func(name string, prov TargetProvider) {
			select {
			case <-ctx.Done():
			case initial, ok := <-updates:
				// Handle the case that a target provider exits and closes the channel
				// before the context is done.
				if !ok {
					break
				}
				// First set of all targets the provider knows.
				for _, tgroup := range initial {
					ts.setTargetGroup(name, tgroup)
				}
			case <-time.After(5 * time.Second):
				// Initial set didn't arrive. Act as if it was empty
				// and wait for updates later on.
			}
			wg.Done()

			// Start listening for further updates.
			for {
				select {
				case <-ctx.Done():
					return
				case tgs, ok := <-updates:
					// Handle the case that a target provider exits and closes the channel
					// before the context is done.
					if !ok {
						return
					}
					for _, tg := range tgs {
						ts.update(name, tg)
					}
				}
			}
		}(name, prov)
	}

	// We wait for a full initial set of target groups before releasing the mutex
	// to ensure the initial sync is complete and there are no races with subsequent updates.
	wg.Wait()
	// Just signal that there are initial sets to sync now. Actual syncing must only
	// happen in the runScraping loop.
	select {
	case ts.syncCh <- struct{}{}:
	default:
	}
}

// update handles a target group update from a target provider identified by the name.
func (ts *TargetSet) update(name string, tgroup *config.TargetGroup) {
	ts.mtx.Lock()
	defer ts.mtx.Unlock()

	ts.setTargetGroup(name, tgroup)

	select {
	case ts.syncCh <- struct{}{}:
	default:
	}
}

func (ts *TargetSet) setTargetGroup(name string, tg *config.TargetGroup) {
	if tg == nil {
		return
	}
	ts.tgroups[name+"/"+tg.Source] = tg
}
<commit_msg>Fix Map Race by Moving Locking closer to the Write (#2476)<commit_after>// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package discovery

import (
	"fmt"
	"sync"
	"time"

	"github.com/prometheus/common/log"
	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/discovery/azure"
	"github.com/prometheus/prometheus/discovery/consul"
	"github.com/prometheus/prometheus/discovery/dns"
	"github.com/prometheus/prometheus/discovery/ec2"
	"github.com/prometheus/prometheus/discovery/file"
	"github.com/prometheus/prometheus/discovery/gce"
	"github.com/prometheus/prometheus/discovery/kubernetes"
	"github.com/prometheus/prometheus/discovery/marathon"
	"github.com/prometheus/prometheus/discovery/triton"
	"github.com/prometheus/prometheus/discovery/zookeeper"
	"golang.org/x/net/context"
)

// A TargetProvider provides information about target groups. It maintains a set
// of sources from which TargetGroups can originate. Whenever a target provider
// detects a potential change, it sends the TargetGroup through its provided channel.
//
// The TargetProvider does not have to guarantee that an actual change happened.
// It does guarantee that it sends the new TargetGroup whenever a change happens.
//
// TargetProviders should initially send a full set of all discoverable TargetGroups.
type TargetProvider interface {
	// Run hands a channel to the target provider through which it can send
	// updated target groups.
	// Must returns if the context gets canceled. It should not close the update
	// channel on returning.
	Run(ctx context.Context, up chan<- []*config.TargetGroup)
}

// ProvidersFromConfig returns all TargetProviders configured in cfg.
func ProvidersFromConfig(cfg config.ServiceDiscoveryConfig) map[string]TargetProvider {
	providers := map[string]TargetProvider{}

	app := func(mech string, i int, tp TargetProvider) {
		providers[fmt.Sprintf("%s/%d", mech, i)] = tp
	}

	for i, c := range cfg.DNSSDConfigs {
		app("dns", i, dns.NewDiscovery(c))
	}
	for i, c := range cfg.FileSDConfigs {
		app("file", i, file.NewDiscovery(c))
	}
	for i, c := range cfg.ConsulSDConfigs {
		k, err := consul.NewDiscovery(c)
		if err != nil {
			log.Errorf("Cannot create Consul discovery: %s", err)
			continue
		}
		app("consul", i, k)
	}
	for i, c := range cfg.MarathonSDConfigs {
		m, err := marathon.NewDiscovery(c)
		if err != nil {
			log.Errorf("Cannot create Marathon discovery: %s", err)
			continue
		}
		app("marathon", i, m)
	}
	for i, c := range cfg.KubernetesSDConfigs {
		k, err := kubernetes.New(log.Base(), c)
		if err != nil {
			log.Errorf("Cannot create Kubernetes discovery: %s", err)
			continue
		}
		app("kubernetes", i, k)
	}
	for i, c := range cfg.ServersetSDConfigs {
		app("serverset", i, zookeeper.NewServersetDiscovery(c))
	}
	for i, c := range cfg.NerveSDConfigs {
		app("nerve", i, zookeeper.NewNerveDiscovery(c))
	}
	for i, c := range cfg.EC2SDConfigs {
		app("ec2", i, ec2.NewDiscovery(c))
	}
	for i, c := range cfg.GCESDConfigs {
		gced, err := gce.NewDiscovery(c)
		if err != nil {
			log.Errorf("Cannot initialize GCE discovery: %s", err)
			continue
		}
		app("gce", i, gced)
	}
	for i, c := range cfg.AzureSDConfigs {
		app("azure", i, azure.NewDiscovery(c))
	}
	for i, c := range cfg.TritonSDConfigs {
		t, err := triton.New(log.With("sd", "triton"), c)
		if err != nil {
			log.Errorf("Cannot create Triton discovery: %s", err)
			continue
		}
		app("triton", i, t)
	}
	if len(cfg.StaticConfigs) > 0 {
		app("static", 0, NewStaticProvider(cfg.StaticConfigs))
	}

	return providers
}

// StaticProvider holds a list of target groups that never change.
type StaticProvider struct {
	TargetGroups []*config.TargetGroup
}

// NewStaticProvider returns a StaticProvider configured with the given
// target groups.
func NewStaticProvider(groups []*config.TargetGroup) *StaticProvider {
	for i, tg := range groups {
		tg.Source = fmt.Sprintf("%d", i)
	}
	return &StaticProvider{groups}
}

// Run implements the TargetProvider interface.
func (sd *StaticProvider) Run(ctx context.Context, ch chan<- []*config.TargetGroup) {
	// We still have to consider that the consumer exits right away in which case
	// the context will be canceled.
	select {
	case ch <- sd.TargetGroups:
	case <-ctx.Done():
	}
	close(ch)
}

// TargetSet handles multiple TargetProviders and sends a full overview of their
// discovered TargetGroups to a Syncer.
type TargetSet struct {
	mtx sync.RWMutex
	// Sets of targets by a source string that is unique across target providers.
	tgroups map[string]*config.TargetGroup

	syncer Syncer

	syncCh          chan struct{}
	providerCh      chan map[string]TargetProvider
	cancelProviders func()
}

// Syncer receives updates complete sets of TargetGroups.
type Syncer interface {
	Sync([]*config.TargetGroup)
}

// NewTargetSet returns a new target sending TargetGroups to the Syncer.
func NewTargetSet(s Syncer) *TargetSet {
	return &TargetSet{
		syncCh:     make(chan struct{}, 1),
		providerCh: make(chan map[string]TargetProvider),
		syncer:     s,
	}
}

// Run starts the processing of target providers and their updates.
// It blocks until the context gets canceled.
func (ts *TargetSet) Run(ctx context.Context) {
Loop:
	for {
		// Throttle syncing to once per five seconds.
		select {
		case <-ctx.Done():
			break Loop
		case p := <-ts.providerCh:
			ts.updateProviders(ctx, p)
		case <-time.After(5 * time.Second):
		}

		select {
		case <-ctx.Done():
			break Loop
		case <-ts.syncCh:
			ts.sync()
		case p := <-ts.providerCh:
			ts.updateProviders(ctx, p)
		}
	}
}

func (ts *TargetSet) sync() {
	ts.mtx.RLock()
	var all []*config.TargetGroup
	for _, tg := range ts.tgroups {
		all = append(all, tg)
	}
	ts.mtx.RUnlock()

	ts.syncer.Sync(all)
}

// UpdateProviders sets new target providers for the target set.
func (ts *TargetSet) UpdateProviders(p map[string]TargetProvider) {
	ts.providerCh <- p
}

func (ts *TargetSet) updateProviders(ctx context.Context, providers map[string]TargetProvider) {

	// Stop all previous target providers of the target set.
	if ts.cancelProviders != nil {
		ts.cancelProviders()
	}
	ctx, ts.cancelProviders = context.WithCancel(ctx)

	var wg sync.WaitGroup
	// (Re-)create a fresh tgroups map to not keep stale targets around. We
	// will retrieve all targets below anyway, so cleaning up everything is
	// safe and doesn't inflict any additional cost.
	ts.mtx.Lock()
	ts.tgroups = map[string]*config.TargetGroup{}
	ts.mtx.Unlock()

	for name, prov := range providers {
		wg.Add(1)

		updates := make(chan []*config.TargetGroup)
		go prov.Run(ctx, updates)

		go func(name string, prov TargetProvider) {
			select {
			case <-ctx.Done():
			case initial, ok := <-updates:
				// Handle the case that a target provider exits and closes the channel
				// before the context is done.
				if !ok {
					break
				}
				// First set of all targets the provider knows.
				for _, tgroup := range initial {
					ts.setTargetGroup(name, tgroup)
				}
			case <-time.After(5 * time.Second):
				// Initial set didn't arrive. Act as if it was empty
				// and wait for updates later on.
			}
			wg.Done()

			// Start listening for further updates.
			for {
				select {
				case <-ctx.Done():
					return
				case tgs, ok := <-updates:
					// Handle the case that a target provider exits and closes the channel
					// before the context is done.
					if !ok {
						return
					}
					for _, tg := range tgs {
						ts.update(name, tg)
					}
				}
			}
		}(name, prov)
	}

	// We wait for a full initial set of target groups before releasing the mutex
	// to ensure the initial sync is complete and there are no races with subsequent updates.
	wg.Wait()
	// Just signal that there are initial sets to sync now. Actual syncing must only
	// happen in the runScraping loop.
	select {
	case ts.syncCh <- struct{}{}:
	default:
	}
}

// update handles a target group update from a target provider identified by the name.
func (ts *TargetSet) update(name string, tgroup *config.TargetGroup) {
	ts.setTargetGroup(name, tgroup)

	select {
	case ts.syncCh <- struct{}{}:
	default:
	}
}

func (ts *TargetSet) setTargetGroup(name string, tg *config.TargetGroup) {
	ts.mtx.Lock()
	defer ts.mtx.Unlock()

	if tg == nil {
		return
	}
	ts.tgroups[name+"/"+tg.Source] = tg
}
<|endoftext|>


<commit_before>package server

import (
	"bufio"
	"bytes"
	"io"
	"net/http"

	"github.com/influxdata/chronograf"
)

// URLPrefixer is a wrapper for an http.Handler that will prefix all occurrences of a relative URL with the configured Prefix
type URLPrefixer struct {
	Prefix string            // the prefix to be appended after any detected Attrs
	Next   http.Handler      // the http.Handler which will generate the content to be modified by this handler
	Attrs  [][]byte          // a list of attrs that should have their URLs prefixed. For example `src="` or `href="` would be valid
	Logger chronograf.Logger // The logger where prefixing errors will be dispatched to
}

type wrapResponseWriter struct {
	http.ResponseWriter
	Substitute *io.PipeWriter

	headerWritten bool
	dupHeader     http.Header
}

func (wrw wrapResponseWriter) Write(p []byte) (int, error) {
	return wrw.Substitute.Write(p)
}

func (wrw wrapResponseWriter) WriteHeader(code int) {
	if !wrw.headerWritten {
		wrw.ResponseWriter.Header().Set("Content-Type", wrw.Header().Get("Content-Type"))
		wrw.headerWritten = true
	}
	wrw.ResponseWriter.WriteHeader(code)
}

// Header() copies the Header map from the underlying ResponseWriter to prevent
// modifications to it by callers
func (wrw wrapResponseWriter) Header() http.Header {
	wrw.dupHeader = http.Header{}
	origHeader := wrw.ResponseWriter.Header()
	for k, v := range origHeader {
		wrw.dupHeader[k] = v
	}
	return wrw.dupHeader
}

const CHUNK_SIZE int = 512

// ServeHTTP implements an http.Handler that prefixes relative URLs from the
// Next handler with the configured prefix. It does this by examining the
// stream through the ResponseWriter, and appending the Prefix after any of the
// Attrs detected in the stream.
func (up *URLPrefixer) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	// chunked transfer because we're modifying the response on the fly, so we
	// won't know the final content-length
	rw.Header().Set("Connection", "Keep-Alive")
	rw.Header().Set("Transfer-Encoding", "chunked")

	writtenCount := 0 // number of bytes written to rw

	// extract the flusher for flushing chunks
	flusher, ok := rw.(http.Flusher)
	if !ok {
		up.Logger.
			WithField("component", "prefixer").
			Fatal("Expected http.ResponseWriter to be an http.Flusher, but wasn't")
	}

	nextRead, nextWrite := io.Pipe()
	go func() {
		defer nextWrite.Close()
		up.Next.ServeHTTP(wrapResponseWriter{ResponseWriter: rw, Substitute: nextWrite}, r)
	}()

	// setup a buffer which is the max length of our target attrs
	b := make([]byte, up.maxlen(up.Attrs...))
	io.ReadFull(nextRead, b) // prime the buffer with the start of the input
	buf := bytes.NewBuffer(b)

	// Read next handler's response byte by byte
	src := bufio.NewScanner(nextRead)
	src.Split(bufio.ScanBytes)
	for {
		window := buf.Bytes()

		// advance a byte if window is not a src attr
		if matchlen, match := up.match(window, up.Attrs...); matchlen == 0 {
			if src.Scan() {
				// shift the next byte into buf
				rw.Write(buf.Next(1))
				writtenCount++
				buf.Write(src.Bytes())

				if writtenCount >= CHUNK_SIZE {
					flusher.Flush()
					writtenCount = 0
				}
			} else {
				if err := src.Err(); err != nil {
					up.Logger.
						WithField("component", "prefixer").
						Error("Error encountered while scanning: err:", err)
				}
				rw.Write(window)
				flusher.Flush()
				break
			}
			continue
		} else {
			buf.Next(matchlen) // advance to the relative URL
			for i := 0; i < matchlen; i++ {
				src.Scan()
				buf.Write(src.Bytes())
			}
			rw.Write(match)               // add the src attr to the output
			io.WriteString(rw, up.Prefix) // write the prefix
		}
	}
}

// match compares the subject against a list of targets. If there is a match
// between any of them a non-zero value is returned. The returned value is the
// length of the match. It is assumed that subject's length > length of all
// targets. The matching []byte is also returned as the second return parameter
func (up *URLPrefixer) match(subject []byte, targets ...[]byte) (int, []byte) {
	for _, target := range targets {
		if bytes.Equal(subject[:len(target)], target) {
			return len(target), target
		}
	}
	return 0, []byte{}
}

// maxlen returns the length of the largest []byte provided to it as an argument
func (up *URLPrefixer) maxlen(targets ...[]byte) int {
	max := 0
	for _, tgt := range targets {
		if tlen := len(tgt); tlen > max {
			max = tlen
		}
	}
	return max
}

// NewDefaultURLPrefixer returns a URLPrefixer that will prefix any src and
// href attributes found in HTML as well as any url() directives found in CSS
// with the provided prefix. Additionally, it will prefix any `data-basepath`
// attributes as well for informing front end logic about any prefixes. `next`
// is the next http.Handler that will have its output prefixed
func NewDefaultURLPrefixer(prefix string, next http.Handler, lg chronograf.Logger) *URLPrefixer {
	return &URLPrefixer{
		Prefix: prefix,
		Next:   next,
		Logger: lg,
		Attrs: [][]byte{
			[]byte(`src="`),
			[]byte(`href="`),
			[]byte(`url(`),
			[]byte(`data-basepath="`), // for forwarding basepath to frontend
		},
	}
}
<commit_msg>Fix url_prefixer to write asset headers<commit_after>package server

import (
	"bufio"
	"bytes"
	"io"
	"net/http"

	"github.com/influxdata/chronograf"
)

// URLPrefixer is a wrapper for an http.Handler that will prefix all occurrences of a relative URL with the configured Prefix
type URLPrefixer struct {
	Prefix string            // the prefix to be appended after any detected Attrs
	Next   http.Handler      // the http.Handler which will generate the content to be modified by this handler
	Attrs  [][]byte          // a list of attrs that should have their URLs prefixed. For example `src="` or `href="` would be valid
	Logger chronograf.Logger // The logger where prefixing errors will be dispatched to
}

type wrapResponseWriter struct {
	http.ResponseWriter
	Substitute *io.PipeWriter

	headerWritten bool
	dupHeader     *http.Header
}

func (wrw *wrapResponseWriter) Write(p []byte) (int, error) {
	return wrw.Substitute.Write(p)
}

func (wrw *wrapResponseWriter) WriteHeader(code int) {
	if !wrw.headerWritten {
		wrw.ResponseWriter.Header().Set("Content-Type", wrw.dupHeader.Get("Content-Type"))
		header := wrw.ResponseWriter.Header()
		// Filter out content length header to prevent stopping writing
		if wrw.dupHeader != nil {
			for k, v := range *wrw.dupHeader {
				if k == "Content-Length" {
					continue
				}
				header[k] = v
			}
		}

		wrw.headerWritten = true
	}
	wrw.ResponseWriter.WriteHeader(code)
}

// Header() copies the Header map from the underlying ResponseWriter to prevent
// modifications to it by callers
func (wrw *wrapResponseWriter) Header() http.Header {
	if wrw.dupHeader == nil {
		h := http.Header{}
		origHeader := wrw.ResponseWriter.Header()
		for k, v := range origHeader {
			h[k] = v
		}
		wrw.dupHeader = &h
	}
	return *wrw.dupHeader
}

const CHUNK_SIZE int = 512

// ServeHTTP implements an http.Handler that prefixes relative URLs from the
// Next handler with the configured prefix. It does this by examining the
// stream through the ResponseWriter, and appending the Prefix after any of the
// Attrs detected in the stream.
func (up *URLPrefixer) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	// chunked transfer because we're modifying the response on the fly, so we
	// won't know the final content-length
	rw.Header().Set("Connection", "Keep-Alive")
	rw.Header().Set("Transfer-Encoding", "chunked")

	writtenCount := 0 // number of bytes written to rw

	// extract the flusher for flushing chunks
	flusher, ok := rw.(http.Flusher)
	if !ok {
		up.Logger.
			WithField("component", "prefixer").
			Fatal("Expected http.ResponseWriter to be an http.Flusher, but wasn't")
	}

	nextRead, nextWrite := io.Pipe()
	go func() {
		defer nextWrite.Close()
		up.Next.ServeHTTP(&wrapResponseWriter{ResponseWriter: rw, Substitute: nextWrite}, r)
	}()

	// setup a buffer which is the max length of our target attrs
	b := make([]byte, up.maxlen(up.Attrs...))
	io.ReadFull(nextRead, b) // prime the buffer with the start of the input
	buf := bytes.NewBuffer(b)

	// Read next handler's response byte by byte
	src := bufio.NewScanner(nextRead)
	src.Split(bufio.ScanBytes)
	for {
		window := buf.Bytes()

		// advance a byte if window is not a src attr
		if matchlen, match := up.match(window, up.Attrs...); matchlen == 0 {
			if src.Scan() {
				// shift the next byte into buf
				rw.Write(buf.Next(1))
				writtenCount++
				buf.Write(src.Bytes())

				if writtenCount >= CHUNK_SIZE {
					flusher.Flush()
					writtenCount = 0
				}
			} else {
				if err := src.Err(); err != nil {
					up.Logger.
						WithField("component", "prefixer").
						Error("Error encountered while scanning: err:", err)
				}
				rw.Write(window)
				flusher.Flush()
				break
			}
			continue
		} else {
			buf.Next(matchlen) // advance to the relative URL
			for i := 0; i < matchlen; i++ {
				src.Scan()
				buf.Write(src.Bytes())
			}
			rw.Write(match)               // add the src attr to the output
			io.WriteString(rw, up.Prefix) // write the prefix
		}
	}
}

// match compares the subject against a list of targets. If there is a match
// between any of them a non-zero value is returned. The returned value is the
// length of the match. It is assumed that subject's length > length of all
// targets. The matching []byte is also returned as the second return parameter
func (up *URLPrefixer) match(subject []byte, targets ...[]byte) (int, []byte) {
	for _, target := range targets {
		if bytes.Equal(subject[:len(target)], target) {
			return len(target), target
		}
	}
	return 0, []byte{}
}

// maxlen returns the length of the largest []byte provided to it as an argument
func (up *URLPrefixer) maxlen(targets ...[]byte) int {
	max := 0
	for _, tgt := range targets {
		if tlen := len(tgt); tlen > max {
			max = tlen
		}
	}
	return max
}

// NewDefaultURLPrefixer returns a URLPrefixer that will prefix any src and
// href attributes found in HTML as well as any url() directives found in CSS
// with the provided prefix. Additionally, it will prefix any `data-basepath`
// attributes as well for informing front end logic about any prefixes. `next`
// is the next http.Handler that will have its output prefixed
func NewDefaultURLPrefixer(prefix string, next http.Handler, lg chronograf.Logger) *URLPrefixer {
	return &URLPrefixer{
		Prefix: prefix,
		Next:   next,
		Logger: lg,
		Attrs: [][]byte{
			[]byte(`src="`),
			[]byte(`href="`),
			[]byte(`url(`),
			[]byte(`data-basepath="`), // for forwarding basepath to frontend
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/lorserker/ballanceboard/logger"
	"github.com/lorserker/ballanceboard/sensor/wimu"
	"github.com/lorserker/ballanceboard/wsock"
)

func main() {
	sens := wimu.New(":5555")
	err := sens.Start()
	if err != nil {
		logger.Error().Println("error starting sensor", err)
		return
	}

	wsServer := wsock.New("localhost:8000", "/ws", sens.Accelerometer)
	wsServer.Start()
}
<commit_msg>Update accel.go<commit_after>package main

import (
	"github.com/lorserker/ballanceboard/logger"
	"github.com/lorserker/ballanceboard/sensor/wimu"
	"github.com/lorserker/ballanceboard/wsock"
)

func main() {
	sens := wimu.New(":5555")
	err := sens.Start()
	if err != nil {
		logger.Error().Println("error starting sensor", err)
		return
	}

	wsServer := wsock.New("0.0.0.0:8000", "/ws", sens.Accelerometer)
	wsServer.Start()
}
<|endoftext|>


<commit_before>package gorocksdb

// #include "rocksdb/c.h"
import "C"
import "unsafe"

// ReadTier controls fetching of data during a read request.
// An application can issue a read request (via Get/Iterators) and specify
// if that read should process data that ALREADY resides on a specified cache
// level. For example, if an application specifies BlockCacheTier then the
// Get call will process data that is already processed in the memtable or
// the block cache. It will not page in data from the OS cache or data that
// resides in storage.
type ReadTier uint

const (
	// ReadAllTier reads data in memtable, block cache, OS cache or storage.
	ReadAllTier = ReadTier(0)
	// BlockCacheTier reads data in memtable or block cache.
	BlockCacheTier = ReadTier(1)
)

// ReadOptions represent all of the available options when reading from a
// database.
type ReadOptions struct {
	c *C.rocksdb_readoptions_t
}

// NewDefaultReadOptions creates a default ReadOptions object.
func NewDefaultReadOptions() *ReadOptions {
	return NewNativeReadOptions(C.rocksdb_readoptions_create())
}

// NewNativeReadOptions creates a ReadOptions object.
func NewNativeReadOptions(c *C.rocksdb_readoptions_t) *ReadOptions {
	return &ReadOptions{c}
}

// UnsafeGetReadOptions returns the underlying c read options object.
func (opts *ReadOptions) UnsafeGetReadOptions() unsafe.Pointer {
	return unsafe.Pointer(opts.c)
}

// SetVerifyChecksums speciy if all data read from underlying storage will be
// verified against corresponding checksums.
// Default: false
func (opts *ReadOptions) SetVerifyChecksums(value bool) {
	C.rocksdb_readoptions_set_verify_checksums(opts.c, boolToChar(value))
}

// SetFillCache specify whether the "data block"/"index block"/"filter block"
// read for this iteration should be cached in memory?
// Callers may wish to set this field to false for bulk scans.
// Default: true
func (opts *ReadOptions) SetFillCache(value bool) {
	C.rocksdb_readoptions_set_fill_cache(opts.c, boolToChar(value))
}

// SetSnapshot sets the snapshot which should be used for the read.
// The snapshot must belong to the DB that is being read and must
// not have been released.
// Default: nil
func (opts *ReadOptions) SetSnapshot(snap *Snapshot) {
	C.rocksdb_readoptions_set_snapshot(opts.c, snap.c)
}

// SetReadTier specify if this read request should process data that ALREADY
// resides on a particular cache. If the required data is not
// found at the specified cache, then Status::Incomplete is returned.
// Default: ReadAllTier
func (opts *ReadOptions) SetReadTier(value ReadTier) {
	C.rocksdb_readoptions_set_read_tier(opts.c, C.int(value))
}

// SetTailing specify if to create a tailing iterator.
// A special iterator that has a view of the complete database
// (i.e. it can also be used to read newly added data) and
// is optimized for sequential reads. It will return records
// that were inserted into the database after the creation of the iterator.
// Default: false
func (opts *ReadOptions) SetTailing(value bool) {
	C.rocksdb_readoptions_set_tailing(opts.c, boolToChar(value))
}

// Destroy deallocates the ReadOptions object.
func (opts *ReadOptions) Destroy() {
	C.rocksdb_readoptions_destroy(opts.c)
	opts.c = nil
}
<commit_msg>Add SetIterateUpperBound & SetPinData to ReadOptions<commit_after>package gorocksdb

// #include "rocksdb/c.h"
import "C"
import "unsafe"

// ReadTier controls fetching of data during a read request.
// An application can issue a read request (via Get/Iterators) and specify
// if that read should process data that ALREADY resides on a specified cache
// level. For example, if an application specifies BlockCacheTier then the
// Get call will process data that is already processed in the memtable or
// the block cache. It will not page in data from the OS cache or data that
// resides in storage.
type ReadTier uint

const (
	// ReadAllTier reads data in memtable, block cache, OS cache or storage.
	ReadAllTier = ReadTier(0)
	// BlockCacheTier reads data in memtable or block cache.
	BlockCacheTier = ReadTier(1)
)

// ReadOptions represent all of the available options when reading from a
// database.
type ReadOptions struct {
	c *C.rocksdb_readoptions_t
}

// NewDefaultReadOptions creates a default ReadOptions object.
func NewDefaultReadOptions() *ReadOptions {
	return NewNativeReadOptions(C.rocksdb_readoptions_create())
}

// NewNativeReadOptions creates a ReadOptions object.
func NewNativeReadOptions(c *C.rocksdb_readoptions_t) *ReadOptions {
	return &ReadOptions{c}
}

// UnsafeGetReadOptions returns the underlying c read options object.
func (opts *ReadOptions) UnsafeGetReadOptions() unsafe.Pointer {
	return unsafe.Pointer(opts.c)
}

// SetVerifyChecksums speciy if all data read from underlying storage will be
// verified against corresponding checksums.
// Default: false
func (opts *ReadOptions) SetVerifyChecksums(value bool) {
	C.rocksdb_readoptions_set_verify_checksums(opts.c, boolToChar(value))
}

// SetFillCache specify whether the "data block"/"index block"/"filter block"
// read for this iteration should be cached in memory?
// Callers may wish to set this field to false for bulk scans.
// Default: true
func (opts *ReadOptions) SetFillCache(value bool) {
	C.rocksdb_readoptions_set_fill_cache(opts.c, boolToChar(value))
}

// SetSnapshot sets the snapshot which should be used for the read.
// The snapshot must belong to the DB that is being read and must
// not have been released.
// Default: nil
func (opts *ReadOptions) SetSnapshot(snap *Snapshot) {
	C.rocksdb_readoptions_set_snapshot(opts.c, snap.c)
}

// SetReadTier specify if this read request should process data that ALREADY
// resides on a particular cache. If the required data is not
// found at the specified cache, then Status::Incomplete is returned.
// Default: ReadAllTier
func (opts *ReadOptions) SetReadTier(value ReadTier) {
	C.rocksdb_readoptions_set_read_tier(opts.c, C.int(value))
}

// SetTailing specify if to create a tailing iterator.
// A special iterator that has a view of the complete database
// (i.e. it can also be used to read newly added data) and
// is optimized for sequential reads. It will return records
// that were inserted into the database after the creation of the iterator.
// Default: false
func (opts *ReadOptions) SetTailing(value bool) {
	C.rocksdb_readoptions_set_tailing(opts.c, boolToChar(value))
}

// SetIterateUpperBound specifies "iterate_upper_bound", which defines
// the extent upto which the forward iterator can returns entries.
// Once the bound is reached, Valid() will be false.
// "iterate_upper_bound" is exclusive ie the bound value is
// not a valid entry.  If iterator_extractor is not null, the Seek target
// and iterator_upper_bound need to have the same prefix.
// This is because ordering is not guaranteed outside of prefix domain.
// There is no lower bound on the iterator. If needed, that can be easily
// implemented.
// Default: nullptr
func (opts *ReadOptions) SetIterateUpperBound(key []byte) {
	cKey := byteToChar(key)
	cKeyLen := C.size_t(len(key))
	C.rocksdb_readoptions_set_iterate_upper_bound(opts.c, cKey, cKeyLen)
}

// SetPinData specifies the value of "pin_data". If true, it keeps the blocks
// loaded by the iterator pinned in memory as long as the iterator is not deleted,
// If used when reading from tables created with
// BlockBasedTableOptions::use_delta_encoding = false,
// Iterator's property "rocksdb.iterator.is-key-pinned" is guaranteed to
// return 1.
// Default: false
func (opts *ReadOptions) SetPinData(value bool) {
	C.rocksdb_readoptions_set_pin_data(opts.c, boolToChar(value))
}

// Destroy deallocates the ReadOptions object.
func (opts *ReadOptions) Destroy() {
	C.rocksdb_readoptions_destroy(opts.c)
	opts.c = nil
}
<|endoftext|>