<commit_before>package td

import (
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"math/rand"
	"os"
	"strconv"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/prometheus/common/model"
	"github.com/ugorji/go/codec"

	td_client "github.com/treasure-data/td-client-go"
)

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
}

type Config struct {
	apiKey string
	db     string
	table  string
	router td_client.EndpointRouter // for test
	port   int                      // for test
}

func ParseFlags(cfg *Config) *Config {
	flag.StringVar(&cfg.apiKey, "td.apikey", "", "The API Key for Treasure Data")
	flag.StringVar(&cfg.db, "td.db", "", "The Database Name for Treasure Data")
	flag.StringVar(&cfg.table, "td.table", "", "The Table Name for Treasure Data")
	return cfg
}

func validateConfig(cfg *Config) error {
	if cfg.apiKey == "" {
		return fmt.Errorf("td.apikey is required")
	}
	if cfg.db == "" {
		return fmt.Errorf("td.db is required")
	}
	if cfg.table == "" {
		return fmt.Errorf("td.table is required")
	}

	return nil
}

type Client struct {
	logger log.Logger

	client *td_client.TDClient
	db     string
	table  string
}

func NewClient(logger log.Logger, cfg *Config) *Client {
	if err := validateConfig(cfg); err != nil {
		level.Error(logger).Log("msg", "Failed to parse td options", "err", err)
		os.Exit(1)
	}

	settings := td_client.Settings{
		ApiKey: cfg.apiKey,
	}

	if cfg.router != nil {
		settings.Router = cfg.router
	}
	if cfg.port != 0 {
		settings.Port = cfg.port
	}

	c, err := td_client.NewTDClient(settings)

	if err != nil {
		level.Error(logger).Log("err", err)
		os.Exit(1)
	}

	if logger == nil {
		logger = log.NewNopLogger()
	}

	// TODO: validate db, table
	return &Client{
		logger: logger,
		client: c,
		db:     cfg.db,
		table:  cfg.table,
	}
}

func compressWithGzip(b []byte) []byte {
	retval := bytes.Buffer{}
	w := gzip.NewWriter(&retval)
	w.Write(b)
	w.Close()
	return retval.Bytes()
}

func generateUniqueId() string {
	now := time.Now().UTC()
	sec := now.Second()
	usec := now.UnixNano() / int64(time.Microsecond)
	u1 := ((sec*1000*1000 + int(usec)) << 12) | rand.Intn(0xfff)
	a := []uint32{uint32(u1) >> 32, uint32(u1) & 0xffffffff, uint32(rand.Intn(0xffffffff)), uint32(rand.Intn(0xffffffff))}

	var buffer bytes.Buffer
	for _, u := range a {
		buffer.WriteString(fmt.Sprintf("%02s", strconv.FormatUint(uint64(byte(u>>24)), 16)))
		buffer.WriteString(fmt.Sprintf("%02s", strconv.FormatUint(uint64(byte(u>>16)), 16)))
		buffer.WriteString(fmt.Sprintf("%02s", strconv.FormatUint(uint64(byte(u>>8)), 16)))
		buffer.WriteString(fmt.Sprintf("%02s", strconv.FormatUint(uint64(byte(u)), 16)))
	}
	return buffer.String()
}

func (c *Client) Write(samples model.Samples) error {
	data := bytes.Buffer{}
	handle := codec.MsgpackHandle{}
	encoder := codec.NewEncoder(&data, &handle)
	for _, s := range samples {
		record := map[string]interface{}{
			"time":  s.Timestamp.Unix(),
			"name":  string(s.Metric[model.MetricNameLabel]),
			"value": float64(s.Value),
		}
		for l, v := range s.Metric {
			if l != model.MetricNameLabel {
				record["label_"+string(l)] = string(v)
			}
		}

		encoder.Encode(record)
	}

	payload := compressWithGzip(data.Bytes())
	uniqueId := generateUniqueId()
	_, err := c.client.Import(c.db, c.table, "msgpack.gz", (td_client.InMemoryBlob)(payload), uniqueId)
	if err != nil {
		return err
	}

	level.Debug(c.logger).Log("msg", "wrote sample in td", "num_sumples", len(samples), "payload_size", len(payload))

	return nil
}

// Name identifies the client as an Treasure Data client.
func (c Client) Name() string {
	return "TreasureData"
}
<commit_msg>Enable set configs related td via env<commit_after>package td

import (
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"math/rand"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/prometheus/common/model"
	"github.com/ugorji/go/codec"

	td_client "github.com/treasure-data/td-client-go"
)

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
}

type Config struct {
	apiKey string
	db     string
	table  string
	router td_client.EndpointRouter // for test
	port   int                      // for test
}

func ParseFlags(cfg *Config) *Config {
	flag.StringVar(&cfg.apiKey, "td.apikey", "", "The API Key for Treasure Data")
	flag.StringVar(&cfg.db, "td.db", "", "The Database Name for Treasure Data")
	flag.StringVar(&cfg.table, "td.table", "", "The Table Name for Treasure Data")

	// Enable set config via enviroment variables
	flag.VisitAll(func(f *flag.Flag) {
		keys := []string{"td.apikey", "td.db", "td.table"}
		for _, key := range keys {
			if f.Name == key {
				env := strings.Replace(strings.ToUpper(key), ".", "_", -1)
				if s := os.Getenv(env); s != "" {
					f.Value.Set(s)
				}
			}
		}
	})
	return cfg
}

func validateConfig(cfg *Config) error {
	if cfg.apiKey == "" {
		return fmt.Errorf("td.apikey is required")
	}
	if cfg.db == "" {
		return fmt.Errorf("td.db is required")
	}
	if cfg.table == "" {
		return fmt.Errorf("td.table is required")
	}

	return nil
}

type Client struct {
	logger log.Logger

	client *td_client.TDClient
	db     string
	table  string
}

func NewClient(logger log.Logger, cfg *Config) *Client {
	if err := validateConfig(cfg); err != nil {
		level.Error(logger).Log("msg", "Failed to parse td options", "err", err)
		os.Exit(1)
	}

	settings := td_client.Settings{
		ApiKey: cfg.apiKey,
	}

	if cfg.router != nil {
		settings.Router = cfg.router
	}
	if cfg.port != 0 {
		settings.Port = cfg.port
	}

	c, err := td_client.NewTDClient(settings)

	if err != nil {
		level.Error(logger).Log("err", err)
		os.Exit(1)
	}

	if logger == nil {
		logger = log.NewNopLogger()
	}

	// TODO: validate db, table
	return &Client{
		logger: logger,
		client: c,
		db:     cfg.db,
		table:  cfg.table,
	}
}

func compressWithGzip(b []byte) []byte {
	retval := bytes.Buffer{}
	w := gzip.NewWriter(&retval)
	w.Write(b)
	w.Close()
	return retval.Bytes()
}

func generateUniqueId() string {
	now := time.Now().UTC()
	sec := now.Second()
	usec := now.UnixNano() / int64(time.Microsecond)
	u1 := ((sec*1000*1000 + int(usec)) << 12) | rand.Intn(0xfff)
	a := []uint32{uint32(u1) >> 32, uint32(u1) & 0xffffffff, uint32(rand.Intn(0xffffffff)), uint32(rand.Intn(0xffffffff))}

	var buffer bytes.Buffer
	for _, u := range a {
		buffer.WriteString(fmt.Sprintf("%02s", strconv.FormatUint(uint64(byte(u>>24)), 16)))
		buffer.WriteString(fmt.Sprintf("%02s", strconv.FormatUint(uint64(byte(u>>16)), 16)))
		buffer.WriteString(fmt.Sprintf("%02s", strconv.FormatUint(uint64(byte(u>>8)), 16)))
		buffer.WriteString(fmt.Sprintf("%02s", strconv.FormatUint(uint64(byte(u)), 16)))
	}
	return buffer.String()
}

func (c *Client) Write(samples model.Samples) error {
	data := bytes.Buffer{}
	handle := codec.MsgpackHandle{}
	encoder := codec.NewEncoder(&data, &handle)
	for _, s := range samples {
		record := map[string]interface{}{
			"time":  s.Timestamp.Unix(),
			"name":  string(s.Metric[model.MetricNameLabel]),
			"value": float64(s.Value),
		}
		for l, v := range s.Metric {
			if l != model.MetricNameLabel {
				record["label_"+string(l)] = string(v)
			}
		}

		encoder.Encode(record)
	}

	payload := compressWithGzip(data.Bytes())
	uniqueId := generateUniqueId()
	_, err := c.client.Import(c.db, c.table, "msgpack.gz", (td_client.InMemoryBlob)(payload), uniqueId)
	if err != nil {
		return err
	}

	level.Debug(c.logger).Log("msg", "wrote sample in td", "num_sumples", len(samples), "payload_size", len(payload))

	return nil
}

// Name identifies the client as an Treasure Data client.
func (c Client) Name() string {
	return "TreasureData"
}
<|endoftext|>


<commit_before>// Copyright 2009 Peter H. Froehlich. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package db

import "os"

// TODO: the new way of returning results from Execute()
type ResultSet interface {
	Iter() <-chan Result;
	Close() os.Error;
}
<commit_msg>Added documentation.<commit_after>// Copyright 2009 Peter H. Froehlich. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package db

import "os"

// The simplest way of processing query results.
//
// Iter() returns a channel of Result objects which can be
// examined one at a time. Note that called Iter() several
// times will return the same channel over and over again.
//
// Close() shuts down the iteration mechanism and frees all
// associated resources. After a result set has been closed,
// Iter() will return nil.
type ResultSet interface {
	Iter() <-chan Result;
	Close() os.Error;
}
<|endoftext|>


<commit_before>package discrete

import ()

// On one hand, using an interface{} as a key works on some levels.
// On the other hand, from experience, I can say that working with interface{} is a pain
// so I don't like it in an API. An alternate idea is to make Set an interface with a method that allows you to GRAB a map[interface{}]struct{} from
// the implementation, but that adds a lot of calls and needless operations, making the library slower
//
// Another point, using an interface{} may be pointless because a map key MUST have == and != defined, limiting the possible keys anyway (for instance, if you had a set of [3]floats I don't think it will do a deep
// comparison, making it rather pointless). Also, keying with a float will mean it does a strict == with the floats, possibly causing bad behavior. It may be best to just make it a map[int]struct{}. Thoughts?
type Set map[interface{}]struct{}

// I highly doubt we have to worry about running out of IDs, but we could add a little reclaimID function if we're worried
var globalid uint64 = 0

// For cleanliness
var flag struct{} = struct{}{}

func NewSet() *Set {
	s := make(Set)
	return &s
}

func (s1 *Set) Clear() *Set {
	if len(*s1) == 0 {
		return s1
	}

	(*s1) = make(Set)

	return s1
}

// Ensures a perfect copy from s1 to dst (meaning the sets will be equal)
func (dst *Set) Copy(src *Set) *Set {
	if src == dst {
		return dst
	}

	if len(*dst) > 0 {
		*(dst) = *NewSet()
	}

	for el := range *src {
		(*dst)[el] = flag
	}

	return dst
}

func (s1 *Set) Equal(s2 *Set) bool {
	if s1 == s2 {
		return true
	} else if len(*s1) != len(*s2) {
		return false
	}

	for el := range *s1 {
		if _, ok := (*s2)[el]; !ok {
			return false
		}
	}

	return true
}

func (dst *Set) Union(s1, s2 *Set) *Set {
	if s1 == s2 {
		return dst.Copy(s1)
	}

	if s1 != dst && s2 != dst {
		dst.Clear()
	}

	if dst != s1 {
		for el := range *s1 {
			(*dst)[el] = flag
		}
	}

	if dst != s2 {
		for el := range *s2 {
			(*dst)[el] = flag
		}
	}

	return dst
}

func (dst *Set) Intersection(s1, s2 *Set) *Set {
	var swap *Set

	if s1 == s2 {
		return dst.Copy(s1)
	} else if s1 == dst {
		swap = s2
	} else if s2 == dst {
		swap = s1
	} else {
		dst.Clear()

		if len(*s1) > len(*s2) {
			s1, s2 = s2, s1
		}
		for el := range *s1 {
			if _, ok := (*s2)[el]; ok {
				(*dst)[el] = flag
			}
		}

		return dst
	}

	for el := range *dst {
		if _, ok := (*swap)[el]; !ok {
			delete(*dst, el)
		}
	}

	return dst
}

func (dst *Set) Diff(s1, s2 *Set) *Set {
	if s1 == s2 {
		return dst.Clear()
	} else if s2 == dst {
		tmp := NewSet()

		return dst.Copy(tmp.Diff(s1, s2))
	} else if s1 == dst {
		for el := range *dst {
			if _, ok := (*s2)[el]; ok {
				delete(*dst, el)
			}
		}

	} else {
		dst.Clear()
		for el := range *s1 {
			if _, ok := (*s2)[el]; !ok {
				(*dst)[el] = flag
			}
		}
	}

	return dst
}

func (s *Set) Contains(el interface{}) bool {
	_, ok := (*s)[el]
	return ok
}

// Are Add/Remove necessary?
func (s1 *Set) Add(element interface{}) {
	(*s1)[element] = flag
}

func (s1 *Set) Remove(element interface{}) {
	delete(*s1, element)
}

func (s1 *Set) Cardinality() int {
	return len(*s1)
}

/* Should probably be re-implemented as a tree later */
type DisjointSet struct {
	master  *Set
	subsets []*Set
}

func NewDisjointSet() *DisjointSet {
	return &DisjointSet{NewSet(), make([]*Set, 0)}
}

func (ds *DisjointSet) MasterSet() *Set {
	return ds.master
}

func (ds *DisjointSet) MakeSet(el interface{}) {
	if ds.master.Contains(el) {
		return
	}
	ds.master.Add(el)
	ns := NewSet()
	ns.Add(el)
	ds.subsets = append(ds.subsets, ns)
}

func (ds *DisjointSet) Find(el interface{}) *Set {
	if !ds.master.Contains(el) {
		return nil
	}

	for _, subset := range ds.subsets {
		if subset.Contains(el) {
			return subset
		}
	}

	return nil
}

func (ds *DisjointSet) Union(s1, s2 *Set) {
	if s1 == s2 {
		return
	}
	s1Found, s2Found := false, false

	newSubsetList := make([]*Set, 0, len(ds.subsets)-1)

	for _, subset := range ds.subsets {
		if s1 == subset {
			s1Found = true
			continue
		} else if s2 == subset {
			s2Found = true
			continue
		}

		newSubsetList = append(newSubsetList, subset)
	}

	if s1Found && s2Found {
		newSubsetList = append(newSubsetList, s1.Union(s1, s2))
		ds.subsets = newSubsetList
	}
}
<commit_msg>Documentation<commit_after>package discrete

import ()

// On one hand, using an interface{} as a key works on some levels.
// On the other hand, from experience, I can say that working with interface{} is a pain
// so I don't like it in an API. An alternate idea is to make Set an interface with a method that allows you to GRAB a map[interface{}]struct{} from
// the implementation, but that adds a lot of calls and needless operations, making the library slower
//
// Another point, using an interface{} may be pointless because a map key MUST have == and != defined, limiting the possible keys anyway (for instance, if you had a set of [3]floats I don't think it will do a deep
// comparison, making it rather pointless). Also, keying with a float will mean it does a strict == with the floats, possibly causing bad behavior. It may be best to just make it a map[int]struct{}. Thoughts?
type Set map[interface{}]struct{}

// I highly doubt we have to worry about running out of IDs, but we could add a little reclaimID function if we're worried
var globalid uint64 = 0

// For cleanliness
var flag struct{} = struct{}{}

func NewSet() *Set {
	s := make(Set)
	return &s
}

func (s1 *Set) Clear() *Set {
	if len(*s1) == 0 {
		return s1
	}

	(*s1) = make(Set)

	return s1
}

// Ensures a perfect copy from s1 to dst (meaning the sets will be equal)
func (dst *Set) Copy(src *Set) *Set {
	if src == dst {
		return dst
	}

	if len(*dst) > 0 {
		*(dst) = *NewSet()
	}

	for el := range *src {
		(*dst)[el] = flag
	}

	return dst
}

func (s1 *Set) Equal(s2 *Set) bool {
	if s1 == s2 {
		return true
	} else if len(*s1) != len(*s2) {
		return false
	}

	for el := range *s1 {
		if _, ok := (*s2)[el]; !ok {
			return false
		}
	}

	return true
}

func (dst *Set) Union(s1, s2 *Set) *Set {
	if s1 == s2 {
		return dst.Copy(s1)
	}

	if s1 != dst && s2 != dst {
		dst.Clear()
	}

	if dst != s1 {
		for el := range *s1 {
			(*dst)[el] = flag
		}
	}

	if dst != s2 {
		for el := range *s2 {
			(*dst)[el] = flag
		}
	}

	return dst
}

func (dst *Set) Intersection(s1, s2 *Set) *Set {
	var swap *Set

	if s1 == s2 {
		return dst.Copy(s1)
	} else if s1 == dst {
		swap = s2
	} else if s2 == dst {
		swap = s1
	} else {
		dst.Clear()

		if len(*s1) > len(*s2) {
			s1, s2 = s2, s1
		}
		for el := range *s1 {
			if _, ok := (*s2)[el]; ok {
				(*dst)[el] = flag
			}
		}

		return dst
	}

	for el := range *dst {
		if _, ok := (*swap)[el]; !ok {
			delete(*dst, el)
		}
	}

	return dst
}

func (dst *Set) Diff(s1, s2 *Set) *Set {
	if s1 == s2 {
		return dst.Clear()
	} else if s2 == dst {
		tmp := NewSet()

		return dst.Copy(tmp.Diff(s1, s2))
	} else if s1 == dst {
		for el := range *dst {
			if _, ok := (*s2)[el]; ok {
				delete(*dst, el)
			}
		}

	} else {
		dst.Clear()
		for el := range *s1 {
			if _, ok := (*s2)[el]; !ok {
				(*dst)[el] = flag
			}
		}
	}

	return dst
}

func (s *Set) Contains(el interface{}) bool {
	_, ok := (*s)[el]
	return ok
}

// Are Add/Remove necessary?
func (s1 *Set) Add(element interface{}) {
	(*s1)[element] = flag
}

func (s1 *Set) Remove(element interface{}) {
	delete(*s1, element)
}

func (s1 *Set) Cardinality() int {
	return len(*s1)
}

/* Should probably be re-implemented as a tree later */
type DisjointSet struct {
	master  *Set
	subsets []*Set
}

func NewDisjointSet() *DisjointSet {
	return &DisjointSet{NewSet(), make([]*Set, 0)}
}

func (ds *DisjointSet) MasterSet() *Set {
	return ds.master
}

// If the element isn't already somewhere in there, adds it to the master set and its own tiny set
func (ds *DisjointSet) MakeSet(el interface{}) {
	if ds.master.Contains(el) {
		return
	}
	ds.master.Add(el)
	ns := NewSet()
	ns.Add(el)
	ds.subsets = append(ds.subsets, ns)
}

// Returns the set the element belongs to, or nil if none
func (ds *DisjointSet) Find(el interface{}) *Set {
	if !ds.master.Contains(el) {
		return nil
	}

	for _, subset := range ds.subsets {
		if subset.Contains(el) {
			return subset
		}
	}

	return nil
}

// Unions two subsets within the DisjointSet
func (ds *DisjointSet) Union(s1, s2 *Set) {
	if s1 == s2 {
		return
	}
	s1Found, s2Found := false, false

	newSubsetList := make([]*Set, 0, len(ds.subsets)-1)

	for _, subset := range ds.subsets {
		if s1 == subset {
			s1Found = true
			continue
		} else if s2 == subset {
			s2Found = true
			continue
		}

		newSubsetList = append(newSubsetList, subset)
	}

	if s1Found && s2Found {
		newSubsetList = append(newSubsetList, s1.Union(s1, s2))
		ds.subsets = newSubsetList
	}
}
<|endoftext|>


<commit_before>package hob

import ()

type SetData map[string]bool

type Set struct {
	setData SetData
	Set     []string
}

func NewSet() (set *Set) {
	return &Set{
		setData: newSetData(),
		Set:     newSetSlice(),
	}
}

func newSetData() (setData SetData) {
	setData = make(SetData)
	return
}

func newSetSlice() (setSlice []string) {
	setSlice = make([]string, 0)
	return
}

func (set *Set) Add(value string) (ok bool) {
	set.setData[value] = true
	ok = true
	return
}

func (set *Set) Remove(value string) (ok bool) {
	delete(set.setData, value)
	ok = true
	return
}

func (set *Set) Test(value string) (ok bool) {
	ok = set.setData[value]
	return
}

func (set *Set) Clone() (clone *Set) {
	clone = &Set{
		setData: set.setData,
		Set:     set.Set,
	}
	return
}

func (set *Set) Union(oset *Set) (union *Set) {
	union = set.Clone()
	for value, _ := range oset.setData {
		union.setData[value] = true
	}
	return
}

func (set *Set) Intersection(oset *Set) (intersection *Set) {
	intersection = NewSet()

	if len(set.setData) > len(oset.setData) {
		for value, _ := range oset.setData {
			if ok := set.setData[value]; ok {
				intersection.setData[value] = true
			}
		}
	} else {
		for value, _ := range set.setData {
			if ok := oset.setData[value]; ok {
				intersection.setData[value] = true
			}
		}
	}

	return
}

func (set *Set) Slice() []string {
	set.Set = newSetSlice()

	for value, ok := range set.setData {
		if ok {
			set.Set = append(set.Set, value)
		}
	}

	return set.Set
}
<commit_msg>One loop<commit_after>package hob

import ()

type SetData map[string]bool

type Set struct {
	setData SetData
	Set     []string
}

func NewSet() (set *Set) {
	return &Set{
		setData: newSetData(),
		Set:     newSetSlice(),
	}
}

func newSetData() (setData SetData) {
	setData = make(SetData)
	return
}

func newSetSlice() (setSlice []string) {
	setSlice = make([]string, 0)
	return
}

func (set *Set) Add(value string) (ok bool) {
	set.setData[value] = true
	ok = true
	return
}

func (set *Set) Remove(value string) (ok bool) {
	delete(set.setData, value)
	ok = true
	return
}

func (set *Set) Test(value string) (ok bool) {
	ok = set.setData[value]
	return
}

func (set *Set) Clone() (clone *Set) {
	clone = &Set{
		setData: set.setData,
		Set:     set.Set,
	}
	return
}

func (set *Set) Union(oset *Set) (union *Set) {
	union = set.Clone()
	for value, _ := range oset.setData {
		union.setData[value] = true
	}
	return
}

func (set *Set) Intersection(oset *Set) (intersection *Set) {
	intersection = NewSet()

	var shorterSet, longerSet *Set

	if len(set.setData) > len(oset.setData) {
		shorterSet = oset
		longerSet = set
	} else {
		shorterSet = set
		longerSet = oset
	}

	for value, _ := range shorterSet.setData {
		if ok := longerSet.setData[value]; ok {
			intersection.setData[value] = true
		}
	}

	return
}

func (set *Set) Slice() []string {
	set.Set = newSetSlice()

	for value, ok := range set.setData {
		if ok {
			set.Set = append(set.Set, value)
		}
	}

	return set.Set
}
<|endoftext|>


<commit_before>package json

import (
	"io"
	"strconv"
	"time"

	"github.com/segmentio/objconv"
)

var (
	nullBytes  = [...]byte{'n', 'u', 'l', 'l'}
	trueBytes  = [...]byte{'t', 'r', 'u', 'e'}
	falseBytes = [...]byte{'f', 'a', 'l', 's', 'e'}

	arrayOpen  = [...]byte{'['}
	arrayClose = [...]byte{']'}

	mapOpen  = [...]byte{'{'}
	mapClose = [...]byte{'}'}

	comma  = [...]byte{','}
	column = [...]byte{':'}
)

// Emitter implements a JSON emitter that satisfies the objconv.Emitter
// interface.
type Emitter struct {
	w io.Writer
	s []byte
	b [128]byte
}

func NewEmitter(w io.Writer) *Emitter {
	return &Emitter{w: w}
}

func (e *Emitter) Reset(w io.Writer) {
	e.w = w
}

func (e *Emitter) EmitNil() (err error) {
	_, err = e.w.Write(nullBytes[:])
	return
}

func (e *Emitter) EmitBool(v bool) (err error) {
	if v {
		_, err = e.w.Write(trueBytes[:])
	} else {
		_, err = e.w.Write(falseBytes[:])
	}
	return
}

func (e *Emitter) EmitInt(v int64) (err error) {
	_, err = e.w.Write(strconv.AppendInt(e.b[:0], v, 10))
	return
}

func (e *Emitter) EmitUint(v uint64) (err error) {
	_, err = e.w.Write(strconv.AppendUint(e.b[:0], v, 10))
	return
}

func (e *Emitter) EmitFloat(v float64) (err error) {
	_, err = e.w.Write(strconv.AppendFloat(e.b[:0], v, 'g', -1, 64))
	return
}

func (e *Emitter) EmitString(v string) (err error) {
	i := 0
	j := 0
	n := len(v)

	if e.s == nil {
		e.s = e.b[:0]
	}

	s := e.s[:0]
	s = append(s, '"')

	for j != n {
		b := v[j]
		j++

		switch b {
		case '"', '\\', '/':
			// b = b

		case '\b':
			b = 'b'

		case '\f':
			b = 'f'

		case '\n':
			b = 'n'

		case '\r':
			b = 'r'

		case '\t':
			b = 't'

		default:
			continue
		}

		s = append(s, v[i:j-1]...)
		s = append(s, '\\', b)
		i = j
	}

	s = append(s, v[i:j]...)
	s = append(s, '"')
	e.s = s[:0] // in case the buffer was reallocated

	_, err = e.w.Write(s)
	return
}

func (e *Emitter) EmitBytes(v []byte) error {
	return e.EmitString(string(v))
}

func (e *Emitter) EmitTime(v time.Time) error {
	return e.EmitString(string(v.AppendFormat(e.b[:0], time.RFC3339Nano)))
}

func (e *Emitter) EmitDuration(v time.Duration) error {
	return e.EmitString(string(objconv.AppendDuration(e.b[:0], v)))
}

func (e *Emitter) EmitError(v error) error {
	return e.EmitString(v.Error())
}

func (e *Emitter) EmitArrayBegin(_ int) (err error) {
	_, err = e.w.Write(arrayOpen[:])
	return
}

func (e *Emitter) EmitArrayEnd() (err error) {
	_, err = e.w.Write(arrayClose[:])
	return
}

func (e *Emitter) EmitArrayNext() (err error) {
	_, err = e.w.Write(comma[:])
	return
}

func (e *Emitter) EmitMapBegin(_ int) (err error) {
	_, err = e.w.Write(mapOpen[:])
	return
}

func (e *Emitter) EmitMapEnd() (err error) {
	_, err = e.w.Write(mapClose[:])
	return
}

func (e *Emitter) EmitMapValue() (err error) {
	_, err = e.w.Write(column[:])
	return
}

func (e *Emitter) EmitMapNext() (err error) {
	_, err = e.w.Write(comma[:])
	return
}
<commit_msg>optimize json encoder<commit_after>package json

import (
	"io"
	"strconv"
	"time"

	"github.com/segmentio/objconv"
)

var (
	nullBytes  = [...]byte{'n', 'u', 'l', 'l'}
	trueBytes  = [...]byte{'t', 'r', 'u', 'e'}
	falseBytes = [...]byte{'f', 'a', 'l', 's', 'e'}

	arrayOpen  = [...]byte{'['}
	arrayClose = [...]byte{']'}

	mapOpen  = [...]byte{'{'}
	mapClose = [...]byte{'}'}

	comma  = [...]byte{','}
	column = [...]byte{':'}
)

// Emitter implements a JSON emitter that satisfies the objconv.Emitter
// interface.
type Emitter struct {
	w io.Writer
	s []byte
	b [128]byte
}

func NewEmitter(w io.Writer) *Emitter {
	e := &Emitter{w: w}
	e.s = e.b[:0]
	return e
}

func (e *Emitter) Reset(w io.Writer) {
	e.w = w
}

func (e *Emitter) EmitNil() (err error) {
	_, err = e.w.Write(nullBytes[:])
	return
}

func (e *Emitter) EmitBool(v bool) (err error) {
	if v {
		_, err = e.w.Write(trueBytes[:])
	} else {
		_, err = e.w.Write(falseBytes[:])
	}
	return
}

func (e *Emitter) EmitInt(v int64) (err error) {
	_, err = e.w.Write(strconv.AppendInt(e.b[:0], v, 10))
	return
}

func (e *Emitter) EmitUint(v uint64) (err error) {
	_, err = e.w.Write(strconv.AppendUint(e.b[:0], v, 10))
	return
}

func (e *Emitter) EmitFloat(v float64) (err error) {
	_, err = e.w.Write(strconv.AppendFloat(e.b[:0], v, 'g', -1, 64))
	return
}

func (e *Emitter) EmitString(v string) (err error) {
	i := 0
	j := 0
	n := len(v)
	s := append(e.s[:0], '"')

	for j != n {
		b := v[j]
		j++

		switch b {
		case '"', '\\', '/':
			// b = b

		case '\b':
			b = 'b'

		case '\f':
			b = 'f'

		case '\n':
			b = 'n'

		case '\r':
			b = 'r'

		case '\t':
			b = 't'

		default:
			continue
		}

		s = append(s, v[i:j-1]...)
		s = append(s, '\\', b)
		i = j
	}

	s = append(s, v[i:j]...)
	s = append(s, '"')
	e.s = s[:0] // in case the buffer was reallocated

	_, err = e.w.Write(s)
	return
}

func (e *Emitter) EmitBytes(v []byte) error {
	return e.EmitString(string(v))
}

func (e *Emitter) EmitTime(v time.Time) error {
	return e.EmitString(string(v.AppendFormat(e.b[:0], time.RFC3339Nano)))
}

func (e *Emitter) EmitDuration(v time.Duration) error {
	return e.EmitString(string(objconv.AppendDuration(e.b[:0], v)))
}

func (e *Emitter) EmitError(v error) error {
	return e.EmitString(v.Error())
}

func (e *Emitter) EmitArrayBegin(_ int) (err error) {
	_, err = e.w.Write(arrayOpen[:])
	return
}

func (e *Emitter) EmitArrayEnd() (err error) {
	_, err = e.w.Write(arrayClose[:])
	return
}

func (e *Emitter) EmitArrayNext() (err error) {
	_, err = e.w.Write(comma[:])
	return
}

func (e *Emitter) EmitMapBegin(_ int) (err error) {
	_, err = e.w.Write(mapOpen[:])
	return
}

func (e *Emitter) EmitMapEnd() (err error) {
	_, err = e.w.Write(mapClose[:])
	return
}

func (e *Emitter) EmitMapValue() (err error) {
	_, err = e.w.Write(column[:])
	return
}

func (e *Emitter) EmitMapNext() (err error) {
	_, err = e.w.Write(comma[:])
	return
}
<|endoftext|>


<commit_before>package config

import (
	"path"

	cmn "github.com/tendermint/tmlibs/common"
)

/****** these are for production settings ***********/
func EnsureRoot(rootDir string, network string) {
	cmn.EnsureDir(rootDir, 0700)
	cmn.EnsureDir(rootDir+"/data", 0700)

	configFilePath := path.Join(rootDir, "config.toml")

	// Write default config file if missing.
	if !cmn.FileExists(configFilePath) {
		cmn.MustWriteFile(configFilePath, []byte(selectNetwork(network)), 0644)
	}
}

var defaultConfigTmpl = `# This is a TOML config file.
# For more information, see https://github.com/toml-lang/toml
fast_sync = true
db_backend = "leveldb"
api_addr = "0.0.0.0:9888"
`

var mainNetConfigTmpl = `chain_id = "mainnet"
[p2p]
laddr = "tcp://0.0.0.0:46656"
seeds = ""
`

var testNetConfigTmpl = `chain_id = "testnet"
[p2p]
laddr = "tcp://0.0.0.0:46656"
seeds = "172.104.224.219:46656,198.74.61.131:46656,212.111.41.245:46656"
`

// Select network seeds to merge a new string.
func selectNetwork(network string) string {
	if network == "testnet" {
		return defaultConfigTmpl + testNetConfigTmpl
	} else {
		return defaultConfigTmpl + mainNetConfigTmpl
	}
}
<commit_msg>Add mainnet seeds<commit_after>package config

import (
	"path"

	cmn "github.com/tendermint/tmlibs/common"
)

/****** these are for production settings ***********/
func EnsureRoot(rootDir string, network string) {
	cmn.EnsureDir(rootDir, 0700)
	cmn.EnsureDir(rootDir+"/data", 0700)

	configFilePath := path.Join(rootDir, "config.toml")

	// Write default config file if missing.
	if !cmn.FileExists(configFilePath) {
		cmn.MustWriteFile(configFilePath, []byte(selectNetwork(network)), 0644)
	}
}

var defaultConfigTmpl = `# This is a TOML config file.
# For more information, see https://github.com/toml-lang/toml
fast_sync = true
db_backend = "leveldb"
api_addr = "0.0.0.0:9888"
`

var mainNetConfigTmpl = `chain_id = "mainnet"
[p2p]
laddr = "tcp://0.0.0.0:46657"
seeds = "45.79.213.28:46657,198.74.61.131:46657,212.111.41.245:46657,47.100.214.154:46657,47.100.109.199:46657,47.100.105.165:46657"
`

var testNetConfigTmpl = `chain_id = "testnet"
[p2p]
laddr = "tcp://0.0.0.0:46656"
seeds = "47.96.42.1:46656,172.104.224.219:46656,45.118.132.164:46656"
`

// Select network seeds to merge a new string.
func selectNetwork(network string) string {
	if network == "testnet" {
		return defaultConfigTmpl + testNetConfigTmpl
	} else {
		return defaultConfigTmpl + mainNetConfigTmpl
	}
}
<|endoftext|>


<commit_before>package contnet

import (
	"github.com/asaskevich/EventBus"
	"log"
	"math/rand"
	"sync"
	"time"
)

func init() {
	rand.Seed(time.Now().Unix())
}

type NetConfig struct {
	MaxContentAge           time.Duration
	CheckContentAgeInterval time.Duration
	ItemsPerPage            uint8
	NoveltyPct              float64
	SnapshotPath            string
	SnapshotInterval        time.Duration
}

type Net struct {
	sync.RWMutex
	config       *NetConfig
	bus          *EventBus.EventBus
	contentStore *ContentStore
	profileStore *ProfileStore
	trendStore   *TrendStore
	index        *Index
}
type NetFactory struct{}

func (factory NetFactory) New(config *NetConfig) *Net {
	bus := EventBus.New()
	contentStore := Object.ContentStore.New(config, bus)
	net := &Net{
		config:       config,
		bus:          bus,
		contentStore: contentStore,
		profileStore: Object.ProfileStore.New(),
		trendStore:   Object.TrendStore.New(bus),
		index:        Object.Index.New(config, bus, contentStore),
	}
	if err := net.Restore(); err != nil {
		log.Print("Failed to restore net object, proceeding empty.")
	}
	go net.__snapshot()
	go net.index.__refresh()

	return net
}

func (net *Net) __snapshot() {
	for {
		net.Snapshot()
		time.Sleep(net.config.SnapshotInterval)
	}
}

func (net *Net) Snapshot() error {
	if err := net.contentStore.Snapshot(net.config.SnapshotPath, "content"); err != nil {
		return err
	}

	if err := net.index.Snapshot(net.config.SnapshotPath, "index"); err != nil {
		return err
	}

	if err := net.profileStore.Snapshot(net.config.SnapshotPath, "profiles"); err != nil {
		return err
	}

	if err := net.trendStore.Snapshot(net.config.SnapshotPath, "trends"); err != nil {
		return err
	}

	return nil
}

func (net *Net) Restore() error {
	if err := net.contentStore.RestoreFromSnapshot(net.config.SnapshotPath, "content"); err != nil {
		return err
	}

	if err := net.index.RestoreFromSnapshot(net.config.SnapshotPath, "index"); err != nil {
		return err
	}

	if err := net.profileStore.RestoreFromSnapshot(net.config.SnapshotPath, "profiles"); err != nil {
		return err
	}

	if err := net.trendStore.RestoreFromSnapshot(net.config.SnapshotPath, "trends"); err != nil {
		return err
	}

	return nil
}

// Attempts to update network object with content specified.
// If content did not exist, it is added to the network.
// If content did exist, it is updated.
func (net *Net) SaveContent(content *Content) {
	net.contentStore.Upsert(content)
}

// Attempts to update network object with action for profile specified.
func (net *Net) SaveAction(action *Action) error {
	// get related content's copy, if any
	relatedContent := net.contentStore.Get(action.ContentID)

	// if content does not exist
	if relatedContent == nil {
		return Errors.ContentNotFound
	}

	// inject related content to action
	action.Content = relatedContent

	// save action to profile
	net.profileStore.Save(action)

	// everything is okay
	return nil
}

type SelectionCacheInfo struct {
	CurrentIndex int
	MaximumIndex int
}

func (net *Net) Select(profileID int64, page uint8) []ID {
	// get user's profile, if any
	profile := net.profileStore.Get(ID(profileID))

	// assign topic interests & sort
	var interests TopicInterests
	if profile != nil {
		TopicInterestBy(topicInterestCriteria).Sort(profile.TopicInterests)
		interests = profile.TopicInterests
	} else {
		interests = TopicInterests{}
	}

	totalContentsToRetrieve := int(page) * int(net.config.ItemsPerPage)
	totalContentsByInterest := int((1 - net.config.NoveltyPct) * float64(totalContentsToRetrieve))

	// select interesting contents
	interestingContents := net.__selectOfInterest(interests, totalContentsByInterest)
	//log.Println("Interesting: ", len(interestingContents))

	// select popular contents (how many remaining / 2)
	totalContentsByTrend := int((totalContentsToRetrieve - len(interestingContents)) / 2)
	trendingContents := net.__selectOfTrending(interestingContents, totalContentsByTrend)
	//log.Println("Trending: ", len(trendingContents))

	// fill the remainder randomly
	totalContentsRemaining := totalContentsToRetrieve - len(interestingContents) - len(trendingContents)
	out := append(interestingContents, trendingContents...)
	remainingContents := net.__selectOfRemaining(out, totalContentsRemaining)
	//log.Println("Remaining: ", len(remainingContents))

	// prep the final result
	out = append(out, remainingContents...)
	return out
}

func (net *Net) __selectOfInterest(interests TopicInterests, howMany int) []ID {
	// abort early if nothing to select..
	if len(interests) == 0 {
		return []ID{}
	}

	// we have sorted topic interests for this user.
	// now we extract topics from topic-interest objects for querying the index
	// and we use dem iterations to form cumulative probabilities slice to be used later for randomly drawing topics
	interestTopics := Topics{}
	cumulativeProbabilities := []float64{}
	for i := 0; i < len(interests); i++ {
		interestTopics = append(interestTopics, &interests[i].Topic)
		cumulativeProbabilities = append(cumulativeProbabilities, float64(interests[i].Interest))
		if i > 0 {
			cumulativeProbabilities[i] += cumulativeProbabilities[i-1]
		}
	}

	// use the previously created topics slice for querying the index
	topicContents := net.index.GetForTopics(interestTopics)

	// now we have all content IDs for all topics interesting to this user
	// to control where we're at, we will create temporary cache that will mark extraction location for each topic
	cache := map[Topic]*SelectionCacheInfo{}
	// filling in the cache
	for i := 0; i < len(interests); i++ {
		interest := interests[i]
		cache[interest.Topic] = &SelectionCacheInfo{
			CurrentIndex: 0,
			MaximumIndex: len(topicContents[i]) - 1,
		}
	}

	// now we have the following data available:
	// 1. topics of interest for user sorted in descending order by how interesting they are
	// 2. for each topic of interest there is an array of ID's available for that topic
	// 3. map showing what index to choose for each topic
	// 4. cumulative probabilities

	// while there is anything interesting for user available & we haven't selected as many items as needed
	out := []ID{}
	for len(cache) > 0 && len(out) < howMany {
		// select a random topic of interest
		topic, i := __drawRandomTopicInterest(interests, cumulativeProbabilities)

		// select its best content (first index available)
		indexToSelect := cache[topic].CurrentIndex

		// current index can be ahead of the max index in some special cases
		if indexToSelect <= cache[topic].MaximumIndex {
			selectedID := topicContents[i][indexToSelect]
			if !__idsContainID(out, selectedID) {
				out = append(out, selectedID)
			}
		}

		// advance current index in cache
		cache[topic].CurrentIndex++

		// if we overshot the maximum index for this topic, remove it
		if cache[topic].CurrentIndex > cache[topic].MaximumIndex {
			// remove from cache
			delete(cache, topic)
			// remove from interest topics
			interestTopics = append(interestTopics[:i], interestTopics[i+1:]...)
			// remove from topic contents
			topicContents = append(topicContents[:i], topicContents[i+1:]...)
			// remove from interests
			interests = append(interests[:i], interests[i+1:]...)
			// recalculate interests
			cumulativeProbabilities = __recalculateCumulativeProbabilities(interests)
		}
	}

	return out
}

func __idsContainID(ids []ID, id ID) bool {
	for i := 0; i < len(ids); i++ {
		if ids[i] == id {
			return true
		}
	}
	return false
}

func __drawRandomTopicInterest(topicInterests TopicInterests, cumulativeProbabilities []float64) (Topic, int) {
	randomFloat := rand.Float64()
	for i := 0; i < len(cumulativeProbabilities); i++ {
		if cumulativeProbabilities[i] >= randomFloat {
			return topicInterests[i].Topic, i
		}
	}
	// this is guaranteed not to happen
	return topicInterests[len(topicInterests)-1].Topic, len(topicInterests) - 1
}

func __recalculateCumulativeProbabilities(topicInterests TopicInterests) []float64 {
	// calculate sum of cumulative interests
	sum := float64(0)
	for i := 0; i < len(topicInterests); i++ {
		sum += float64(topicInterests[i].CumulativeInterest)
	}

	output := []float64{}

	for i := 0; i < len(topicInterests); i++ {
		output = append(output, float64(topicInterests[i].CumulativeInterest)/sum)
		if i > 0 {
			output[i] += output[i-1]
		}
	}

	return output
}

func (net *Net) __selectOfTrending(ignore []ID, howMany int) []ID {
	// select top 10 trending topics
	trendingTopics := net.trendStore.GetTopN(10)

	// nothing trending, bail early
	if len(trendingTopics) == 0 {
		return []ID{}
	}

	// get content IDs for trending topics
	contentIDs := net.index.GetForTopics(trendingTopics)

	// cache indices foreach contentID
	cache := map[Topic]*SelectionCacheInfo{}
	// filling in the cache
	for i := 0; i < len(trendingTopics); i++ {
		cache[*trendingTopics[i]] = &SelectionCacheInfo{
			CurrentIndex: 0,
			MaximumIndex: len(contentIDs[i]) - 1,
		}
	}

	nextTrendInd := 0
	out := []ID{}
	for len(cache) > 0 && len(out) < howMany {
		trendingTopic := *trendingTopics[nextTrendInd]

		// select its best content (first index available)
		indexToSelect := cache[trendingTopic].CurrentIndex

		// current index can be ahead of the max index in some special cases
		if indexToSelect <= cache[trendingTopic].MaximumIndex {
			selectedID := contentIDs[nextTrendInd][indexToSelect]

			if !__idsContainID(out, selectedID) && !__idsContainID(ignore, selectedID) {
				out = append(out, selectedID)
			}
		}

		// advance current index in cache
		cache[trendingTopic].CurrentIndex++

		// if we overshot the maximum index for this topic, remove it
		if cache[trendingTopic].CurrentIndex > cache[trendingTopic].MaximumIndex {
			// remove from cache
			delete(cache, trendingTopic)
			// remove from trending topics
			trendingTopics = append(trendingTopics[:nextTrendInd], trendingTopics[nextTrendInd+1:]...)
			// go back one position
			nextTrendInd--
		}

		// reset the counter
		nextTrendInd++
		if nextTrendInd == len(trendingTopics) {
			nextTrendInd = 0
		}
	}

	return out
}

func (net *Net) __selectOfRemaining(ignore []ID, howMany int) []ID {
	// if we are asking for more content items then there are available in content store
	if howMany >= (net.contentStore.Count() - len(ignore)) {
		ids := net.contentStore.GetAllContentIDs()
		// filter out ignored ones
		for i := 0; i < len(ids); i++ {
			if __idsContainID(ignore, ids[i]) {
				ids = append(ids[:i], ids[i+1:]...)
				i--
			}
		}
		return ids

	}

	// otherwise, draw random elements until howMany is satisifed. ignore duplicates of course.
	out := []ID{}

	for len(out) < howMany {
		randomID := net.contentStore.GetAnyContentID()
		if !__idsContainID(out, randomID) && !__idsContainID(ignore, randomID) {
			out = append(out, randomID)
		}
	}

	return out

}

func (net *Net) Describe() *NetDescription {
	return &NetDescription{
		Contents: net.contentStore.Describe(),
		Index:    net.index.Describe(),
		Profiles: net.profileStore.Describe(),
		Trends:   net.trendStore.Describe(),
	}
}
<commit_msg>Bugfix<commit_after>package contnet

import (
	"github.com/asaskevich/EventBus"
	"log"
	"math/rand"
	"sync"
	"time"
)

func init() {
	rand.Seed(time.Now().Unix())
}

type NetConfig struct {
	MaxContentAge           time.Duration
	CheckContentAgeInterval time.Duration
	ItemsPerPage            uint8
	NoveltyPct              float64
	SnapshotPath            string
	SnapshotInterval        time.Duration
}

type Net struct {
	sync.RWMutex
	config       *NetConfig
	bus          *EventBus.EventBus
	contentStore *ContentStore
	profileStore *ProfileStore
	trendStore   *TrendStore
	index        *Index
}
type NetFactory struct{}

func (factory NetFactory) New(config *NetConfig) *Net {
	bus := EventBus.New()
	contentStore := Object.ContentStore.New(config, bus)
	net := &Net{
		config:       config,
		bus:          bus,
		contentStore: contentStore,
		profileStore: Object.ProfileStore.New(),
		trendStore:   Object.TrendStore.New(bus),
		index:        Object.Index.New(config, bus, contentStore),
	}
	if err := net.Restore(); err != nil {
		log.Print("Failed to restore net object, proceeding empty.")
	}
	go net.__snapshot()
	go net.index.__refresh()

	return net
}

func (net *Net) __snapshot() {
	for {
		net.Snapshot()
		time.Sleep(net.config.SnapshotInterval)
	}
}

func (net *Net) Snapshot() error {
	if err := net.contentStore.Snapshot(net.config.SnapshotPath, "content"); err != nil {
		return err
	}

	if err := net.index.Snapshot(net.config.SnapshotPath, "index"); err != nil {
		return err
	}

	if err := net.profileStore.Snapshot(net.config.SnapshotPath, "profiles"); err != nil {
		return err
	}

	if err := net.trendStore.Snapshot(net.config.SnapshotPath, "trends"); err != nil {
		return err
	}

	return nil
}

func (net *Net) Restore() error {
	if err := net.contentStore.RestoreFromSnapshot(net.config.SnapshotPath, "content"); err != nil {
		return err
	}

	if err := net.index.RestoreFromSnapshot(net.config.SnapshotPath, "index"); err != nil {
		return err
	}

	if err := net.profileStore.RestoreFromSnapshot(net.config.SnapshotPath, "profiles"); err != nil {
		return err
	}

	if err := net.trendStore.RestoreFromSnapshot(net.config.SnapshotPath, "trends"); err != nil {
		return err
	}

	return nil
}

// Attempts to update network object with content specified.
// If content did not exist, it is added to the network.
// If content did exist, it is updated.
func (net *Net) SaveContent(content *Content) {
	net.contentStore.Upsert(content)
}

// Attempts to update network object with action for profile specified.
func (net *Net) SaveAction(action *Action) error {
	// get related content's copy, if any
	relatedContent := net.contentStore.Get(action.ContentID)

	// if content does not exist
	if relatedContent == nil {
		return Errors.ContentNotFound
	}

	// inject related content to action
	action.Content = relatedContent

	// save action to profile
	net.profileStore.Save(action)

	// everything is okay
	return nil
}

type SelectionCacheInfo struct {
	CurrentIndex int
	MaximumIndex int
}

func (net *Net) Select(profileID int64, page uint8) []ID {
	// get user's profile, if any
	profile := net.profileStore.Get(ID(profileID))

	// assign topic interests & sort
	var interests TopicInterests
	if profile != nil {
		TopicInterestBy(topicInterestCriteria).Sort(profile.TopicInterests)
		interests = profile.TopicInterests
	} else {
		interests = TopicInterests{}
	}

	totalContentsToRetrieve := int(page) * int(net.config.ItemsPerPage)
	totalContentsByInterest := int((1 - net.config.NoveltyPct) * float64(totalContentsToRetrieve))

	// select interesting contents
	interestingContents := net.__selectOfInterest(interests, totalContentsByInterest)
	//log.Println("Interesting: ", len(interestingContents))

	// select popular contents (how many remaining / 2)
	totalContentsByTrend := int((totalContentsToRetrieve - len(interestingContents)) / 2)
	trendingContents := net.__selectOfTrending(interestingContents, totalContentsByTrend)
	//log.Println("Trending: ", len(trendingContents))

	// fill the remainder randomly
	totalContentsRemaining := totalContentsToRetrieve - len(interestingContents) - len(trendingContents)
	out := append(interestingContents, trendingContents...)
	remainingContents := net.__selectOfRemaining(out, totalContentsRemaining)
	//log.Println("Remaining: ", len(remainingContents))

	// prep the final result
	out = append(out, remainingContents...)
	return out
}

func (net *Net) __selectOfInterest(interests TopicInterests, howMany int) []ID {
	// abort early if nothing to select..
	if len(interests) == 0 {
		return []ID{}
	}

	// we have sorted topic interests for this user.
	// now we extract topics from topic-interest objects for querying the index
	// and we use dem iterations to form cumulative probabilities slice to be used later for randomly drawing topics
	interestTopics := Topics{}
	cumulativeProbabilities := []float64{}
	for i := 0; i < len(interests); i++ {
		interestTopics = append(interestTopics, &interests[i].Topic)
		cumulativeProbabilities = append(cumulativeProbabilities, float64(interests[i].Interest))
		if i > 0 {
			cumulativeProbabilities[i] += cumulativeProbabilities[i-1]
		}
	}

	// use the previously created topics slice for querying the index
	topicContents := net.index.GetForTopics(interestTopics)

	// now we have all content IDs for all topics interesting to this user
	// to control where we're at, we will create temporary cache that will mark extraction location for each topic
	cache := map[Topic]*SelectionCacheInfo{}
	// filling in the cache
	for i := 0; i < len(interests); i++ {
		interest := interests[i]
		cache[interest.Topic] = &SelectionCacheInfo{
			CurrentIndex: 0,
			MaximumIndex: len(topicContents[i]) - 1,
		}
	}

	// now we have the following data available:
	// 1. topics of interest for user sorted in descending order by how interesting they are
	// 2. for each topic of interest there is an array of ID's available for that topic
	// 3. map showing what index to choose for each topic
	// 4. cumulative probabilities

	// while there is anything interesting for user available & we haven't selected as many items as needed
	out := []ID{}
	for len(cache) > 0 && len(out) < howMany {
		// select a random topic of interest
		topic, i := __drawRandomTopicInterest(interests, cumulativeProbabilities)

		// select its best content (first index available)
		indexToSelect := cache[topic].CurrentIndex

		// current index can be ahead of the max index in some special cases
		if indexToSelect <= cache[topic].MaximumIndex {
			selectedID := topicContents[i][indexToSelect]
			if !__idsContainID(out, selectedID) {
				out = append(out, selectedID)
			}
		}

		// advance current index in cache
		cache[topic].CurrentIndex++

		// if we overshot the maximum index for this topic, remove it
		if cache[topic].CurrentIndex > cache[topic].MaximumIndex {
			// remove from cache
			delete(cache, topic)
			// remove from interest topics
			interestTopics = append(interestTopics[:i], interestTopics[i+1:]...)
			// remove from topic contents
			topicContents = append(topicContents[:i], topicContents[i+1:]...)
			// remove from interests
			interests = append(interests[:i], interests[i+1:]...)
			// recalculate interests
			cumulativeProbabilities = __recalculateCumulativeProbabilities(interests)
		}
	}

	return out
}

func __idsContainID(ids []ID, id ID) bool {
	for i := 0; i < len(ids); i++ {
		if ids[i] == id {
			return true
		}
	}
	return false
}

func __drawRandomTopicInterest(topicInterests TopicInterests, cumulativeProbabilities []float64) (Topic, int) {
	randomFloat := rand.Float64()
	for i := 0; i < len(cumulativeProbabilities); i++ {
		if cumulativeProbabilities[i] >= randomFloat {
			return topicInterests[i].Topic, i
		}
	}
	// this is guaranteed not to happen
	return topicInterests[len(topicInterests)-1].Topic, len(topicInterests) - 1
}

func __recalculateCumulativeProbabilities(topicInterests TopicInterests) []float64 {
	// calculate sum of cumulative interests
	sum := float64(0)
	for i := 0; i < len(topicInterests); i++ {
		sum += float64(topicInterests[i].CumulativeInterest)
	}

	output := []float64{}

	for i := 0; i < len(topicInterests); i++ {
		output = append(output, float64(topicInterests[i].CumulativeInterest)/sum)
		if i > 0 {
			output[i] += output[i-1]
		}
	}

	return output
}

func (net *Net) __selectOfTrending(ignore []ID, howMany int) []ID {
	// select top 10 trending topics
	trendingTopics := net.trendStore.GetTopN(10)

	// nothing trending, bail early
	if len(trendingTopics) == 0 {
		return []ID{}
	}

	// get content IDs for trending topics
	contentIDs := net.index.GetForTopics(trendingTopics)

	// cache indices foreach contentID
	cache := map[Topic]*SelectionCacheInfo{}
	// filling in the cache
	for i := 0; i < len(trendingTopics); i++ {
		cache[*trendingTopics[i]] = &SelectionCacheInfo{
			CurrentIndex: 0,
			MaximumIndex: len(contentIDs[i]) - 1,
		}
	}

	nextTrendInd := 0
	out := []ID{}
	for len(cache) > 0 && len(out) < howMany {
		trendingTopic := *trendingTopics[nextTrendInd]

		// select its best content (first index available)
		indexToSelect := cache[trendingTopic].CurrentIndex

		// current index can be ahead of the max index in some special cases
		if indexToSelect <= cache[trendingTopic].MaximumIndex {
			selectedID := contentIDs[nextTrendInd][indexToSelect]

			if !__idsContainID(out, selectedID) && !__idsContainID(ignore, selectedID) {
				out = append(out, selectedID)
			}
		}

		// advance current index in cache
		cache[trendingTopic].CurrentIndex++

		// if we overshot the maximum index for this topic, remove it
		if cache[trendingTopic].CurrentIndex > cache[trendingTopic].MaximumIndex {
			// remove from cache
			delete(cache, trendingTopic)
			// remove from trending topics
			trendingTopics = append(trendingTopics[:nextTrendInd], trendingTopics[nextTrendInd+1:]...)
			// remove from content IDs
			contentIDs = append(contentIDs[:nextTrendInd], contentIDs[nextTrendInd+1:]...)

			// go back one position
			nextTrendInd--
		}

		// reset the counter
		nextTrendInd++
		if nextTrendInd == len(trendingTopics) {
			nextTrendInd = 0
		}
	}

	return out
}

func (net *Net) __selectOfRemaining(ignore []ID, howMany int) []ID {
	// if we are asking for more content items then there are available in content store
	if howMany >= (net.contentStore.Count() - len(ignore)) {
		ids := net.contentStore.GetAllContentIDs()
		// filter out ignored ones
		for i := 0; i < len(ids); i++ {
			if __idsContainID(ignore, ids[i]) {
				ids = append(ids[:i], ids[i+1:]...)
				i--
			}
		}
		return ids

	}

	// otherwise, draw random elements until howMany is satisifed. ignore duplicates of course.
	out := []ID{}

	for len(out) < howMany {
		randomID := net.contentStore.GetAnyContentID()
		if !__idsContainID(out, randomID) && !__idsContainID(ignore, randomID) {
			out = append(out, randomID)
		}
	}

	return out

}

func (net *Net) Describe() *NetDescription {
	return &NetDescription{
		Contents: net.contentStore.Describe(),
		Index:    net.index.Describe(),
		Profiles: net.profileStore.Describe(),
		Trends:   net.trendStore.Describe(),
	}
}
<|endoftext|>


<commit_before>package influxdb

import (
	"errors"
	"sync"
	"time"

	"github.com/influxdb/influxdb/data"
	"github.com/influxdb/influxdb/meta"
)

const defaultWriteTimeout = 5 * time.Second

// ConsistencyLevel represent a required replication criteria before a write can
// be returned as successful
type ConsistencyLevel int

const (
	// ConsistencyLevelAny allows for hinted hand off, potentially no write happened yet
	ConsistencyLevelAny ConsistencyLevel = iota

	// ConsistencyLevelOne requires at least one data node acknowledged a write
	ConsistencyLevelOne

	// ConsistencyLevelOne requires a quorum of data nodes to acknowledge a write
	ConsistencyLevelQuorum

	// ConsistencyLevelAll requires all data nodes to acknowledge a write
	ConsistencyLevelAll
)

var (
	// ErrTimeout is returned when a write times out
	ErrTimeout = errors.New("timeout")

	// ErrPartialWrite is returned when a write partially succeeds but does
	// not meet the requested consistency level
	ErrPartialWrite = errors.New("partial write")

	// ErrWriteFailed is returned when no writes succeeded
	ErrWriteFailed = errors.New("write failed")
)

// Coordinator handle queries and writes across multiple local and remote
// data nodes.
type Coordinator struct {
	mu      sync.RWMutex
	closing chan struct{}

	MetaStore    meta.Store
	shardWriters []ShardWriter
}

// ShardMapping contiains a mapping of a shards to a points.
type ShardMapping struct {
	Points map[uint64][]data.Point   // The points associated with a shard ID
	Shards map[uint64]meta.ShardInfo // The shards that have been mapped, keyed by shard ID
}

func NewCoordinator() *Coordinator {
	return &Coordinator{
		closing: make(chan struct{}),
	}
}

// NewShardMapping creates an empty ShardMapping
func NewShardMapping() *ShardMapping {
	return &ShardMapping{
		Points: map[uint64][]data.Point{},
		Shards: map[uint64]meta.ShardInfo{},
	}
}

// MapPoint maps a point to shard
func (s *ShardMapping) MapPoint(shardInfo meta.ShardInfo, p data.Point) {
	points, ok := s.Points[shardInfo.ID]
	if !ok {
		s.Points[shardInfo.ID] = []data.Point{p}
	} else {
		s.Points[shardInfo.ID] = append(points, p)
	}
	s.Shards[shardInfo.ID] = shardInfo
}

// ShardWriter provides the ability to write a slice of points to s given shard ID.
// It should return the number of times the set of points was written or an error
// if the write failed.
type ShardWriter interface {
	WriteShard(shardID uint64, points []data.Point) (int, error)
}

func (c *Coordinator) Open() error {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.closing == nil {
		c.closing = make(chan struct{})
	}
	return nil
}

func (c *Coordinator) Close() error {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.closing != nil {
		close(c.closing)
		c.closing = nil
	}
	return nil
}

func (c *Coordinator) AddShardWriter(s ShardWriter) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.shardWriters = append(c.shardWriters, s)
}

// MapShards maps the points contained in wp to a ShardMapping.  If a point
// maps to a shard group or shard that does not currently exist, it will be
// created before returning the mapping.
func (c *Coordinator) MapShards(wp *WritePointsRequest) (*ShardMapping, error) {

	// holds the start time ranges for required shard groups
	timeRanges := map[time.Time]*meta.ShardGroupInfo{}

	rp, err := c.MetaStore.RetentionPolicy(wp.Database, wp.RetentionPolicy)
	if err != nil {
		return nil, err
	}

	for _, p := range wp.Points {
		timeRanges[p.Time.Truncate(rp.ShardGroupDuration)] = nil
	}

	// holds all the shard groups and shards that are required for writes
	for t := range timeRanges {
		g, err := c.MetaStore.CreateShardGroupIfNotExists(wp.Database, wp.RetentionPolicy, t)
		if err != nil {
			return nil, err
		}
		timeRanges[t] = g
	}

	shardMapping := NewShardMapping()
	for _, p := range wp.Points {
		g := timeRanges[p.Time.Truncate(rp.ShardGroupDuration)]
		sid := p.SeriesID()
		shardInfo := g.Shards[sid%uint64(len(g.Shards))]
		shardMapping.MapPoint(shardInfo, p)
	}
	return shardMapping, nil
}

// Write is coordinates multiple writes across local and remote data nodes
// according the request consistency level
func (c *Coordinator) Write(p *WritePointsRequest) error {
	shardMappings, err := c.MapShards(p)
	if err != nil {
		return err
	}

	ch := make(chan error, len(shardMappings.Points))
	for shardID, points := range shardMappings.Points {
		go func(shard meta.ShardInfo, points []data.Point) {
			ch <- c.writeToShards(shard, p.ConsistencyLevel, points)
		}(shardMappings.Shards[shardID], points)
	}

	for range shardMappings.Points {
		select {
		case <-c.closing:
			return ErrWriteFailed
		case err := <-ch:
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// writeToShards writes points to a shard and ensures a write consistency level has been met.  If the write
// partially succceds, ErrPartialWrite is returned.
func (c *Coordinator) writeToShards(shard meta.ShardInfo, consistency ConsistencyLevel, points []data.Point) error {
	c.mu.RLock()
	defer c.mu.RUnlock()

	if len(c.shardWriters) == 0 {
		return ErrWriteFailed
	}

	requiredResponses := len(shard.OwnerIDs)
	switch consistency {
	case ConsistencyLevelAny, ConsistencyLevelOne:
		requiredResponses = 1
	case ConsistencyLevelQuorum:
		requiredResponses = requiredResponses/2 + 1
	}

	// holds the response to the ShardWriter.Write calls
	type result struct {
		wrote int
		err   error
	}

	// response channel for each shard writer go routine
	ch := make(chan result, len(c.shardWriters))

	for _, w := range c.shardWriters {
		// write to each ShardWriter (local and remote), in parallel
		go func(w ShardWriter, shardID uint64, points []data.Point) {
			wrote, err := w.WriteShard(shardID, points)
			ch <- result{wrote, err}
		}(w, shard.ID, points)
	}

	var wrote int
	timeout := time.After(defaultWriteTimeout)
	for range c.shardWriters {
		select {
		case <-c.closing:
			return ErrWriteFailed
		case <-timeout:
			// return timeout error to caller
			return ErrTimeout
		case res := <-ch:
			wrote += res.wrote

			// ErrShardNotLocal might be returned from a local writer. Ignore it.
			if res.err != nil && res.err != ErrShardNotLocal {
				return res.err
			}

			// We wrote the required consistency level
			if wrote >= requiredResponses {
				return nil
			}
		}
	}

	if wrote > 0 {
		return ErrPartialWrite
	}

	return ErrWriteFailed
}

func (c *Coordinator) Execute(q *QueryRequest) (chan *Result, error) {
	return nil, nil
}
<commit_msg>Clarify comments<commit_after>package influxdb

import (
	"errors"
	"sync"
	"time"

	"github.com/influxdb/influxdb/data"
	"github.com/influxdb/influxdb/meta"
)

const defaultWriteTimeout = 5 * time.Second

// ConsistencyLevel represent a required replication criteria before a write can
// be returned as successful
type ConsistencyLevel int

const (
	// ConsistencyLevelAny allows for hinted hand off, potentially no write happened yet
	ConsistencyLevelAny ConsistencyLevel = iota

	// ConsistencyLevelOne requires at least one data node acknowledged a write
	ConsistencyLevelOne

	// ConsistencyLevelOne requires a quorum of data nodes to acknowledge a write
	ConsistencyLevelQuorum

	// ConsistencyLevelAll requires all data nodes to acknowledge a write
	ConsistencyLevelAll
)

var (
	// ErrTimeout is returned when a write times out
	ErrTimeout = errors.New("timeout")

	// ErrPartialWrite is returned when a write partially succeeds but does
	// not meet the requested consistency level
	ErrPartialWrite = errors.New("partial write")

	// ErrWriteFailed is returned when no writes succeeded
	ErrWriteFailed = errors.New("write failed")
)

// Coordinator handle queries and writes across multiple local and remote
// data nodes.
type Coordinator struct {
	mu      sync.RWMutex
	closing chan struct{}

	MetaStore    meta.Store
	shardWriters []ShardWriter
}

// ShardMapping contiains a mapping of a shards to a points.
type ShardMapping struct {
	Points map[uint64][]data.Point   // The points associated with a shard ID
	Shards map[uint64]meta.ShardInfo // The shards that have been mapped, keyed by shard ID
}

func NewCoordinator() *Coordinator {
	return &Coordinator{
		closing: make(chan struct{}),
	}
}

// NewShardMapping creates an empty ShardMapping
func NewShardMapping() *ShardMapping {
	return &ShardMapping{
		Points: map[uint64][]data.Point{},
		Shards: map[uint64]meta.ShardInfo{},
	}
}

// MapPoint maps a point to shard
func (s *ShardMapping) MapPoint(shardInfo meta.ShardInfo, p data.Point) {
	points, ok := s.Points[shardInfo.ID]
	if !ok {
		s.Points[shardInfo.ID] = []data.Point{p}
	} else {
		s.Points[shardInfo.ID] = append(points, p)
	}
	s.Shards[shardInfo.ID] = shardInfo
}

// ShardWriter provides the ability to write a slice of points to s given shard ID.
// It should return the number of times the set of points was written or an error
// if the write failed.
type ShardWriter interface {
	WriteShard(shardID uint64, points []data.Point) (int, error)
}

func (c *Coordinator) Open() error {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.closing == nil {
		c.closing = make(chan struct{})
	}
	return nil
}

func (c *Coordinator) Close() error {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.closing != nil {
		close(c.closing)
		c.closing = nil
	}
	return nil
}

func (c *Coordinator) AddShardWriter(s ShardWriter) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.shardWriters = append(c.shardWriters, s)
}

// MapShards maps the points contained in wp to a ShardMapping.  If a point
// maps to a shard group or shard that does not currently exist, it will be
// created before returning the mapping.
func (c *Coordinator) MapShards(wp *WritePointsRequest) (*ShardMapping, error) {

	// holds the start time ranges for required shard groups
	timeRanges := map[time.Time]*meta.ShardGroupInfo{}

	rp, err := c.MetaStore.RetentionPolicy(wp.Database, wp.RetentionPolicy)
	if err != nil {
		return nil, err
	}

	for _, p := range wp.Points {
		timeRanges[p.Time.Truncate(rp.ShardGroupDuration)] = nil
	}

	// holds all the shard groups and shards that are required for writes
	for t := range timeRanges {
		g, err := c.MetaStore.CreateShardGroupIfNotExists(wp.Database, wp.RetentionPolicy, t)
		if err != nil {
			return nil, err
		}
		timeRanges[t] = g
	}

	shardMapping := NewShardMapping()
	for _, p := range wp.Points {
		g := timeRanges[p.Time.Truncate(rp.ShardGroupDuration)]
		sid := p.SeriesID()
		shardInfo := g.Shards[sid%uint64(len(g.Shards))]
		shardMapping.MapPoint(shardInfo, p)
	}
	return shardMapping, nil
}

// Write is coordinates multiple writes across local and remote data nodes
// according the request consistency level
func (c *Coordinator) Write(p *WritePointsRequest) error {
	shardMappings, err := c.MapShards(p)
	if err != nil {
		return err
	}

	// Write each shard in it's own goroutine and return as soon
	// as one fails.
	ch := make(chan error, len(shardMappings.Points))
	for shardID, points := range shardMappings.Points {
		go func(shard meta.ShardInfo, points []data.Point) {
			ch <- c.writeToShards(shard, p.ConsistencyLevel, points)
		}(shardMappings.Shards[shardID], points)
	}

	for range shardMappings.Points {
		select {
		case <-c.closing:
			return ErrWriteFailed
		case err := <-ch:
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// writeToShards writes points to a shard and ensures a write consistency level has been met.  If the write
// partially succceds, ErrPartialWrite is returned.
func (c *Coordinator) writeToShards(shard meta.ShardInfo, consistency ConsistencyLevel, points []data.Point) error {
	c.mu.RLock()
	defer c.mu.RUnlock()

	// Abort early if there are no writer configured yet
	if len(c.shardWriters) == 0 {
		return ErrWriteFailed
	}

	// The required number of writes to achieve the requested consistency level
	required := len(shard.OwnerIDs)
	switch consistency {
	case ConsistencyLevelAny, ConsistencyLevelOne:
		required = 1
	case ConsistencyLevelQuorum:
		required = required/2 + 1
	}

	// holds the response to the ShardWriter.Write calls
	type result struct {
		wrote int
		err   error
	}

	// response channel for each shard writer go routine
	ch := make(chan result, len(c.shardWriters))

	for _, w := range c.shardWriters {
		// write to each ShardWriter (local and remote), in parallel
		go func(w ShardWriter, shardID uint64, points []data.Point) {
			wrote, err := w.WriteShard(shardID, points)
			ch <- result{wrote, err}
		}(w, shard.ID, points)
	}

	var wrote int
	timeout := time.After(defaultWriteTimeout)
	for range c.shardWriters {
		select {
		case <-c.closing:
			return ErrWriteFailed
		case <-timeout:
			// return timeout error to caller
			return ErrTimeout
		case res := <-ch:
			wrote += res.wrote

			// ErrShardNotLocal might be returned from a local writer. Ignore it.
			if res.err != nil && res.err != ErrShardNotLocal {
				return res.err
			}

			// We wrote the required consistency level
			if wrote >= required {
				return nil
			}
		}
	}

	if wrote > 0 {
		return ErrPartialWrite
	}

	return ErrWriteFailed
}

func (c *Coordinator) Execute(q *QueryRequest) (chan *Result, error) {
	return nil, nil
}
<|endoftext|>


<commit_before>package image

import (
	"encoding/json"

	"github.com/containers/image/docker/reference"
	"github.com/containers/image/manifest"
	"github.com/containers/image/types"
	"github.com/opencontainers/go-digest"
	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

type manifestSchema1 struct {
	m *manifest.Schema1
}

func manifestSchema1FromManifest(manifestBlob []byte) (genericManifest, error) {
	m, err := manifest.Schema1FromManifest(manifestBlob)
	if err != nil {
		return nil, err
	}
	return &manifestSchema1{m: m}, nil
}

// manifestSchema1FromComponents builds a new manifestSchema1 from the supplied data.
func manifestSchema1FromComponents(ref reference.Named, fsLayers []manifest.Schema1FSLayers, history []manifest.Schema1History, architecture string) genericManifest {
	return &manifestSchema1{m: manifest.Schema1FromComponents(ref, fsLayers, history, architecture)}
}

func (m *manifestSchema1) serialize() ([]byte, error) {
	return m.m.Serialize()
}

func (m *manifestSchema1) manifestMIMEType() string {
	return manifest.DockerV2Schema1SignedMediaType
}

// ConfigInfo returns a complete BlobInfo for the separate config object, or a BlobInfo{Digest:""} if there isn't a separate object.
// Note that the config object may not exist in the underlying storage in the return value of UpdatedImage! Use ConfigBlob() below.
func (m *manifestSchema1) ConfigInfo() types.BlobInfo {
	return m.m.ConfigInfo()
}

// ConfigBlob returns the blob described by ConfigInfo, iff ConfigInfo().Digest != ""; nil otherwise.
// The result is cached; it is OK to call this however often you need.
func (m *manifestSchema1) ConfigBlob() ([]byte, error) {
	return nil, nil
}

// OCIConfig returns the image configuration as per OCI v1 image-spec. Information about
// layers in the resulting configuration isn't guaranteed to be returned to due how
// old image manifests work (docker v2s1 especially).
func (m *manifestSchema1) OCIConfig() (*imgspecv1.Image, error) {
	v2s2, err := m.convertToManifestSchema2(nil, nil)
	if err != nil {
		return nil, err
	}
	return v2s2.OCIConfig()
}

// LayerInfos returns a list of BlobInfos of layers referenced by this image, in order (the root layer first, and then successive layered layers).
// The Digest field is guaranteed to be provided; Size may be -1.
// WARNING: The list may contain duplicates, and they are semantically relevant.
func (m *manifestSchema1) LayerInfos() []types.BlobInfo {
	return m.m.LayerInfos()
}

// EmbeddedDockerReferenceConflicts whether a Docker reference embedded in the manifest, if any, conflicts with destination ref.
// It returns false if the manifest does not embed a Docker reference.
// (This embedding unfortunately happens for Docker schema1, please do not add support for this in any new formats.)
func (m *manifestSchema1) EmbeddedDockerReferenceConflicts(ref reference.Named) bool {
	// This is a bit convoluted: We can’t just have a "get embedded docker reference" method
	// and have the “does it conflict” logic in the generic copy code, because the manifest does not actually
	// embed a full docker/distribution reference, but only the repo name and tag (without the host name).
	// So we would have to provide a “return repo without host name, and tag” getter for the generic code,
	// which would be very awkward.  Instead, we do the matching here in schema1-specific code, and all the
	// generic copy code needs to know about is reference.Named and that a manifest may need updating
	// for some destinations.
	name := reference.Path(ref)
	var tag string
	if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
		tag = tagged.Tag()
	} else {
		tag = ""
	}
	return m.m.Name != name || m.m.Tag != tag
}

func (m *manifestSchema1) imageInspectInfo() (*types.ImageInspectInfo, error) {
	return m.m.Inspect(nil)
}

// UpdatedImageNeedsLayerDiffIDs returns true iff UpdatedImage(options) needs InformationOnly.LayerDiffIDs.
// This is a horribly specific interface, but computing InformationOnly.LayerDiffIDs can be very expensive to compute
// (most importantly it forces us to download the full layers even if they are already present at the destination).
func (m *manifestSchema1) UpdatedImageNeedsLayerDiffIDs(options types.ManifestUpdateOptions) bool {
	return options.ManifestMIMEType == manifest.DockerV2Schema2MediaType
}

// UpdatedImage returns a types.Image modified according to options.
// This does not change the state of the original Image object.
func (m *manifestSchema1) UpdatedImage(options types.ManifestUpdateOptions) (types.Image, error) {
	copy := manifestSchema1{m: manifest.Schema1Clone(m.m)}
	if options.LayerInfos != nil {
		if err := copy.m.UpdateLayerInfos(options.LayerInfos); err != nil {
			return nil, err
		}
	}
	if options.EmbeddedDockerReference != nil {
		copy.m.Name = reference.Path(options.EmbeddedDockerReference)
		if tagged, isTagged := options.EmbeddedDockerReference.(reference.NamedTagged); isTagged {
			copy.m.Tag = tagged.Tag()
		} else {
			copy.m.Tag = ""
		}
	}

	switch options.ManifestMIMEType {
	case "": // No conversion, OK
	case manifest.DockerV2Schema1MediaType, manifest.DockerV2Schema1SignedMediaType:
	// We have 2 MIME types for schema 1, which are basically equivalent (even the un-"Signed" MIME type will be rejected if there isn’t a signature; so,
	// handle conversions between them by doing nothing.
	case manifest.DockerV2Schema2MediaType:
		m2, err := copy.convertToManifestSchema2(options.InformationOnly.LayerInfos, options.InformationOnly.LayerDiffIDs)
		if err != nil {
			return nil, err
		}
		return memoryImageFromManifest(m2), nil
	case imgspecv1.MediaTypeImageManifest:
		// We can't directly convert to OCI, but we can transitively convert via a Docker V2.2 Distribution manifest
		m2, err := copy.convertToManifestSchema2(options.InformationOnly.LayerInfos, options.InformationOnly.LayerDiffIDs)
		if err != nil {
			return nil, err
		}
		return m2.UpdatedImage(types.ManifestUpdateOptions{
			ManifestMIMEType: imgspecv1.MediaTypeImageManifest,
			InformationOnly:  options.InformationOnly,
		})
	default:
		return nil, errors.Errorf("Conversion of image manifest from %s to %s is not implemented", manifest.DockerV2Schema1SignedMediaType, options.ManifestMIMEType)
	}

	return memoryImageFromManifest(&copy), nil
}

// Based on github.com/docker/docker/distribution/pull_v2.go
func (m *manifestSchema1) convertToManifestSchema2(uploadedLayerInfos []types.BlobInfo, layerDiffIDs []digest.Digest) (genericManifest, error) {
	if len(m.m.History) == 0 {
		// What would this even mean?! Anyhow, the rest of the code depends on fsLayers[0] and history[0] existing.
		return nil, errors.Errorf("Cannot convert an image with 0 history entries to %s", manifest.DockerV2Schema2MediaType)
	}
	if len(m.m.History) != len(m.m.FSLayers) {
		return nil, errors.Errorf("Inconsistent schema 1 manifest: %d history entries, %d fsLayers entries", len(m.m.History), len(m.m.FSLayers))
	}
	if uploadedLayerInfos != nil && len(uploadedLayerInfos) != len(m.m.FSLayers) {
		return nil, errors.Errorf("Internal error: uploaded %d blobs, but schema1 manifest has %d fsLayers", len(uploadedLayerInfos), len(m.m.FSLayers))
	}
	if layerDiffIDs != nil && len(layerDiffIDs) != len(m.m.FSLayers) {
		return nil, errors.Errorf("Internal error: collected %d DiffID values, but schema1 manifest has %d fsLayers", len(layerDiffIDs), len(m.m.FSLayers))
	}

	// Build a list of the diffIDs for the non-empty layers.
	diffIDs := []digest.Digest{}
	var layers []manifest.Schema2Descriptor
	for v1Index := len(m.m.History) - 1; v1Index >= 0; v1Index-- {
		v2Index := (len(m.m.History) - 1) - v1Index

		var v1compat manifest.Schema1V1Compatibility
		if err := json.Unmarshal([]byte(m.m.History[v1Index].V1Compatibility), &v1compat); err != nil {
			return nil, errors.Wrapf(err, "Error decoding history entry %d", v1Index)
		}
		if !v1compat.ThrowAway {
			var size int64
			if uploadedLayerInfos != nil {
				size = uploadedLayerInfos[v2Index].Size
			}
			var d digest.Digest
			if layerDiffIDs != nil {
				d = layerDiffIDs[v2Index]
			}
			layers = append(layers, manifest.Schema2Descriptor{
				MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
				Size:      size,
				Digest:    m.m.FSLayers[v1Index].BlobSum,
			})
			diffIDs = append(diffIDs, d)
		}
	}
	configJSON, err := m.m.ToSchema2(diffIDs)
	if err != nil {
		return nil, err
	}
	configDescriptor := manifest.Schema2Descriptor{
		MediaType: "application/vnd.docker.container.image.v1+json",
		Size:      int64(len(configJSON)),
		Digest:    digest.FromBytes(configJSON),
	}

	return manifestSchema2FromComponents(configDescriptor, nil, configJSON, layers), nil
}
<commit_msg>image: fix docker schema v1 -> OCI conversion<commit_after>package image

import (
	"encoding/json"

	"github.com/containers/image/docker/reference"
	"github.com/containers/image/manifest"
	"github.com/containers/image/types"
	"github.com/opencontainers/go-digest"
	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

type manifestSchema1 struct {
	m *manifest.Schema1
}

func manifestSchema1FromManifest(manifestBlob []byte) (genericManifest, error) {
	m, err := manifest.Schema1FromManifest(manifestBlob)
	if err != nil {
		return nil, err
	}
	return &manifestSchema1{m: m}, nil
}

// manifestSchema1FromComponents builds a new manifestSchema1 from the supplied data.
func manifestSchema1FromComponents(ref reference.Named, fsLayers []manifest.Schema1FSLayers, history []manifest.Schema1History, architecture string) genericManifest {
	return &manifestSchema1{m: manifest.Schema1FromComponents(ref, fsLayers, history, architecture)}
}

func (m *manifestSchema1) serialize() ([]byte, error) {
	return m.m.Serialize()
}

func (m *manifestSchema1) manifestMIMEType() string {
	return manifest.DockerV2Schema1SignedMediaType
}

// ConfigInfo returns a complete BlobInfo for the separate config object, or a BlobInfo{Digest:""} if there isn't a separate object.
// Note that the config object may not exist in the underlying storage in the return value of UpdatedImage! Use ConfigBlob() below.
func (m *manifestSchema1) ConfigInfo() types.BlobInfo {
	return m.m.ConfigInfo()
}

// ConfigBlob returns the blob described by ConfigInfo, iff ConfigInfo().Digest != ""; nil otherwise.
// The result is cached; it is OK to call this however often you need.
func (m *manifestSchema1) ConfigBlob() ([]byte, error) {
	return nil, nil
}

// OCIConfig returns the image configuration as per OCI v1 image-spec. Information about
// layers in the resulting configuration isn't guaranteed to be returned to due how
// old image manifests work (docker v2s1 especially).
func (m *manifestSchema1) OCIConfig() (*imgspecv1.Image, error) {
	v2s2, err := m.convertToManifestSchema2(nil, nil)
	if err != nil {
		return nil, err
	}
	return v2s2.OCIConfig()
}

// LayerInfos returns a list of BlobInfos of layers referenced by this image, in order (the root layer first, and then successive layered layers).
// The Digest field is guaranteed to be provided; Size may be -1.
// WARNING: The list may contain duplicates, and they are semantically relevant.
func (m *manifestSchema1) LayerInfos() []types.BlobInfo {
	return m.m.LayerInfos()
}

// EmbeddedDockerReferenceConflicts whether a Docker reference embedded in the manifest, if any, conflicts with destination ref.
// It returns false if the manifest does not embed a Docker reference.
// (This embedding unfortunately happens for Docker schema1, please do not add support for this in any new formats.)
func (m *manifestSchema1) EmbeddedDockerReferenceConflicts(ref reference.Named) bool {
	// This is a bit convoluted: We can’t just have a "get embedded docker reference" method
	// and have the “does it conflict” logic in the generic copy code, because the manifest does not actually
	// embed a full docker/distribution reference, but only the repo name and tag (without the host name).
	// So we would have to provide a “return repo without host name, and tag” getter for the generic code,
	// which would be very awkward.  Instead, we do the matching here in schema1-specific code, and all the
	// generic copy code needs to know about is reference.Named and that a manifest may need updating
	// for some destinations.
	name := reference.Path(ref)
	var tag string
	if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
		tag = tagged.Tag()
	} else {
		tag = ""
	}
	return m.m.Name != name || m.m.Tag != tag
}

func (m *manifestSchema1) imageInspectInfo() (*types.ImageInspectInfo, error) {
	return m.m.Inspect(nil)
}

// UpdatedImageNeedsLayerDiffIDs returns true iff UpdatedImage(options) needs InformationOnly.LayerDiffIDs.
// This is a horribly specific interface, but computing InformationOnly.LayerDiffIDs can be very expensive to compute
// (most importantly it forces us to download the full layers even if they are already present at the destination).
func (m *manifestSchema1) UpdatedImageNeedsLayerDiffIDs(options types.ManifestUpdateOptions) bool {
	return (options.ManifestMIMEType == manifest.DockerV2Schema2MediaType || options.ManifestMIMEType == imgspecv1.MediaTypeImageManifest)
}

// UpdatedImage returns a types.Image modified according to options.
// This does not change the state of the original Image object.
func (m *manifestSchema1) UpdatedImage(options types.ManifestUpdateOptions) (types.Image, error) {
	copy := manifestSchema1{m: manifest.Schema1Clone(m.m)}
	if options.LayerInfos != nil {
		if err := copy.m.UpdateLayerInfos(options.LayerInfos); err != nil {
			return nil, err
		}
	}
	if options.EmbeddedDockerReference != nil {
		copy.m.Name = reference.Path(options.EmbeddedDockerReference)
		if tagged, isTagged := options.EmbeddedDockerReference.(reference.NamedTagged); isTagged {
			copy.m.Tag = tagged.Tag()
		} else {
			copy.m.Tag = ""
		}
	}

	switch options.ManifestMIMEType {
	case "": // No conversion, OK
	case manifest.DockerV2Schema1MediaType, manifest.DockerV2Schema1SignedMediaType:
	// We have 2 MIME types for schema 1, which are basically equivalent (even the un-"Signed" MIME type will be rejected if there isn’t a signature; so,
	// handle conversions between them by doing nothing.
	case manifest.DockerV2Schema2MediaType:
		m2, err := copy.convertToManifestSchema2(options.InformationOnly.LayerInfos, options.InformationOnly.LayerDiffIDs)
		if err != nil {
			return nil, err
		}
		return memoryImageFromManifest(m2), nil
	case imgspecv1.MediaTypeImageManifest:
		// We can't directly convert to OCI, but we can transitively convert via a Docker V2.2 Distribution manifest
		m2, err := copy.convertToManifestSchema2(options.InformationOnly.LayerInfos, options.InformationOnly.LayerDiffIDs)
		if err != nil {
			return nil, err
		}
		return m2.UpdatedImage(types.ManifestUpdateOptions{
			ManifestMIMEType: imgspecv1.MediaTypeImageManifest,
			InformationOnly:  options.InformationOnly,
		})
	default:
		return nil, errors.Errorf("Conversion of image manifest from %s to %s is not implemented", manifest.DockerV2Schema1SignedMediaType, options.ManifestMIMEType)
	}

	return memoryImageFromManifest(&copy), nil
}

// Based on github.com/docker/docker/distribution/pull_v2.go
func (m *manifestSchema1) convertToManifestSchema2(uploadedLayerInfos []types.BlobInfo, layerDiffIDs []digest.Digest) (genericManifest, error) {
	if len(m.m.History) == 0 {
		// What would this even mean?! Anyhow, the rest of the code depends on fsLayers[0] and history[0] existing.
		return nil, errors.Errorf("Cannot convert an image with 0 history entries to %s", manifest.DockerV2Schema2MediaType)
	}
	if len(m.m.History) != len(m.m.FSLayers) {
		return nil, errors.Errorf("Inconsistent schema 1 manifest: %d history entries, %d fsLayers entries", len(m.m.History), len(m.m.FSLayers))
	}
	if uploadedLayerInfos != nil && len(uploadedLayerInfos) != len(m.m.FSLayers) {
		return nil, errors.Errorf("Internal error: uploaded %d blobs, but schema1 manifest has %d fsLayers", len(uploadedLayerInfos), len(m.m.FSLayers))
	}
	if layerDiffIDs != nil && len(layerDiffIDs) != len(m.m.FSLayers) {
		return nil, errors.Errorf("Internal error: collected %d DiffID values, but schema1 manifest has %d fsLayers", len(layerDiffIDs), len(m.m.FSLayers))
	}

	// Build a list of the diffIDs for the non-empty layers.
	diffIDs := []digest.Digest{}
	var layers []manifest.Schema2Descriptor
	for v1Index := len(m.m.History) - 1; v1Index >= 0; v1Index-- {
		v2Index := (len(m.m.History) - 1) - v1Index

		var v1compat manifest.Schema1V1Compatibility
		if err := json.Unmarshal([]byte(m.m.History[v1Index].V1Compatibility), &v1compat); err != nil {
			return nil, errors.Wrapf(err, "Error decoding history entry %d", v1Index)
		}
		if !v1compat.ThrowAway {
			var size int64
			if uploadedLayerInfos != nil {
				size = uploadedLayerInfos[v2Index].Size
			}
			var d digest.Digest
			if layerDiffIDs != nil {
				d = layerDiffIDs[v2Index]
			}
			layers = append(layers, manifest.Schema2Descriptor{
				MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
				Size:      size,
				Digest:    m.m.FSLayers[v1Index].BlobSum,
			})
			diffIDs = append(diffIDs, d)
		}
	}
	configJSON, err := m.m.ToSchema2(diffIDs)
	if err != nil {
		return nil, err
	}
	configDescriptor := manifest.Schema2Descriptor{
		MediaType: "application/vnd.docker.container.image.v1+json",
		Size:      int64(len(configJSON)),
		Digest:    digest.FromBytes(configJSON),
	}

	return manifestSchema2FromComponents(configDescriptor, nil, configJSON, layers), nil
}
<|endoftext|>


<commit_before>package database

import (
	"github.com/paultag/go-dictd/dictd"
)

type FnordDatabase struct {
	dictd.Database
}

func (this *FnordDatabase) Match(query string, strat string) []*dictd.Definition {
	els := make([]*dictd.Definition, 1)
	els[0] = &dictd.Definition{
		Word:       query,
		Definition: query,
	}
	return els
}

func (this *FnordDatabase) Define(query string) []*dictd.Definition {
	els := make([]*dictd.Definition, 1)
	els[0] = &dictd.Definition{
		Word:       query,
		Definition: query,
	}
	return els
}

func (this *FnordDatabase) Info() string {
	return `Some long
Description goes here

and here.`
}

func (this *FnordDatabase) Description() string {
	return "fake testing database"
}
<commit_msg>Def is optional here.<commit_after>package database

import (
	"github.com/paultag/go-dictd/dictd"
)

type FnordDatabase struct {
	dictd.Database
}

func (this *FnordDatabase) Match(query string, strat string) []*dictd.Definition {
	els := make([]*dictd.Definition, 1)
	els[0] = &dictd.Definition{Word: query}
	return els
}

func (this *FnordDatabase) Define(query string) []*dictd.Definition {
	els := make([]*dictd.Definition, 1)
	els[0] = &dictd.Definition{
		Word:       query,
		Definition: query,
	}
	return els
}

func (this *FnordDatabase) Info() string {
	return `Some long
Description goes here

and here.`
}

func (this *FnordDatabase) Description() string {
	return "fake testing database"
}
<|endoftext|>


<commit_before>package dataframe

import (
	"fmt"
	"os"
	"reflect"
	"testing"
)

func TestFrame(t *testing.T) {
	c1 := NewColumn("second1")
	for i := 0; i < 100; i++ {
		d := c1.PushBack(NewStringValue(fmt.Sprintf("%d", i)))
		if i+1 != d {
			t.Fatalf("expected %d, got %d", i+1, d)
		}
	}
	c1.UpdateHeader("aaa")
	if c1.GetHeader() != "aaa" {
		t.Fatalf("expected 'aaa', got %v", c1.GetHeader())
	}
	c1.UpdateHeader("second1")
	if c1.GetHeader() != "second1" {
		t.Fatalf("expected 'second1', got %v", c1.GetHeader())
	}

	c2 := NewColumn("second2")
	for i := 0; i < 100; i++ {
		d := c2.PushBack(NewStringValue(fmt.Sprintf("%d", i)))
		if i+1 != d {
			t.Fatalf("expected %d, got %d", i+1, d)
		}
	}

	fr := New()
	if err := fr.AddColumn(c1); err != nil {
		t.Fatal(err)
	}
	if err := fr.AddColumn(c1); err == nil {
		t.Fatal("expected error")
	}
	if err := fr.AddColumn(c2); err != nil {
		t.Fatal(err)
	}
	if err := fr.AddColumn(c2); err == nil {
		t.Fatal("expected error")
	}

	if c, err := fr.GetColumn("second1"); c == nil || err != nil {
		t.Fatal(err)
	}
	if c, err := fr.GetColumn("second2"); c == nil || err != nil {
		t.Fatal(err)
	}

	if err := fr.UpdateHeader("second2", "aaa"); err != nil {
		t.Fatal(err)
	}
	if c, err := fr.GetColumn("aaa"); c == nil || err != nil {
		t.Fatal(err)
	}
	if hs := fr.GetHeader(); !reflect.DeepEqual(hs, []string{"second1", "aaa"}) {
		t.Fatalf("expected equal, got %q != %q", hs, []string{"second1", "aaa"})
	}

	if err := fr.UpdateHeader("aaa", "second2"); err != nil {
		t.Fatal(err)
	}
	if hs := fr.GetHeader(); !reflect.DeepEqual(hs, []string{"second1", "second2"}) {
		t.Fatalf("expected equal, got %q != %q", hs, []string{"second1", "second2"})
	}
	if ok := fr.DeleteColumn("second1"); !ok {
		t.Fatalf("expected 'true', got %v", ok)
	}
	if cd := fr.GetColumnNumber(); cd != 1 {
		t.Fatalf("expected 1, got %v", cd)
	}
	if ok := fr.DeleteColumn("second1"); ok {
		t.Fatalf("expected 'false', got %v", ok)
	}
	if c, err := fr.GetColumn("second1"); c != nil || err == nil {
		t.Fatal("expected <nil, 'second1 does not exist'>, but <%v, %v>", c, err)
	}
	if ok := fr.DeleteColumn("second2"); !ok {
		t.Fatalf("expected 'true', got %v", ok)
	}
	if cd := fr.GetColumnNumber(); cd != 0 {
		t.Fatalf("expected 0, got %v", cd)
	}
}

func TestNewFromCSV(t *testing.T) {
	if _, err := NewFromCSV([]string{"second"}, "testdata/bench-compared.csv"); err == nil {
		t.Fatal("expected error, got nil")
	}
	fr, err := NewFromCSV(nil, "testdata/bench-compared.csv")
	if err != nil {
		t.Fatal(err)
	}
	cols := []string{"second", "avg_latency_ms_consul", "throughput_consul", "avg_cpu_consul", "avg_memory_mb_consul", "avg_latency_ms_etcd", "throughput_etcd", "avg_cpu_etcd", "avg_memory_mb_etcd", "avg_latency_ms_etcd2", "throughput_etcd2", "avg_cpu_etcd2", "avg_memory_mb_etcd2", "avg_latency_ms_zk", "throughput_zk", "avg_cpu_zk", "avg_memory_mb_zk"}
	if !reflect.DeepEqual(fr.GetHeader(), cols) {
		t.Fatalf("expected %q, got %q", cols, fr.GetHeader())
	}
	ac, err := fr.GetColumn("avg_latency_ms_etcd")
	if err != nil {
		t.Fatal(err)
	}
	if v, err := ac.GetValue(77); !v.IsNil() || err != nil {
		t.Fatalf("expected <nil, nil>, got <%v, %v>", v.IsNil(), err)
	}
	if v, err := ac.GetValue(0); v.IsNil() || !v.EqualTo(NewStringValue("51.607760")) || err != nil {
		t.Fatalf("expected <nil, nil>, got <%v, %v>", v, err)
	}
	ac2, err := fr.GetColumn("avg_latency_ms_etcd2")
	if err != nil {
		t.Fatal(err)
	}
	if ac.Len() != ac2.Len() {
		t.Fatalf("expected equal %v != %v", ac.Len(), ac2.Len())
	}

	fpath := "test.csv"
	if err := fr.ToCSV(fpath); err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(fpath)

	{
		if _, err := NewFromCSV([]string{"second"}, fpath); err == nil {
			t.Fatal("expected error, got nil")
		}
		fr, err := NewFromCSV(nil, fpath)
		if err != nil {
			t.Fatal(err)
		}
		cols := []string{"second", "avg_latency_ms_consul", "throughput_consul", "avg_cpu_consul", "avg_memory_mb_consul", "avg_latency_ms_etcd", "throughput_etcd", "avg_cpu_etcd", "avg_memory_mb_etcd", "avg_latency_ms_etcd2", "throughput_etcd2", "avg_cpu_etcd2", "avg_memory_mb_etcd2", "avg_latency_ms_zk", "throughput_zk", "avg_cpu_zk", "avg_memory_mb_zk"}
		if !reflect.DeepEqual(fr.GetHeader(), cols) {
			t.Fatalf("expected %q, got %q", cols, fr.GetHeader())
		}
		ac, err := fr.GetColumn("avg_latency_ms_etcd")
		if err != nil {
			t.Fatal(err)
		}
		if v, err := ac.GetValue(77); !v.IsNil() || err != nil {
			t.Fatalf("expected <nil, nil>, got <%v, %v>", v.IsNil(), err)
		}
		if v, err := ac.GetValue(0); v.IsNil() || !v.EqualTo(NewStringValue("51.607760")) || err != nil {
			t.Fatalf("expected <nil, nil>, got <%v, %v>", v, err)
		}
		ac2, err := fr.GetColumn("avg_latency_ms_etcd2")
		if err != nil {
			t.Fatal(err)
		}
		if ac.Len() != ac2.Len() {
			t.Fatalf("expected equal %v != %v", ac.Len(), ac2.Len())
		}
	}
}

func TestNewFromRows(t *testing.T) {
	fr, err := NewFromCSV(nil, "testdata/bench-compared.csv")
	if err != nil {
		t.Fatal(err)
	}
	header, rows := fr.ToRows()
	fr2, err := NewFromRows(header, rows)
	if err != nil {
		t.Fatal(err)
	}
	fpath := "test.csv"
	if err := fr2.ToCSV(fpath); err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(fpath)

	fr, err = NewFromCSV(nil, fpath)
	if err != nil {
		t.Fatal(err)
	}
	cols := []string{"second", "avg_latency_ms_consul", "throughput_consul", "avg_cpu_consul", "avg_memory_mb_consul", "avg_latency_ms_etcd", "throughput_etcd", "avg_cpu_etcd", "avg_memory_mb_etcd", "avg_latency_ms_etcd2", "throughput_etcd2", "avg_cpu_etcd2", "avg_memory_mb_etcd2", "avg_latency_ms_zk", "throughput_zk", "avg_cpu_zk", "avg_memory_mb_zk"}
	if !reflect.DeepEqual(fr.GetHeader(), cols) {
		t.Fatalf("expected %q, got %q", cols, fr.GetHeader())
	}
	ac, err := fr.GetColumn("avg_latency_ms_etcd")
	if err != nil {
		t.Fatal(err)
	}
	if v, err := ac.GetValue(77); !v.IsNil() || err != nil {
		t.Fatalf("expected <nil, nil>, got <%v, %v>", v.IsNil(), err)
	}
	if v, err := ac.GetValue(0); v.IsNil() || !v.EqualTo(NewStringValue("51.607760")) || err != nil {
		t.Fatalf("expected <nil, nil>, got <%v, %v>", v, err)
	}
	ac2, err := fr.GetColumn("avg_latency_ms_etcd2")
	if err != nil {
		t.Fatal(err)
	}
	if ac.Len() != ac2.Len() {
		t.Fatalf("expected equal %v != %v", ac.Len(), ac2.Len())
	}
}

func TestSort(t *testing.T) {
	fr, err := NewFromCSV(nil, "testdata/bench-compared.csv")
	if err != nil {
		t.Fatal(err)
	}
	if err := fr.Sort("second", SortType_Number, SortOption_Descending); err != nil {
		t.Fatal(err)
	}
	fpath := "test.csv"
	if err := fr.ToCSV(fpath); err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(fpath)
}
<commit_msg>fix govet<commit_after>package dataframe

import (
	"fmt"
	"os"
	"reflect"
	"testing"
)

func TestFrame(t *testing.T) {
	c1 := NewColumn("second1")
	for i := 0; i < 100; i++ {
		d := c1.PushBack(NewStringValue(fmt.Sprintf("%d", i)))
		if i+1 != d {
			t.Fatalf("expected %d, got %d", i+1, d)
		}
	}
	c1.UpdateHeader("aaa")
	if c1.GetHeader() != "aaa" {
		t.Fatalf("expected 'aaa', got %v", c1.GetHeader())
	}
	c1.UpdateHeader("second1")
	if c1.GetHeader() != "second1" {
		t.Fatalf("expected 'second1', got %v", c1.GetHeader())
	}

	c2 := NewColumn("second2")
	for i := 0; i < 100; i++ {
		d := c2.PushBack(NewStringValue(fmt.Sprintf("%d", i)))
		if i+1 != d {
			t.Fatalf("expected %d, got %d", i+1, d)
		}
	}

	fr := New()
	if err := fr.AddColumn(c1); err != nil {
		t.Fatal(err)
	}
	if err := fr.AddColumn(c1); err == nil {
		t.Fatal("expected error")
	}
	if err := fr.AddColumn(c2); err != nil {
		t.Fatal(err)
	}
	if err := fr.AddColumn(c2); err == nil {
		t.Fatal("expected error")
	}

	if c, err := fr.GetColumn("second1"); c == nil || err != nil {
		t.Fatal(err)
	}
	if c, err := fr.GetColumn("second2"); c == nil || err != nil {
		t.Fatal(err)
	}

	if err := fr.UpdateHeader("second2", "aaa"); err != nil {
		t.Fatal(err)
	}
	if c, err := fr.GetColumn("aaa"); c == nil || err != nil {
		t.Fatal(err)
	}
	if hs := fr.GetHeader(); !reflect.DeepEqual(hs, []string{"second1", "aaa"}) {
		t.Fatalf("expected equal, got %q != %q", hs, []string{"second1", "aaa"})
	}

	if err := fr.UpdateHeader("aaa", "second2"); err != nil {
		t.Fatal(err)
	}
	if hs := fr.GetHeader(); !reflect.DeepEqual(hs, []string{"second1", "second2"}) {
		t.Fatalf("expected equal, got %q != %q", hs, []string{"second1", "second2"})
	}
	if ok := fr.DeleteColumn("second1"); !ok {
		t.Fatalf("expected 'true', got %v", ok)
	}
	if cd := fr.GetColumnNumber(); cd != 1 {
		t.Fatalf("expected 1, got %v", cd)
	}
	if ok := fr.DeleteColumn("second1"); ok {
		t.Fatalf("expected 'false', got %v", ok)
	}
	if c, err := fr.GetColumn("second1"); c != nil || err == nil {
		t.Fatalf("expected <nil, 'second1 does not exist'>, but <%v, %v>", c, err)
	}
	if ok := fr.DeleteColumn("second2"); !ok {
		t.Fatalf("expected 'true', got %v", ok)
	}
	if cd := fr.GetColumnNumber(); cd != 0 {
		t.Fatalf("expected 0, got %v", cd)
	}
}

func TestNewFromCSV(t *testing.T) {
	if _, err := NewFromCSV([]string{"second"}, "testdata/bench-compared.csv"); err == nil {
		t.Fatal("expected error, got nil")
	}
	fr, err := NewFromCSV(nil, "testdata/bench-compared.csv")
	if err != nil {
		t.Fatal(err)
	}
	cols := []string{"second", "avg_latency_ms_consul", "throughput_consul", "avg_cpu_consul", "avg_memory_mb_consul", "avg_latency_ms_etcd", "throughput_etcd", "avg_cpu_etcd", "avg_memory_mb_etcd", "avg_latency_ms_etcd2", "throughput_etcd2", "avg_cpu_etcd2", "avg_memory_mb_etcd2", "avg_latency_ms_zk", "throughput_zk", "avg_cpu_zk", "avg_memory_mb_zk"}
	if !reflect.DeepEqual(fr.GetHeader(), cols) {
		t.Fatalf("expected %q, got %q", cols, fr.GetHeader())
	}
	ac, err := fr.GetColumn("avg_latency_ms_etcd")
	if err != nil {
		t.Fatal(err)
	}
	if v, err := ac.GetValue(77); !v.IsNil() || err != nil {
		t.Fatalf("expected <nil, nil>, got <%v, %v>", v.IsNil(), err)
	}
	if v, err := ac.GetValue(0); v.IsNil() || !v.EqualTo(NewStringValue("51.607760")) || err != nil {
		t.Fatalf("expected <nil, nil>, got <%v, %v>", v, err)
	}
	ac2, err := fr.GetColumn("avg_latency_ms_etcd2")
	if err != nil {
		t.Fatal(err)
	}
	if ac.Len() != ac2.Len() {
		t.Fatalf("expected equal %v != %v", ac.Len(), ac2.Len())
	}

	fpath := "test.csv"
	if err := fr.ToCSV(fpath); err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(fpath)

	{
		if _, err := NewFromCSV([]string{"second"}, fpath); err == nil {
			t.Fatal("expected error, got nil")
		}
		fr, err := NewFromCSV(nil, fpath)
		if err != nil {
			t.Fatal(err)
		}
		cols := []string{"second", "avg_latency_ms_consul", "throughput_consul", "avg_cpu_consul", "avg_memory_mb_consul", "avg_latency_ms_etcd", "throughput_etcd", "avg_cpu_etcd", "avg_memory_mb_etcd", "avg_latency_ms_etcd2", "throughput_etcd2", "avg_cpu_etcd2", "avg_memory_mb_etcd2", "avg_latency_ms_zk", "throughput_zk", "avg_cpu_zk", "avg_memory_mb_zk"}
		if !reflect.DeepEqual(fr.GetHeader(), cols) {
			t.Fatalf("expected %q, got %q", cols, fr.GetHeader())
		}
		ac, err := fr.GetColumn("avg_latency_ms_etcd")
		if err != nil {
			t.Fatal(err)
		}
		if v, err := ac.GetValue(77); !v.IsNil() || err != nil {
			t.Fatalf("expected <nil, nil>, got <%v, %v>", v.IsNil(), err)
		}
		if v, err := ac.GetValue(0); v.IsNil() || !v.EqualTo(NewStringValue("51.607760")) || err != nil {
			t.Fatalf("expected <nil, nil>, got <%v, %v>", v, err)
		}
		ac2, err := fr.GetColumn("avg_latency_ms_etcd2")
		if err != nil {
			t.Fatal(err)
		}
		if ac.Len() != ac2.Len() {
			t.Fatalf("expected equal %v != %v", ac.Len(), ac2.Len())
		}
	}
}

func TestNewFromRows(t *testing.T) {
	fr, err := NewFromCSV(nil, "testdata/bench-compared.csv")
	if err != nil {
		t.Fatal(err)
	}
	header, rows := fr.ToRows()
	fr2, err := NewFromRows(header, rows)
	if err != nil {
		t.Fatal(err)
	}
	fpath := "test.csv"
	if err := fr2.ToCSV(fpath); err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(fpath)

	fr, err = NewFromCSV(nil, fpath)
	if err != nil {
		t.Fatal(err)
	}
	cols := []string{"second", "avg_latency_ms_consul", "throughput_consul", "avg_cpu_consul", "avg_memory_mb_consul", "avg_latency_ms_etcd", "throughput_etcd", "avg_cpu_etcd", "avg_memory_mb_etcd", "avg_latency_ms_etcd2", "throughput_etcd2", "avg_cpu_etcd2", "avg_memory_mb_etcd2", "avg_latency_ms_zk", "throughput_zk", "avg_cpu_zk", "avg_memory_mb_zk"}
	if !reflect.DeepEqual(fr.GetHeader(), cols) {
		t.Fatalf("expected %q, got %q", cols, fr.GetHeader())
	}
	ac, err := fr.GetColumn("avg_latency_ms_etcd")
	if err != nil {
		t.Fatal(err)
	}
	if v, err := ac.GetValue(77); !v.IsNil() || err != nil {
		t.Fatalf("expected <nil, nil>, got <%v, %v>", v.IsNil(), err)
	}
	if v, err := ac.GetValue(0); v.IsNil() || !v.EqualTo(NewStringValue("51.607760")) || err != nil {
		t.Fatalf("expected <nil, nil>, got <%v, %v>", v, err)
	}
	ac2, err := fr.GetColumn("avg_latency_ms_etcd2")
	if err != nil {
		t.Fatal(err)
	}
	if ac.Len() != ac2.Len() {
		t.Fatalf("expected equal %v != %v", ac.Len(), ac2.Len())
	}
}

func TestSort(t *testing.T) {
	fr, err := NewFromCSV(nil, "testdata/bench-compared.csv")
	if err != nil {
		t.Fatal(err)
	}
	if err := fr.Sort("second", SortType_Number, SortOption_Descending); err != nil {
		t.Fatal(err)
	}
	fpath := "test.csv"
	if err := fr.ToCSV(fpath); err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(fpath)
}
<|endoftext|>