<commit_before>package speed

import (
	"errors"
	"io"
	"os"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/performancecopilot/speed/bytebuffer"
)

// byte lengths of different components in an mmv file
const (
	HeaderLength         = 40
	TocLength            = 16
	MetricLength         = 104
	ValueLength          = 32
	InstanceLength       = 80
	InstanceDomainLength = 32
	StringBlockLength    = 256
)

// MaxMetricNameLength is the maximum length for a metric name
const MaxMetricNameLength = 63

// MaxDataValueSize is the maximum byte length for a stored metric value, unless it is a string
const MaxDataValueSize = 16

// Writer defines the interface of a MMV file writer's properties
type Writer interface {
	io.Writer

	// a writer must contain a registry of metrics and instance domains
	Registry() Registry

	// writes an mmv file
	Start() error

	// stops writing and cleans up
	Stop() error

	// adds a metric to be written
	Register(Metric)

	// adds an instance domain to be written
	RegisterIndom(InstanceDomain)

	// adds metric and instance domain from a string
	RegisterString(s string)

	// update a metric
	Update(Metric, interface{})
}

func mmvFileLocation(name string) (string, error) {
	if strings.ContainsRune(name, os.PathSeparator) {
		return "", errors.New("name cannot have path separator")
	}

	tdir, present := config["PCP_TMP_DIR"]
	var loc string
	if present {
		loc = path.Join(rootPath, tdir)
	} else {
		loc = os.TempDir()
	}

	return path.Join(loc, "mmv", name), nil
}

// PCPClusterIDBitLength is the bit length of the cluster id
// for a set of PCP metrics
const PCPClusterIDBitLength = 12

// MMVFlag represents an enumerated type to represent mmv flag values
type MMVFlag int

// values for MMVFlag
const (
	NoPrefixFlag MMVFlag = 1 << iota
	ProcessFlag
	SentinelFlag
)

//go:generate stringer -type=MMVFlag

// PCPWriter implements a writer that can write PCP compatible MMV files
type PCPWriter struct {
	sync.Mutex
	loc       string       // absolute location of the mmv file
	clusterID uint32       // cluster identifier for the writer
	flag      MMVFlag      // write flag
	r         *PCPRegistry // current registry
}

// NewPCPWriter initializes a new PCPWriter object
func NewPCPWriter(name string, flag MMVFlag) (*PCPWriter, error) {
	fileLocation, err := mmvFileLocation(name)
	if err != nil {
		return nil, err
	}

	return &PCPWriter{
		loc:       fileLocation,
		r:         NewPCPRegistry(),
		clusterID: getHash(name, PCPClusterIDBitLength),
		flag:      flag,
	}, nil
}

// Registry returns a writer's registry
func (w *PCPWriter) Registry() Registry {
	return w.r
}

func (w *PCPWriter) tocCount() int {
	ans := 2

	if w.Registry().InstanceCount() > 0 {
		ans += 2
	}

	return ans
}

// Length returns the byte length of data in the mmv file written by the current writer
func (w *PCPWriter) Length() int {
	return HeaderLength +
		(w.tocCount() * TocLength) +
		(w.Registry().InstanceCount() * InstanceLength) +
		(w.Registry().InstanceDomainCount() * InstanceDomainLength) +
		(w.Registry().MetricCount() * (MetricLength + ValueLength))
}

func (w *PCPWriter) initializeOffsets() {
	indomoffset := HeaderLength + TocLength*w.tocCount()
	instanceoffset := indomoffset + InstanceDomainLength*w.r.InstanceDomainCount()
	metricsoffset := instanceoffset + InstanceLength*w.r.InstanceCount()
	valuesoffset := metricsoffset + MetricLength*w.r.MetricCount()

	w.r.indomoffset = indomoffset
	w.r.instanceoffset = instanceoffset
	w.r.metricsoffset = metricsoffset
	w.r.valuesoffset = valuesoffset

	for _, indom := range w.r.instanceDomains {
		indom.setOffset(indomoffset)
		indom.instanceOffset = instanceoffset
		indomoffset += InstanceDomainLength

		for _, i := range indom.instances {
			i.setOffset(instanceoffset)
			instanceoffset += InstanceLength
		}
	}

	for _, metric := range w.r.metrics {
		metric.desc.setOffset(metricsoffset)
		metricsoffset += MetricLength
		metric.setOffset(valuesoffset)
		valuesoffset += ValueLength
	}

	// TODO: string offsets
}

func (w *PCPWriter) writeHeaderBlock(buffer bytebuffer.Buffer) (gen2offset int, generation int64) {
	// tag
	buffer.WriteString("MMV")
	buffer.SetPos(buffer.Pos() + 1) // extra null byte is needed and \0 isn't a valid escape character in go

	// version
	buffer.WriteUint32(1)

	// generation
	generation = time.Now().Unix()
	buffer.WriteInt64(generation)

	gen2offset = buffer.Pos()

	buffer.WriteInt64(0)

	// tocCount
	buffer.WriteInt(w.tocCount())

	// flag mask
	buffer.WriteInt(int(w.flag))

	// process identifier
	buffer.WriteInt(os.Getpid())

	// cluster identifier
	buffer.WriteUint32(w.clusterID)

	return
}

func (w *PCPWriter) writeTocBlock(buffer bytebuffer.Buffer) {
	tocpos := HeaderLength

	// instance domains toc
	if w.Registry().InstanceDomainCount() > 0 {
		buffer.SetPos(tocpos)
		buffer.WriteInt(1) // Instance Domain identifier
		buffer.WriteInt(w.Registry().InstanceDomainCount())
		buffer.WriteUint64(uint64(w.r.indomoffset))
		tocpos += TocLength
	}

	// instances toc
	if w.Registry().InstanceCount() > 0 {
		buffer.SetPos(tocpos)
		buffer.WriteInt(2) // Instance identifier
		buffer.WriteInt(w.Registry().InstanceCount())
		buffer.WriteUint64(uint64(w.r.instanceoffset))
		tocpos += TocLength
	}

	metricsoffset, valuesoffset := w.r.metricsoffset, w.r.valuesoffset
	if w.Registry().MetricCount() == 0 {
		metricsoffset, valuesoffset = 0, 0
	}

	// metrics and values toc
	buffer.SetPos(tocpos)
	buffer.WriteInt(3) // Metrics identifier
	buffer.WriteInt(w.Registry().MetricCount())
	buffer.WriteUint64(uint64(metricsoffset))
	tocpos += TocLength

	buffer.SetPos(tocpos)
	buffer.WriteInt(4) // Values identifier
	buffer.WriteInt(w.Registry().MetricCount())
	buffer.WriteUint64(uint64(valuesoffset))
	tocpos += TocLength

	// TODO: strings toc
}

func (w *PCPWriter) writeInstanceAndInstanceDomainBlock(buffer bytebuffer.Buffer) {
	for _, indom := range w.r.instanceDomains {
		buffer.SetPos(indom.Offset())
		buffer.WriteUint32(indom.ID())
		buffer.WriteInt(indom.InstanceCount())
		buffer.WriteInt64(int64(indom.instanceOffset))
		// TODO: write indom string descriptions offsets

		for _, i := range indom.instances {
			buffer.SetPos(i.Offset())
			buffer.WriteInt64(int64(indom.Offset()))
			buffer.WriteInt(0)
			buffer.WriteUint32(i.id)
			buffer.WriteString(i.name)
		}
	}
}

func (w *PCPWriter) writeMetricDesc(desc *pcpMetricDesc, buffer bytebuffer.Buffer) {
	pos := desc.Offset()
	buffer.SetPos(pos)

	buffer.WriteString(desc.name)
	buffer.Write([]byte{0})
	buffer.SetPos(pos + MaxMetricNameLength + 1)
	buffer.WriteUint32(desc.id)
	buffer.WriteInt32(int32(desc.t))
	buffer.WriteInt32(int32(desc.sem))
	buffer.WriteUint32(desc.u.PMAPI())
	if desc.indom != nil {
		buffer.WriteUint32(desc.indom.ID())
	} else {
		buffer.WriteInt32(-1)
	}
	buffer.WriteInt(0)
	// TODO: write string descriptions
}

func (w *PCPWriter) writeMetricVal(m *PCPMetric, buffer bytebuffer.Buffer) {
	pos := m.Offset()
	buffer.SetPos(pos)

	m.desc.t.WriteVal(m.val, buffer)

	buffer.SetPos(pos + MaxDataValueSize)
	buffer.WriteInt64(int64(m.desc.Offset()))
	if m.desc.indom != nil {
		buffer.WriteInt64(int64(m.desc.indom.(*PCPInstanceDomain).instanceOffset))
	} else {
		buffer.WriteInt64(0)
	}
}

func (w *PCPWriter) writeMetricsAndValuesBlock(buffer bytebuffer.Buffer) {
	for _, metric := range w.r.metrics {
		w.writeMetricDesc(metric.desc, buffer)
		w.writeMetricVal(metric, buffer)
	}
}

// fillData will fill a byte slice with the mmv file
// data as long as something doesn't go wrong
func (w *PCPWriter) fillData(buffer bytebuffer.Buffer) error {
	gen2offset, generation := w.writeHeaderBlock(buffer)
	w.writeTocBlock(buffer)
	w.writeInstanceAndInstanceDomainBlock(buffer)
	w.writeMetricsAndValuesBlock(buffer)
	// TODO: write strings block

	buffer.SetPos(gen2offset)
	buffer.WriteUint64(uint64(generation))

	return nil
}

func (w *PCPWriter) Write(data []byte) (int, error) {
	f, err := os.Create(w.loc)
	if err != nil {
		panic(err)
	}

	return f.Write(data)
}

// Start dumps existing registry data
func (w *PCPWriter) Start() {
	w.Lock()
	defer w.Unlock()

	l := w.Length()

	w.initializeOffsets()
	buffer := bytebuffer.NewByteBuffer(l)
	w.fillData(buffer)

	w.Write(buffer.Bytes())
	w.r.mapped = true
}

// Stop removes existing mapping and cleans up
func (w *PCPWriter) Stop() {
	w.Lock()
	defer w.Unlock()

	w.r.mapped = false
}

// Register is simply a shorthand for Registry().AddMetric
func (w *PCPWriter) Register(m Metric) { w.Registry().AddMetric(m) }

// RegisterIndom is simply a shorthand for Registry().AddInstanceDomain
func (w *PCPWriter) RegisterIndom(indom InstanceDomain) { w.Registry().AddInstanceDomain(indom) }

// RegisterString is simply a shorthand for Registry().AddMetricByString
func (w *PCPWriter) RegisterString(str string, initialval interface{}, s MetricSemantics, t MetricType, u MetricUnit) {
	w.Registry().AddMetricByString(str, initialval, s, t, u)
}

// Update is simply a shorthand for Registry().UpdateMetric
func (w *PCPWriter) Update(m Metric, val interface{}) { w.Registry().UpdateMetric(m, val) }
<commit_msg>writer: start writing strings<commit_after>package speed

import (
	"errors"
	"io"
	"os"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/performancecopilot/speed/bytebuffer"
)

// byte lengths of different components in an mmv file
const (
	HeaderLength         = 40
	TocLength            = 16
	MetricLength         = 104
	ValueLength          = 32
	InstanceLength       = 80
	InstanceDomainLength = 32
	StringBlockLength    = 256
)

// MaxMetricNameLength is the maximum length for a metric name
const MaxMetricNameLength = 63

// MaxDataValueSize is the maximum byte length for a stored metric value, unless it is a string
const MaxDataValueSize = 16

// Writer defines the interface of a MMV file writer's properties
type Writer interface {
	io.Writer

	// a writer must contain a registry of metrics and instance domains
	Registry() Registry

	// writes an mmv file
	Start() error

	// stops writing and cleans up
	Stop() error

	// adds a metric to be written
	Register(Metric)

	// adds an instance domain to be written
	RegisterIndom(InstanceDomain)

	// adds metric and instance domain from a string
	RegisterString(s string)

	// update a metric
	Update(Metric, interface{})
}

func mmvFileLocation(name string) (string, error) {
	if strings.ContainsRune(name, os.PathSeparator) {
		return "", errors.New("name cannot have path separator")
	}

	tdir, present := config["PCP_TMP_DIR"]
	var loc string
	if present {
		loc = path.Join(rootPath, tdir)
	} else {
		loc = os.TempDir()
	}

	return path.Join(loc, "mmv", name), nil
}

// PCPClusterIDBitLength is the bit length of the cluster id
// for a set of PCP metrics
const PCPClusterIDBitLength = 12

// MMVFlag represents an enumerated type to represent mmv flag values
type MMVFlag int

// values for MMVFlag
const (
	NoPrefixFlag MMVFlag = 1 << iota
	ProcessFlag
	SentinelFlag
)

//go:generate stringer -type=MMVFlag

// PCPWriter implements a writer that can write PCP compatible MMV files
type PCPWriter struct {
	sync.Mutex
	loc       string       // absolute location of the mmv file
	clusterID uint32       // cluster identifier for the writer
	flag      MMVFlag      // write flag
	r         *PCPRegistry // current registry
}

// NewPCPWriter initializes a new PCPWriter object
func NewPCPWriter(name string, flag MMVFlag) (*PCPWriter, error) {
	fileLocation, err := mmvFileLocation(name)
	if err != nil {
		return nil, err
	}

	return &PCPWriter{
		loc:       fileLocation,
		r:         NewPCPRegistry(),
		clusterID: getHash(name, PCPClusterIDBitLength),
		flag:      flag,
	}, nil
}

// Registry returns a writer's registry
func (w *PCPWriter) Registry() Registry {
	return w.r
}

func (w *PCPWriter) tocCount() int {
	ans := 2

	if w.Registry().InstanceCount() > 0 {
		ans += 2
	}

	return ans
}

// Length returns the byte length of data in the mmv file written by the current writer
func (w *PCPWriter) Length() int {
	return HeaderLength +
		(w.tocCount() * TocLength) +
		(w.Registry().InstanceCount() * InstanceLength) +
		(w.Registry().InstanceDomainCount() * InstanceDomainLength) +
		(w.Registry().MetricCount() * (MetricLength + ValueLength)) +
		(w.Registry().StringCount() * StringBlockLength)
}

func (w *PCPWriter) initializeOffsets() {
	indomoffset := HeaderLength + TocLength*w.tocCount()
	instanceoffset := indomoffset + InstanceDomainLength*w.r.InstanceDomainCount()
	metricsoffset := instanceoffset + InstanceLength*w.r.InstanceCount()
	valuesoffset := metricsoffset + MetricLength*w.r.MetricCount()
	stringsoffset := valuesoffset + ValueLength*w.r.MetricCount()

	w.r.indomoffset = indomoffset
	w.r.instanceoffset = instanceoffset
	w.r.metricsoffset = metricsoffset
	w.r.valuesoffset = valuesoffset
	w.r.stringsoffset = stringsoffset

	for _, indom := range w.r.instanceDomains {
		indom.setOffset(indomoffset)
		indom.instanceOffset = instanceoffset
		indomoffset += InstanceDomainLength

		for _, i := range indom.instances {
			i.setOffset(instanceoffset)
			instanceoffset += InstanceLength
		}

		if indom.shortHelpText.val != "" {
			indom.shortHelpText.setOffset(stringsoffset)
			stringsoffset += StringBlockLength
		}

		if indom.longHelpText.val != "" {
			indom.longHelpText.setOffset(stringsoffset)
			stringsoffset += StringBlockLength
		}
	}

	for _, metric := range w.r.metrics {
		metric.desc.setOffset(metricsoffset)
		metricsoffset += MetricLength
		metric.setOffset(valuesoffset)
		valuesoffset += ValueLength

		if metric.desc.shortDescription.val != "" {
			metric.desc.shortDescription.setOffset(stringsoffset)
			stringsoffset += StringBlockLength
		}

		if metric.desc.longDescription.val != "" {
			metric.desc.longDescription.setOffset(stringsoffset)
			stringsoffset += StringBlockLength
		}
	}
}

func (w *PCPWriter) writeHeaderBlock(buffer bytebuffer.Buffer) (gen2offset int, generation int64) {
	// tag
	buffer.WriteString("MMV")
	buffer.SetPos(buffer.Pos() + 1) // extra null byte is needed and \0 isn't a valid escape character in go

	// version
	buffer.WriteUint32(1)

	// generation
	generation = time.Now().Unix()
	buffer.WriteInt64(generation)

	gen2offset = buffer.Pos()

	buffer.WriteInt64(0)

	// tocCount
	buffer.WriteInt(w.tocCount())

	// flag mask
	buffer.WriteInt(int(w.flag))

	// process identifier
	buffer.WriteInt(os.Getpid())

	// cluster identifier
	buffer.WriteUint32(w.clusterID)

	return
}

func (w *PCPWriter) writeTocBlock(buffer bytebuffer.Buffer) {
	tocpos := HeaderLength

	// instance domains toc
	if w.Registry().InstanceDomainCount() > 0 {
		buffer.SetPos(tocpos)
		buffer.WriteInt(1) // Instance Domain identifier
		buffer.WriteInt(w.Registry().InstanceDomainCount())
		buffer.WriteUint64(uint64(w.r.indomoffset))
		tocpos += TocLength
	}

	// instances toc
	if w.Registry().InstanceCount() > 0 {
		buffer.SetPos(tocpos)
		buffer.WriteInt(2) // Instance identifier
		buffer.WriteInt(w.Registry().InstanceCount())
		buffer.WriteUint64(uint64(w.r.instanceoffset))
		tocpos += TocLength
	}

	metricsoffset, valuesoffset := w.r.metricsoffset, w.r.valuesoffset
	if w.Registry().MetricCount() == 0 {
		metricsoffset, valuesoffset = 0, 0
	}

	// metrics and values toc
	buffer.SetPos(tocpos)
	buffer.WriteInt(3) // Metrics identifier
	buffer.WriteInt(w.Registry().MetricCount())
	buffer.WriteUint64(uint64(metricsoffset))
	tocpos += TocLength

	buffer.SetPos(tocpos)
	buffer.WriteInt(4) // Values identifier
	buffer.WriteInt(w.Registry().MetricCount())
	buffer.WriteUint64(uint64(valuesoffset))
	tocpos += TocLength

	// TODO: strings toc
}

func (w *PCPWriter) writeInstanceAndInstanceDomainBlock(buffer bytebuffer.Buffer) {
	for _, indom := range w.r.instanceDomains {
		buffer.SetPos(indom.Offset())
		buffer.WriteUint32(indom.ID())
		buffer.WriteInt(indom.InstanceCount())
		buffer.WriteInt64(int64(indom.instanceOffset))
		// TODO: write indom string descriptions offsets

		for _, i := range indom.instances {
			buffer.SetPos(i.Offset())
			buffer.WriteInt64(int64(indom.Offset()))
			buffer.WriteInt(0)
			buffer.WriteUint32(i.id)
			buffer.WriteString(i.name)
		}

		so, lo := indom.shortHelpText.offset, indom.longHelpText.offset
		buffer.WriteInt64(int64(so))
		buffer.WriteInt64(int64(lo))

		if so != 0 {
			buffer.SetPos(so)
			buffer.WriteString(indom.shortHelpText.val)
		}

		if lo != 0 {
			buffer.SetPos(lo)
			buffer.WriteString(indom.longHelpText.val)
		}
	}
}

func (w *PCPWriter) writeMetricDesc(desc *pcpMetricDesc, buffer bytebuffer.Buffer) {
	pos := desc.Offset()
	buffer.SetPos(pos)

	buffer.WriteString(desc.name)
	buffer.Write([]byte{0})
	buffer.SetPos(pos + MaxMetricNameLength + 1)
	buffer.WriteUint32(desc.id)
	buffer.WriteInt32(int32(desc.t))
	buffer.WriteInt32(int32(desc.sem))
	buffer.WriteUint32(desc.u.PMAPI())
	if desc.indom != nil {
		buffer.WriteUint32(desc.indom.ID())
	} else {
		buffer.WriteInt32(-1)
	}
	buffer.WriteInt(0)

	so, lo := desc.shortDescription.offset, desc.longDescription.offset
	buffer.WriteInt64(int64(so))
	buffer.WriteInt64(int64(lo))

	if so != 0 {
		buffer.SetPos(so)
		buffer.WriteString(desc.shortDescription.val)
	}

	if lo != 0 {
		buffer.SetPos(lo)
		buffer.WriteString(desc.longDescription.val)
	}
}

func (w *PCPWriter) writeMetricVal(m *PCPMetric, buffer bytebuffer.Buffer) {
	pos := m.Offset()
	buffer.SetPos(pos)

	m.desc.t.WriteVal(m.val, buffer)

	buffer.SetPos(pos + MaxDataValueSize)
	buffer.WriteInt64(int64(m.desc.Offset()))
	if m.desc.indom != nil {
		buffer.WriteInt64(int64(m.desc.indom.(*PCPInstanceDomain).instanceOffset))
	} else {
		buffer.WriteInt64(0)
	}
}

func (w *PCPWriter) writeMetricsAndValuesBlock(buffer bytebuffer.Buffer) {
	for _, metric := range w.r.metrics {
		w.writeMetricDesc(metric.desc, buffer)
		w.writeMetricVal(metric, buffer)
	}
}

// fillData will fill a byte slice with the mmv file
// data as long as something doesn't go wrong
func (w *PCPWriter) fillData(buffer bytebuffer.Buffer) error {
	gen2offset, generation := w.writeHeaderBlock(buffer)
	w.writeTocBlock(buffer)
	w.writeInstanceAndInstanceDomainBlock(buffer)
	w.writeMetricsAndValuesBlock(buffer)
	// TODO: write strings block

	buffer.SetPos(gen2offset)
	buffer.WriteUint64(uint64(generation))

	return nil
}

func (w *PCPWriter) Write(data []byte) (int, error) {
	f, err := os.Create(w.loc)
	if err != nil {
		panic(err)
	}

	return f.Write(data)
}

// Start dumps existing registry data
func (w *PCPWriter) Start() {
	w.Lock()
	defer w.Unlock()

	l := w.Length()

	w.initializeOffsets()
	buffer := bytebuffer.NewByteBuffer(l)
	w.fillData(buffer)

	w.Write(buffer.Bytes())
	w.r.mapped = true
}

// Stop removes existing mapping and cleans up
func (w *PCPWriter) Stop() {
	w.Lock()
	defer w.Unlock()

	w.r.mapped = false
}

// Register is simply a shorthand for Registry().AddMetric
func (w *PCPWriter) Register(m Metric) { w.Registry().AddMetric(m) }

// RegisterIndom is simply a shorthand for Registry().AddInstanceDomain
func (w *PCPWriter) RegisterIndom(indom InstanceDomain) { w.Registry().AddInstanceDomain(indom) }

// RegisterString is simply a shorthand for Registry().AddMetricByString
func (w *PCPWriter) RegisterString(str string, initialval interface{}, s MetricSemantics, t MetricType, u MetricUnit) {
	w.Registry().AddMetricByString(str, initialval, s, t, u)
}

// Update is simply a shorthand for Registry().UpdateMetric
func (w *PCPWriter) Update(m Metric, val interface{}) { w.Registry().UpdateMetric(m, val) }
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/types"

	"github.com/spf13/cobra"
)

var (
	renterExportCmd = &cobra.Command{
		Use:   "export",
		Short: "export renter data to various formats",
		Long:  "Export renter data in various formats.",
		// Run field not provided; export requires a subcommand.
	}

	renterExportContractsCmd = &cobra.Command{
		Use:   "rankings-data [destination]",
		Short: "export the renter's contracts for import to `https://rankings.sia.tech/`",
		Long:  "Export the renter's current contract set in JSON format to the specified " +
			"file. Intended for upload to `https://rankings.sia.tech/`.",
		Run:   wrap(renterexportcontractscmd),
	}
)

// renterexportcontractscmd is the handler for the command `siac renter export contracts`.
// Exports the current contract set to JSON.
func renterexportcontractscmd(destination string) {
	var cs api.RenterContracts
	err := getAPI("/renter/contracts", &cs)
	if err != nil {
		die("Could not retrieve contracts:", err)
	}
	var contractTxns []types.Transaction
	for _, c := range cs.Contracts {
		contractTxns = append(contractTxns, c.LastTransaction)
	}
	destination = abs(destination)
	file, err := os.Create(destination)
	if err != nil {
		die("Could not export to file:", err)
	}
	err = json.NewEncoder(file).Encode(contractTxns)
	if err != nil {
		die("Could not export to file:", err)
	}
	fmt.Println("Exported contract data to", destination)
}
<commit_msg>Fix indentation.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/types"

	"github.com/spf13/cobra"
)

var (
	renterExportCmd = &cobra.Command{
		Use:   "export",
		Short: "export renter data to various formats",
		Long:  "Export renter data in various formats.",
		// Run field not provided; export requires a subcommand.
	}

	renterExportContractsCmd = &cobra.Command{
		Use:   "rankings-data [destination]",
		Short: "export the renter's contracts for import to `https://rankings.sia.tech/`",
		Long: "Export the renter's current contract set in JSON format to the specified " +
			"file. Intended for upload to `https://rankings.sia.tech/`.",
		Run: wrap(renterexportcontractscmd),
	}
)

// renterexportcontractscmd is the handler for the command `siac renter export contracts`.
// Exports the current contract set to JSON.
func renterexportcontractscmd(destination string) {
	var cs api.RenterContracts
	err := getAPI("/renter/contracts", &cs)
	if err != nil {
		die("Could not retrieve contracts:", err)
	}
	var contractTxns []types.Transaction
	for _, c := range cs.Contracts {
		contractTxns = append(contractTxns, c.LastTransaction)
	}
	destination = abs(destination)
	file, err := os.Create(destination)
	if err != nil {
		die("Could not export to file:", err)
	}
	err = json.NewEncoder(file).Encode(contractTxns)
	if err != nil {
		die("Could not export to file:", err)
	}
	fmt.Println("Exported contract data to", destination)
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

var cmdClean = &Command{
	UsageLine: "clean [-i] [-r] [-n] [-x] [packages]",
	Short:     "remove object files",
	Long: `
Clean removes object files from package source directories.
The go command builds most objects in a temporary directory,
so go clean is mainly concerned with object files left by other
tools or by manual invocations of go build.

Specifically, clean removes the following files from each of the
source directories corresponding to the import paths:

	_obj/            old object directory, left from Makefiles
	_test/           old test directory, left from Makefiles
	_testmain.go     old gotest file, left from Makefiles
	test.out         old test log, left from Makefiles
	build.out        old test log, left from Makefiles
	*.[568ao]        object files, left from Makefiles

	DIR(.exe)        from go build
	DIR.test(.exe)   from go test -c
	MAINFILE(.exe)   from go build MAINFILE.go
	*.so             from SWIG

In the list, DIR represents the final path element of the
directory, and MAINFILE is the base name of any Go source
file in the directory that is not included when building
the package.

The -i flag causes clean to remove the corresponding installed
archive or binary (what 'go install' would create).

The -n flag causes clean to print the remove commands it would execute,
but not run them.

The -r flag causes clean to be applied recursively to all the
dependencies of the packages named by the import paths.

The -x flag causes clean to print remove commands as it executes them.

For more about specifying packages, see 'go help packages'.
	`,
}

var cleanI bool // clean -i flag
var cleanN bool // clean -n flag
var cleanR bool // clean -r flag
var cleanX bool // clean -x flag

func init() {
	// break init cycle
	cmdClean.Run = runClean

	cmdClean.Flag.BoolVar(&cleanI, "i", false, "")
	cmdClean.Flag.BoolVar(&cleanN, "n", false, "")
	cmdClean.Flag.BoolVar(&cleanR, "r", false, "")
	cmdClean.Flag.BoolVar(&cleanX, "x", false, "")
}

func runClean(cmd *Command, args []string) {
	for _, pkg := range packagesAndErrors(args) {
		clean(pkg)
	}
}

var cleaned = map[*Package]bool{}

// TODO: These are dregs left by Makefile-based builds.
// Eventually, can stop deleting these.
var cleanDir = map[string]bool{
	"_test": true,
	"_obj":  true,
}

var cleanFile = map[string]bool{
	"_testmain.go": true,
	"test.out":     true,
	"build.out":    true,
	"a.out":        true,
}

var cleanExt = map[string]bool{
	".5":  true,
	".6":  true,
	".8":  true,
	".a":  true,
	".o":  true,
	".so": true,
}

func clean(p *Package) {
	if cleaned[p] {
		return
	}
	cleaned[p] = true

	if p.Dir == "" {
		errorf("can't load package: %v", p.Error)
		return
	}
	dirs, err := ioutil.ReadDir(p.Dir)
	if err != nil {
		errorf("go clean %s: %v", p.Dir, err)
		return
	}

	var b builder
	b.print = fmt.Print

	packageFile := map[string]bool{}
	if p.Name != "main" {
		// Record which files are not in package main.
		// The others are.
		keep := func(list []string) {
			for _, f := range list {
				packageFile[f] = true
			}
		}
		keep(p.GoFiles)
		keep(p.CgoFiles)
		keep(p.TestGoFiles)
		keep(p.XTestGoFiles)
	}

	_, elem := filepath.Split(p.Dir)
	allRemove := []string{
		elem,
		elem + ".exe",
		elem + ".test",
		elem + ".test.exe",
	}
	for _, dir := range dirs {
		name := dir.Name()
		if packageFile[name] {
			continue
		}
		if !dir.IsDir() && strings.HasSuffix(name, ".go") {
			base := name[:len(name)-len(".go")]
			allRemove = append(allRemove, base, base+".exe")
		}
	}
	if cleanN || cleanX {
		b.showcmd(p.Dir, "rm -f %s", strings.Join(allRemove, " "))
	}

	toRemove := map[string]bool{}
	for _, name := range allRemove {
		toRemove[name] = true
	}
	for _, dir := range dirs {
		name := dir.Name()
		if dir.IsDir() {
			// TODO: Remove once Makefiles are forgotten.
			if cleanDir[name] {
				if cleanN || cleanX {
					b.showcmd(p.Dir, "rm -r %s", name)
					if cleanN {
						continue
					}
				}
				if err := os.RemoveAll(filepath.Join(p.Dir, name)); err != nil {
					errorf("go clean: %v", err)
				}
			}
			continue
		}

		if cleanN {
			continue
		}

		if cleanFile[name] || cleanExt[filepath.Ext(name)] || toRemove[name] {
			removeFile(filepath.Join(p.Dir, name))
		}
	}

	if cleanI && p.target != "" {
		if cleanN || cleanX {
			b.showcmd("", "rm -f %s", p.target)
		}
		if !cleanN {
			removeFile(p.target)
		}
	}

	if cleanI && p.usesSwig() {
		for _, f := range stringList(p.SwigFiles, p.SwigCXXFiles) {
			dir := p.swigDir(&buildContext)
			soname := p.swigSoname(f)
			target := filepath.Join(dir, soname)
			if cleanN || cleanX {
				b.showcmd("", "rm -f %s", target)
			}
			if !cleanN {
				removeFile(target)
			}
		}
	}

	if cleanR {
		for _, p1 := range p.imports {
			clean(p1)
		}
	}
}

// removeFile tries to remove file f, if error other than file doesn't exist
// occurs, it will report the error.
func removeFile(f string) {
	if err := os.Remove(f); err != nil && !os.IsNotExist(err) {
		errorf("go clean: %v", err)
	}
}
<commit_msg>cmd/go: only try to clean executables for package main<commit_after>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

var cmdClean = &Command{
	UsageLine: "clean [-i] [-r] [-n] [-x] [packages]",
	Short:     "remove object files",
	Long: `
Clean removes object files from package source directories.
The go command builds most objects in a temporary directory,
so go clean is mainly concerned with object files left by other
tools or by manual invocations of go build.

Specifically, clean removes the following files from each of the
source directories corresponding to the import paths:

	_obj/            old object directory, left from Makefiles
	_test/           old test directory, left from Makefiles
	_testmain.go     old gotest file, left from Makefiles
	test.out         old test log, left from Makefiles
	build.out        old test log, left from Makefiles
	*.[568ao]        object files, left from Makefiles

	DIR(.exe)        from go build
	DIR.test(.exe)   from go test -c
	MAINFILE(.exe)   from go build MAINFILE.go
	*.so             from SWIG

In the list, DIR represents the final path element of the
directory, and MAINFILE is the base name of any Go source
file in the directory that is not included when building
the package.

The -i flag causes clean to remove the corresponding installed
archive or binary (what 'go install' would create).

The -n flag causes clean to print the remove commands it would execute,
but not run them.

The -r flag causes clean to be applied recursively to all the
dependencies of the packages named by the import paths.

The -x flag causes clean to print remove commands as it executes them.

For more about specifying packages, see 'go help packages'.
	`,
}

var cleanI bool // clean -i flag
var cleanN bool // clean -n flag
var cleanR bool // clean -r flag
var cleanX bool // clean -x flag

func init() {
	// break init cycle
	cmdClean.Run = runClean

	cmdClean.Flag.BoolVar(&cleanI, "i", false, "")
	cmdClean.Flag.BoolVar(&cleanN, "n", false, "")
	cmdClean.Flag.BoolVar(&cleanR, "r", false, "")
	cmdClean.Flag.BoolVar(&cleanX, "x", false, "")
}

func runClean(cmd *Command, args []string) {
	for _, pkg := range packagesAndErrors(args) {
		clean(pkg)
	}
}

var cleaned = map[*Package]bool{}

// TODO: These are dregs left by Makefile-based builds.
// Eventually, can stop deleting these.
var cleanDir = map[string]bool{
	"_test": true,
	"_obj":  true,
}

var cleanFile = map[string]bool{
	"_testmain.go": true,
	"test.out":     true,
	"build.out":    true,
	"a.out":        true,
}

var cleanExt = map[string]bool{
	".5":  true,
	".6":  true,
	".8":  true,
	".a":  true,
	".o":  true,
	".so": true,
}

func clean(p *Package) {
	if cleaned[p] {
		return
	}
	cleaned[p] = true

	if p.Dir == "" {
		errorf("can't load package: %v", p.Error)
		return
	}
	dirs, err := ioutil.ReadDir(p.Dir)
	if err != nil {
		errorf("go clean %s: %v", p.Dir, err)
		return
	}

	var b builder
	b.print = fmt.Print

	packageFile := map[string]bool{}
	if p.Name != "main" {
		// Record which files are not in package main.
		// The others are.
		keep := func(list []string) {
			for _, f := range list {
				packageFile[f] = true
			}
		}
		keep(p.GoFiles)
		keep(p.CgoFiles)
		keep(p.TestGoFiles)
		keep(p.XTestGoFiles)
	}

	_, elem := filepath.Split(p.Dir)
	var allRemove []string

	// Remove dir-named executable only if this is package main.
	if p.Name == "main" {
		allRemove = append(allRemove,
			elem,
			elem+".exe",
		)
	}

	// Remove package test executables.
	allRemove = append(allRemove,
		elem+".test",
		elem+".test.exe",
	)

	// Remove a potental executable for each .go file in the directory that
	// is not part of the directory's package.
	for _, dir := range dirs {
		name := dir.Name()
		if packageFile[name] {
			continue
		}
		if !dir.IsDir() && strings.HasSuffix(name, ".go") {
			// TODO(adg,rsc): check that this .go file is actually
			// in "package main", and therefore capable of building
			// to an executable file.
			base := name[:len(name)-len(".go")]
			allRemove = append(allRemove, base, base+".exe")
		}
	}

	if cleanN || cleanX {
		b.showcmd(p.Dir, "rm -f %s", strings.Join(allRemove, " "))
	}

	toRemove := map[string]bool{}
	for _, name := range allRemove {
		toRemove[name] = true
	}
	for _, dir := range dirs {
		name := dir.Name()
		if dir.IsDir() {
			// TODO: Remove once Makefiles are forgotten.
			if cleanDir[name] {
				if cleanN || cleanX {
					b.showcmd(p.Dir, "rm -r %s", name)
					if cleanN {
						continue
					}
				}
				if err := os.RemoveAll(filepath.Join(p.Dir, name)); err != nil {
					errorf("go clean: %v", err)
				}
			}
			continue
		}

		if cleanN {
			continue
		}

		if cleanFile[name] || cleanExt[filepath.Ext(name)] || toRemove[name] {
			removeFile(filepath.Join(p.Dir, name))
		}
	}

	if cleanI && p.target != "" {
		if cleanN || cleanX {
			b.showcmd("", "rm -f %s", p.target)
		}
		if !cleanN {
			removeFile(p.target)
		}
	}

	if cleanI && p.usesSwig() {
		for _, f := range stringList(p.SwigFiles, p.SwigCXXFiles) {
			dir := p.swigDir(&buildContext)
			soname := p.swigSoname(f)
			target := filepath.Join(dir, soname)
			if cleanN || cleanX {
				b.showcmd("", "rm -f %s", target)
			}
			if !cleanN {
				removeFile(target)
			}
		}
	}

	if cleanR {
		for _, p1 := range p.imports {
			clean(p1)
		}
	}
}

// removeFile tries to remove file f, if error other than file doesn't exist
// occurs, it will report the error.
func removeFile(f string) {
	if err := os.Remove(f); err != nil && !os.IsNotExist(err) {
		errorf("go clean: %v", err)
	}
}
<|endoftext|>


<commit_before>package handlers

import (
	"fmt"
	"github.com/sporttech/termcolor"
	"log"
	"net/http"
	"os"
	"time"
)

type (
	loggingHandler struct {
		h http.Handler
		l *log.Logger
	}
)

func NewLoggingHandler(h http.Handler, l *log.Logger) *loggingHandler {
	if l == nil {
		l = log.New(os.Stdout, "HTTP ", 0)
	}

	return &loggingHandler{
		h: h,
		l: l,
	}
}

func (h *loggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.RequestURI == "" {
		// ignore "faked" requests
		h.h.ServeHTTP(w, r)
		return
	}

	start := time.Now()
	tw := &TransparentResponseWriter{w, http.StatusOK, 0}
	defer h.log(start, tw, r)

	h.h.ServeHTTP(tw, r)
}

func (h *loggingHandler) log(start time.Time, tw *TransparentResponseWriter, r *http.Request) {
	dt := time.Now().Sub(start)
	dtC := termcolor.Colorized(fmt.Sprintf("%v", dt), termcolor.CYAN)

	statusC := fmt.Sprintf("%v", tw.Status)
	if tw.Success() {
		statusC = termcolor.Colorized(statusC, termcolor.GREEN)
	} else if tw.Status >= 400 {
		statusC = termcolor.Colorized(statusC, termcolor.RED)
	}

	methodC := termcolor.Colorized(r.Method, termcolor.BLUE)
	requestURIC := termcolor.Colorized(r.RequestURI, termcolor.BOLD)

	sizeS := ""
	if tw.Size != 0 {
		sizeS = fmt.Sprintf(", %v bytes", tw.Size)
	}

	h.l.Printf("%v %v %v %v (%v%v)", statusC, r.RemoteAddr, methodC, requestURIC, dtC, sizeS)
}
<commit_msg>handlers/log: less bold text in the output<commit_after>package handlers

import (
	"fmt"
	"github.com/sporttech/termcolor"
	"log"
	"net/http"
	"os"
	"time"
)

type (
	loggingHandler struct {
		h http.Handler
		l *log.Logger
	}
)

func NewLoggingHandler(h http.Handler, l *log.Logger) *loggingHandler {
	if l == nil {
		l = log.New(os.Stdout, "HTTP ", 0)
	}

	return &loggingHandler{
		h: h,
		l: l,
	}
}

func (h *loggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.RequestURI == "" {
		// ignore "faked" requests
		h.h.ServeHTTP(w, r)
		return
	}

	start := time.Now()
	tw := &TransparentResponseWriter{w, http.StatusOK, 0}
	defer h.log(start, tw, r)

	h.h.ServeHTTP(tw, r)
}

func (h *loggingHandler) log(start time.Time, tw *TransparentResponseWriter, r *http.Request) {
	dt := time.Now().Sub(start)
	dtC := termcolor.Colorized(fmt.Sprintf("%v", dt), termcolor.CYAN)

	statusC := fmt.Sprintf("%v", tw.Status)
	if tw.Success() {
		statusC = termcolor.Colorized(statusC, termcolor.GREEN)
	} else if tw.Status >= 400 {
		statusC = termcolor.Colorized(statusC, termcolor.RED)
	}

	methodC := termcolor.Colorized(r.Method, termcolor.BLUE)
	requestURIC := termcolor.Colorized(r.URL.Path, termcolor.BOLD)
	if r.URL.RawQuery != "" {
		requestURIC += "?" + r.URL.RawQuery
	}

	sizeS := ""
	if tw.Size != 0 {
		sizeS = fmt.Sprintf(", %v bytes", tw.Size)
	}

	h.l.Printf("%v %v %v %v (%v%v)", statusC, r.RemoteAddr, methodC, requestURIC, dtC, sizeS)
}
<|endoftext|>


<commit_before>package hateoas

import (
	"net/http"
	"testing"
)

type testResourceHandler struct {
}

func (rh testResourceHandler) ResourceName() string {
	return "resources"
}
func (rh testResourceHandler) GetOne(string) (Resource, *Error) {
	return nil, nil
}
func (rh testResourceHandler) GetAll(PageOpts) ([]Resource, *Error) {
	return nil, nil
}
func (rh testResourceHandler) Create(Resource) (string, *Error) {
	return "", nil
}
func (rh testResourceHandler) Update(string, Resource) (Resource, *Error) {
	return nil, nil
}
func (rh testResourceHandler) Delete(string) *Error {
	return nil
}

func TestHandle(t *testing.T) {
	testResourceHandler := testResourceHandler{}
	Handle("/api", testResourceHandler)
	http.ListenAndServe(":8080", nil)
}
<commit_msg>Review test to add resource and functions for resourceHandler (for type assertion test)<commit_after>package hateoas

import (
	"net/http"
	"testing"
)

type testResource struct {
	Id string
}

func (r testResource) MarshalJSON() ([]byte, error) {
	return nil, nil
}
func (r testResource) UnmarshalJSON(b []byte) error {
	return nil
}

type testResourceHandler struct {
}

func (rh testResourceHandler) ResourceName() string {
	return "resources"
}
func (rh testResourceHandler) GetOne(id string) (Resource, *Error) {
	testResource := testResource{}
	return testResource, nil
}
func (rh testResourceHandler) GetAll(pageOpts PageOpts) ([]Resource, *Error) {
	testResources := []testResource{}

	resources := make([]Resource, len(testResources))
	for index, value := range testResources {
		resources[index] = value
	}

	return resources, nil
}
func (rh testResourceHandler) Create(newR Resource) (string, *Error) {
	return "", nil
}
func (rh testResourceHandler) Update(id string, updR Resource) (Resource, *Error) {
	testResource := updR.(testResource)
	return testResource, nil
}
func (rh testResourceHandler) Delete(string) *Error {
	return nil
}

func TestHandle(t *testing.T) {
	testResourceHandler := testResourceHandler{}
	Handle("/api", testResourceHandler)
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/titanous/heartbleeder/tls"
)

func main() {
	var timeout = flag.Duration("timeout", 10*time.Second, "Timeout after sending heartbeat")
	flag.Usage = func() {
		fmt.Printf("Usage: %s [options] host[:443]\n", os.Args[0])
		fmt.Println("Options:")
		flag.PrintDefaults()
	}
	flag.Parse()
	host := flag.Arg(0)
	if !strings.Contains(host, ":") {
		host += ":443"
	}
	c, err := tls.Dial("tcp", host, &tls.Config{InsecureSkipVerify: true})
	if err != nil {
		log.Printf("Error connecting to %s: %s\n", host, err)
		os.Exit(2)
	}

	err = c.WriteHeartbeat(32, nil)
	if err == tls.ErrNoHeartbeat {
		fmt.Printf("SECURE - %s does not have the heartbeat extension enabled\n", host)
		os.Exit(0)
	}
	if err != nil {
		fmt.Println("UNKNOWN - Heartbeat enabled, but there was an error writing the payload:", err)
		os.Exit(2)
	}

	readErr := make(chan error)
	go func() {
		_, _, err := c.ReadHeartbeat()
		readErr <- err
	}()

	select {
	case err := <-readErr:
		if err == nil {
			fmt.Printf("VULNERABLE - %s has the heartbeat extension enabled and is vulnerable to CVE-2014-0160\n", host)
			os.Exit(1)
		}
		fmt.Printf("SECURE - %s has heartbeat extension enabled but is not vulnerable\n", host)
		fmt.Printf("This error happened while reading the response to the malformed heartbeat (almost certainly a good thing): %q\n", err)
	case <-time.After(*timeout):
		fmt.Printf("SECURE - %s has the heartbeat extension enabled, but timed out after a malformed heartbeat (this likely means that it is not vulnerable)\n", host)
	}
}
<commit_msg>Drop default timeout to 5s<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/titanous/heartbleeder/tls"
)

func main() {
	timeout := flag.Duration("timeout", 5*time.Second, "Timeout after sending heartbeat")
	flag.Usage = func() {
		fmt.Printf("Usage: %s [options] host[:443]\n", os.Args[0])
		fmt.Println("Options:")
		flag.PrintDefaults()
	}
	flag.Parse()
	host := flag.Arg(0)
	if !strings.Contains(host, ":") {
		host += ":443"
	}
	c, err := tls.Dial("tcp", host, &tls.Config{InsecureSkipVerify: true})
	if err != nil {
		log.Printf("Error connecting to %s: %s\n", host, err)
		os.Exit(2)
	}

	err = c.WriteHeartbeat(32, nil)
	if err == tls.ErrNoHeartbeat {
		fmt.Printf("SECURE - %s does not have the heartbeat extension enabled\n", host)
		os.Exit(0)
	}
	if err != nil {
		fmt.Println("UNKNOWN - Heartbeat enabled, but there was an error writing the payload:", err)
		os.Exit(2)
	}

	readErr := make(chan error)
	go func() {
		_, _, err := c.ReadHeartbeat()
		readErr <- err
	}()

	select {
	case err := <-readErr:
		if err == nil {
			fmt.Printf("VULNERABLE - %s has the heartbeat extension enabled and is vulnerable to CVE-2014-0160\n", host)
			os.Exit(1)
		}
		fmt.Printf("SECURE - %s has heartbeat extension enabled but is not vulnerable\n", host)
		fmt.Printf("This error happened while reading the response to the malformed heartbeat (almost certainly a good thing): %q\n", err)
	case <-time.After(*timeout):
		fmt.Printf("SECURE - %s has the heartbeat extension enabled, but timed out after a malformed heartbeat (this likely means that it is not vulnerable)\n", host)
	}
}
<|endoftext|>


<commit_before>// package tokenbucket implements a simple token bucket filter.
package tokenbucket

import (
	"math/rand"
	"time"
)

type item struct {
	credit uint64
	prev   uint64
}

// Filter implements a token bucket filter.
type Filter struct {
	creditMax uint64
	touchCost uint64

	key0 uint64
	key1 uint64

	items []item
}

// New creates a new token bucket filter with num buckets, accruing tokens at rate per second. The depth specifies
// the depth of the bucket.
func New(num int, rate float64, depth uint64) *Filter {
	b := new(Filter)
	if depth <= 0 {
		panic("depth of bucket must be greater than 0")
	}
	b.touchCost = uint64((float64(1*time.Second) / rate))
	b.creditMax = depth * b.touchCost
	b.items = make([]item, num)

	// Not the full range of a uint64, but we can
	// live with 2 bits of entropy missing
	b.key0 = uint64(rand.Int63())
	b.key1 = uint64(rand.Int63())

	return b
}

func (b *Filter) touch(it *item) bool {
	now := uint64(time.Now().UnixNano())
	delta := now - it.prev
	it.credit += delta
	it.prev = now

	if it.credit > b.creditMax {
		it.credit = b.creditMax
	}

	if it.credit > b.touchCost {
		it.credit -= b.touchCost
		return true
	}
	return false
}

// Touch finds the token bucket for d, takes a token out of it and reports if
// there are still tokens left in the bucket.
func (b *Filter) Touch(d []byte) bool {
	n := len(b.items)
	h := hash(b.key0, b.key1, d)
	i := int(h) % n
	return b.touch(&b.items[i])
}
<commit_msg>Fix potential integer overflow.<commit_after>// package tokenbucket implements a simple token bucket filter.
package tokenbucket

import (
	"math/rand"
	"time"
)

type item struct {
	credit uint64
	prev   uint64
}

// Filter implements a token bucket filter.
type Filter struct {
	creditMax uint64
	touchCost uint64

	key0 uint64
	key1 uint64

	items []item
}

// New creates a new token bucket filter with num buckets, accruing tokens at rate per second. The depth specifies
// the depth of the bucket.
func New(num int, rate float64, depth uint64) *Filter {
	b := new(Filter)
	if depth <= 0 {
		panic("depth of bucket must be greater than 0")
	}
	b.touchCost = uint64((float64(1*time.Second) / rate))
	b.creditMax = depth * b.touchCost
	b.items = make([]item, num)

	// Not the full range of a uint64, but we can
	// live with 2 bits of entropy missing
	b.key0 = uint64(rand.Int63())
	b.key1 = uint64(rand.Int63())

	return b
}

func (b *Filter) touch(it *item) bool {
	now := uint64(time.Now().UnixNano())
	delta := now - it.prev
	it.credit += delta
	it.prev = now

	if it.credit > b.creditMax {
		it.credit = b.creditMax
	}

	if it.credit > b.touchCost {
		it.credit -= b.touchCost
		return true
	}
	return false
}

// Touch finds the token bucket for d, takes a token out of it and reports if
// there are still tokens left in the bucket.
func (b *Filter) Touch(d []byte) bool {
	n := len(b.items)
	h := hash(b.key0, b.key1, d)
	i := h % uint64(n)
	return b.touch(&b.items[i])
}
<|endoftext|>


<commit_before>package helper

import (
	"strconv"
	"strings"
)

const sep = ":"

// GetCacheKey get cache key for data loader
func GetCacheKey(module string, id interface{}) string {
	var str string
	switch id.(type) {
	case string:
		str = id.(string)
	case int64:
		str = strconv.FormatInt(id.(int64), 10)
	}
	return module + sep + str
}

// GetCacheID get cache id for model id
func GetCacheID(key string) (interface{}, error) {
	strs := strings.Split(key, sep)
	switch strs[0] {
	default:
		return strconv.ParseInt(strs[1], 10, 64)
	}
}
<commit_msg>fix: typeSwitchVar<commit_after>package helper

import (
	"strconv"
	"strings"
)

const sep = ":"

// GetCacheKey get cache key for data loader
func GetCacheKey(module string, id interface{}) string {
	var str string
	switch v := id.(type) {
	case int64:
		str = strconv.FormatInt(v, 10)
	case string:
		str = v
	}
	return module + sep + str
}

// GetCacheID get cache id for model id
func GetCacheID(key string) (interface{}, error) {
	strs := strings.Split(key, sep)

	return strconv.ParseInt(strs[1], 10, 64)
}
<|endoftext|>


<commit_before>package hostess

import (
	"testing"
)

const asserts = `
--- Expected ---
%s
---- Actual ----
%s`

const ipv4_pass = `
127.0.0.1
127.0.1.1
10.200.30.50
99.99.99.99
999.999.999.999
0.1.1.0
`

const ipv4_fail = `
1234.1.1.1
123.5.6
12.12
76.76.67.67.45
`

const ipv6 = ``

const domain = "localhost"
const ip = "127.0.0.1"
const enabled = true

func TestHostname(t *testing.T) {

	h := Hostname{}
	h.Domain = domain
	h.Ip = ip
	h.Enabled = enabled

	if h.Domain != domain {
		t.Errorf("Domain should be %s", domain)
	}
	if h.Ip != ip {
		t.Errorf("Domain should be %s", ip)
	}
	if h.Enabled != enabled {
		t.Errorf("Enabled should be %s", enabled)
	}
}

func TestGetHostsPath(t *testing.T) {
	path := GetHostsPath()
	const expected = "/etc/hosts"
	if path != expected {
		t.Error("Hosts path should be " + expected)
	}
}

func TestHostfile(t *testing.T) {
	hostfile := NewHostfile("./hosts")
	hostfile.Add(Hostname{domain, ip, true})
	if hostfile.Hosts[domain].Ip != ip {
		t.Errorf("Hostsfile should have %s pointing to %s", domain, ip)
	}

	hostfile.Disable(domain)
	if hostfile.Hosts[domain].Enabled != false {
		t.Errorf("%s should be disabled", domain)
	}

	hostfile.Enable(domain)
	if hostfile.Hosts[domain].Enabled != true {
		t.Errorf("%s should be enabled", domain)
	}

	hostfile.Delete(domain)
	if hostfile.Hosts[domain] != nil {
		t.Errorf("Did not expect to find %s", domain)
	}
}

func TestHostFileDuplicates(t *testing.T) {
	hostfile := NewHostfile("./hosts")

	const exp_duplicate = "Duplicate hostname entry for localhost -> 127.0.0.1"
	hostfile.Add(Hostname{domain, ip, true})
	err := hostfile.Add(Hostname{domain, ip, true})
	if err.Error() != exp_duplicate {
		t.Errorf(asserts, exp_duplicate, err)
	}

	const exp_conflict = "Conflicting hostname entries for localhost -> 127.0.1.1 and -> 127.0.0.1"
	err2 := hostfile.Add(Hostname{domain, "127.0.1.1", true})
	if err2.Error() != exp_conflict {
		t.Errorf(asserts, exp_conflict, err2)
	}

}

func TestFormatHostname(t *testing.T) {
	hostname := Hostname{domain, ip, enabled}

	const exp_enabled = "127.0.0.1 localhost"
	if hostname.Format() != exp_enabled {
		t.Errorf(asserts, hostname.Format(), exp_enabled)
	}

	hostname.Enabled = false
	const exp_disabled = "# 127.0.0.1 localhost"
	if hostname.Format() != exp_disabled {
		t.Errorf(asserts, hostname.Format(), exp_disabled)
	}
}

func TestFormatHostfile(t *testing.T) {
	// The sort order here is a bit weird.
	// 1. We want localhost entries at the top
	// 2. The rest are sorted by IP as STRINGS, not numeric values, so 10
	//    precedes 8
	const expected = `127.0.0.1 localhost devsite
127.0.1.1 ip-10-37-12-18
10.37.12.18 devsite.com m.devsite.com
# 8.8.8.8 google.com`

	hostfile := NewHostfile("./hosts")
	hostfile.Add(Hostname{"localhost", "127.0.0.1", true})
	hostfile.Add(Hostname{"ip-10-37-12-18", "127.0.1.1", true})
	hostfile.Add(Hostname{"devsite", "127.0.0.1", true})
	hostfile.Add(Hostname{"google.com", "8.8.8.8", false})
	hostfile.Add(Hostname{"devsite.com", "10.37.12.18", true})
	hostfile.Add(Hostname{"m.devsite.com", "10.37.12.18", true})
	f := hostfile.Format()
	if f != expected {
		t.Errorf(asserts, expected, f)
	}
}

func TestTrimWS(t *testing.T) {
	const expected = `  candy

	`
	got := TrimWS(expected)
	if got != "candy" {
		t.Errorf(asserts, expected, got)
	}
}

func TestListDomainsByIp(t *testing.T) {
	hostfile := NewHostfile("./hosts")
	hostfile.Add(Hostname{"devsite.com", "10.37.12.18", true})
	hostfile.Add(Hostname{"m.devsite.com", "10.37.12.18", true})
	hostfile.Add(Hostname{"google.com", "8.8.8.8", false})

	names := hostfile.ListDomainsByIp("10.37.12.18")
	if !(names[0] == "devsite.com" && names[1] == "m.devsite.com") {
		t.Errorf("Expected devsite.com and m.devsite.com. Got %s", names)
	}

	hostfile2 := NewHostfile("./hosts")
	hostfile2.Add(Hostname{"localhost", "127.0.0.1", true})
	hostfile2.Add(Hostname{"ip-10-37-12-18", "127.0.1.1", true})
	hostfile2.Add(Hostname{"devsite", "127.0.0.1", true})

	names2 := hostfile2.ListDomainsByIp("127.0.0.1")
	if !(names2[0] == "localhost" && names2[1] == "devsite") {
		t.Errorf("Expected localhost and devsite. Got %s", names2)
	}
}

func TestParseLine(t *testing.T) {
	var hosts = []Hostname{}

	// Commented stuff
	hosts = parseLine("# The following lines are desirable for IPv6 capable hosts")
	if len(hosts) > 0 {
		t.Error("Expected to find zero hostnames")
	}

	hosts = parseLine("#66.33.99.11              test.domain.com")
	if !ContainsHostname(hosts, Hostname{"test.domain.com", "66.33.99.11", false}) {
		t.Error("Expected to find test.domain.com (disabled)")
	}

	hosts = parseLine("#  66.33.99.11	test.domain.com	domain.com")
	if !ContainsHostname(hosts, Hostname{"test.domain.com", "66.33.99.11", false}) ||
		!ContainsHostname(hosts, Hostname{"domain.com", "66.33.99.11", false}) {
		t.Error("Expected to find domain.com and test.domain.com (disabled)")
	}

	// Not Commented stuff
	hosts = parseLine("255.255.255.255 broadcasthost test.domain.com	domain.com")
	if !ContainsHostname(hosts, Hostname{"broadcasthost", "255.255.255.255", true}) ||
		!ContainsHostname(hosts, Hostname{"test.domain.com", "255.255.255.255", true}) ||
		!ContainsHostname(hosts, Hostname{"domain.com", "255.255.255.255", true}) {
		t.Error("Expected to find broadcasthost, domain.com, and test.domain.com (enabled)")
	}

	// Ipv6 stuff
	hosts = parseLine("::1             localhost")
	if !ContainsHostname(hosts, Hostname{"localhost", "::1", true}) {
		t.Error("Expected to find localhost ipv6 (enabled)")
	}

	hosts = parseLine("ff02::1 ip6-allnodes")
	if !ContainsHostname(hosts, Hostname{"ip6-allnodes", "ff02::1", true}) {
		t.Error("Expected to find ip6-allnodes ipv6 (enabled)")
	}
}
<commit_msg>Added a todo for a test case<commit_after>package hostess

import (
	"testing"
)

const asserts = `
--- Expected ---
%s
---- Actual ----
%s`

const ipv4_pass = `
127.0.0.1
127.0.1.1
10.200.30.50
99.99.99.99
999.999.999.999
0.1.1.0
`

const ipv4_fail = `
1234.1.1.1
123.5.6
12.12
76.76.67.67.45
`

const ipv6 = ``

const domain = "localhost"
const ip = "127.0.0.1"
const enabled = true

func TestHostname(t *testing.T) {

	h := Hostname{}
	h.Domain = domain
	h.Ip = ip
	h.Enabled = enabled

	if h.Domain != domain {
		t.Errorf("Domain should be %s", domain)
	}
	if h.Ip != ip {
		t.Errorf("Domain should be %s", ip)
	}
	if h.Enabled != enabled {
		t.Errorf("Enabled should be %s", enabled)
	}
}

func TestGetHostsPath(t *testing.T) {
	path := GetHostsPath()
	const expected = "/etc/hosts"
	if path != expected {
		t.Error("Hosts path should be " + expected)
	}
}

func TestHostfile(t *testing.T) {
	hostfile := NewHostfile("./hosts")
	hostfile.Add(Hostname{domain, ip, true})
	if hostfile.Hosts[domain].Ip != ip {
		t.Errorf("Hostsfile should have %s pointing to %s", domain, ip)
	}

	hostfile.Disable(domain)
	if hostfile.Hosts[domain].Enabled != false {
		t.Errorf("%s should be disabled", domain)
	}

	hostfile.Enable(domain)
	if hostfile.Hosts[domain].Enabled != true {
		t.Errorf("%s should be enabled", domain)
	}

	hostfile.Delete(domain)
	if hostfile.Hosts[domain] != nil {
		t.Errorf("Did not expect to find %s", domain)
	}
}

func TestHostFileDuplicates(t *testing.T) {
	hostfile := NewHostfile("./hosts")

	const exp_duplicate = "Duplicate hostname entry for localhost -> 127.0.0.1"
	hostfile.Add(Hostname{domain, ip, true})
	err := hostfile.Add(Hostname{domain, ip, true})
	if err.Error() != exp_duplicate {
		t.Errorf(asserts, exp_duplicate, err)
	}

	const exp_conflict = "Conflicting hostname entries for localhost -> 127.0.1.1 and -> 127.0.0.1"
	err2 := hostfile.Add(Hostname{domain, "127.0.1.1", true})
	if err2.Error() != exp_conflict {
		t.Errorf(asserts, exp_conflict, err2)
	}

	// @TODO Add an additional test case here: Adding a domain twice with one
	// enabled and one disabled should just add the domain once enabled.
}

func TestFormatHostname(t *testing.T) {
	hostname := Hostname{domain, ip, enabled}

	const exp_enabled = "127.0.0.1 localhost"
	if hostname.Format() != exp_enabled {
		t.Errorf(asserts, hostname.Format(), exp_enabled)
	}

	hostname.Enabled = false
	const exp_disabled = "# 127.0.0.1 localhost"
	if hostname.Format() != exp_disabled {
		t.Errorf(asserts, hostname.Format(), exp_disabled)
	}
}

func TestFormatHostfile(t *testing.T) {
	// The sort order here is a bit weird.
	// 1. We want localhost entries at the top
	// 2. The rest are sorted by IP as STRINGS, not numeric values, so 10
	//    precedes 8
	const expected = `127.0.0.1 localhost devsite
127.0.1.1 ip-10-37-12-18
10.37.12.18 devsite.com m.devsite.com
# 8.8.8.8 google.com`

	hostfile := NewHostfile("./hosts")
	hostfile.Add(Hostname{"localhost", "127.0.0.1", true})
	hostfile.Add(Hostname{"ip-10-37-12-18", "127.0.1.1", true})
	hostfile.Add(Hostname{"devsite", "127.0.0.1", true})
	hostfile.Add(Hostname{"google.com", "8.8.8.8", false})
	hostfile.Add(Hostname{"devsite.com", "10.37.12.18", true})
	hostfile.Add(Hostname{"m.devsite.com", "10.37.12.18", true})
	f := hostfile.Format()
	if f != expected {
		t.Errorf(asserts, expected, f)
	}
}

func TestTrimWS(t *testing.T) {
	const expected = `  candy

	`
	got := TrimWS(expected)
	if got != "candy" {
		t.Errorf(asserts, expected, got)
	}
}

func TestListDomainsByIp(t *testing.T) {
	hostfile := NewHostfile("./hosts")
	hostfile.Add(Hostname{"devsite.com", "10.37.12.18", true})
	hostfile.Add(Hostname{"m.devsite.com", "10.37.12.18", true})
	hostfile.Add(Hostname{"google.com", "8.8.8.8", false})

	names := hostfile.ListDomainsByIp("10.37.12.18")
	if !(names[0] == "devsite.com" && names[1] == "m.devsite.com") {
		t.Errorf("Expected devsite.com and m.devsite.com. Got %s", names)
	}

	hostfile2 := NewHostfile("./hosts")
	hostfile2.Add(Hostname{"localhost", "127.0.0.1", true})
	hostfile2.Add(Hostname{"ip-10-37-12-18", "127.0.1.1", true})
	hostfile2.Add(Hostname{"devsite", "127.0.0.1", true})

	names2 := hostfile2.ListDomainsByIp("127.0.0.1")
	if !(names2[0] == "localhost" && names2[1] == "devsite") {
		t.Errorf("Expected localhost and devsite. Got %s", names2)
	}
}

func TestParseLine(t *testing.T) {
	var hosts = []Hostname{}

	// Commented stuff
	hosts = parseLine("# The following lines are desirable for IPv6 capable hosts")
	if len(hosts) > 0 {
		t.Error("Expected to find zero hostnames")
	}

	hosts = parseLine("#66.33.99.11              test.domain.com")
	if !ContainsHostname(hosts, Hostname{"test.domain.com", "66.33.99.11", false}) {
		t.Error("Expected to find test.domain.com (disabled)")
	}

	hosts = parseLine("#  66.33.99.11	test.domain.com	domain.com")
	if !ContainsHostname(hosts, Hostname{"test.domain.com", "66.33.99.11", false}) ||
		!ContainsHostname(hosts, Hostname{"domain.com", "66.33.99.11", false}) {
		t.Error("Expected to find domain.com and test.domain.com (disabled)")
	}

	// Not Commented stuff
	hosts = parseLine("255.255.255.255 broadcasthost test.domain.com	domain.com")
	if !ContainsHostname(hosts, Hostname{"broadcasthost", "255.255.255.255", true}) ||
		!ContainsHostname(hosts, Hostname{"test.domain.com", "255.255.255.255", true}) ||
		!ContainsHostname(hosts, Hostname{"domain.com", "255.255.255.255", true}) {
		t.Error("Expected to find broadcasthost, domain.com, and test.domain.com (enabled)")
	}

	// Ipv6 stuff
	hosts = parseLine("::1             localhost")
	if !ContainsHostname(hosts, Hostname{"localhost", "::1", true}) {
		t.Error("Expected to find localhost ipv6 (enabled)")
	}

	hosts = parseLine("ff02::1 ip6-allnodes")
	if !ContainsHostname(hosts, Hostname{"ip6-allnodes", "ff02::1", true}) {
		t.Error("Expected to find ip6-allnodes ipv6 (enabled)")
	}
}
<|endoftext|>


<commit_before>package hyperion

import (
	`encoding/json`
	`io`
	`net`
	`reflect`
	`strconv`
	`time`
	`github.com/pdf/xbmc-callback-daemon/logger`
)

var (
	connAddress string
	conn        net.Conn
	encoder     *json.Encoder
	decoder     *json.Decoder
)

// Response stores Hyperion results for RPC calls.
type Response struct {
	Success bool    `json:"success"`
	Error   *string `json:"error"`
}

// qtfloat64 is a float64 class to ensure marshalling as floats as QT expects
// them.
// Seriously, fuck you QT.
type qtfloat64 float64

// Custom marshaller for correct float output
func (f qtfloat64) MarshalText() ([]byte, error) {
	v := reflect.ValueOf(f)
	return []byte(strconv.FormatFloat(v.Float(), 'f', 6, 64)), nil
}

// Custom marshaller for correct float output
func (f qtfloat64) MarshalJSON() ([]byte, error) {
	v := reflect.ValueOf(f)
	return []byte(strconv.FormatFloat(v.Float(), 'f', 6, 64)), nil
}

// Connect establishes a TCP connection to the specified address and attaches
// JSON encoders/decoders.
func Connect(address string) {
	if connAddress == `` {
		connAddress = address
	}
	conn, err := net.Dial(`tcp`, address)
	for err != nil {
		logger.Error(`Connecting to Hyperion: `, err)
		logger.Info(`Attempting reconnect...`)
		time.Sleep(time.Second)
		conn, err = net.Dial(`tcp`, address)
	}
	logger.Info(`Connected to Hyperion`)
	encoder = json.NewEncoder(conn)
	decoder = json.NewDecoder(conn)
}

// Close Hyperion connection
func Close() {
	conn.Close()
}

// coerce takes a key/value pair and recurses down the value, replacing any
// float64 values with qtfloat64 conversions and returns the result.
// Some known non-float values are instead converted to integers.
// Seriously, fuck you QT.
func coerce(key string, value interface{}) interface{} {
	switch value.(type) {
	case float64:
		switch key {
		case `priority`, `color`:
			return uint8(value.(float64))
		default:
			return qtfloat64(value.(float64))
		}
	case []interface{}:
		result, ok := value.([]interface{})
		if ok == false {
			logger.Panic(`Could not parse array, check configuration near `, value)
		}
		for i := range result {
			result[i] = coerce(key, result[i])
		}
		return result
	case map[string]interface{}:
		result, ok := value.(map[string]interface{})
		if ok == false {
			logger.Panic(`Could not parse object, check configuration near `, value)
		}
		for k, v := range result {
			result[k] = coerce(k, v)
		}
		return result
	default:
		return value
	}
}

// Read and decode JSON from the XBMC connection into the notification pointer.
func Read(response *Response) {
	err := decoder.Decode(&response)
	// Kick off the connection again on EOF, eat any decoding errors otherwise.
	// TODO: This probably needs to be more robust.
	if err == io.EOF {
		logger.Error(`Reading from Hyperion: `, err)
		Connect(connAddress)
	} else if err != nil {
		logger.Error(`Decoding response from Hyperion: `, err)
		return
	}
}

// Execute takes the callback and performs a JSON-RPC request over the
// established Hyperion connection.
func Execute(callback map[string]interface{}) {
	response := &Response{}

	// Drop properties that the backend doesn't understand, and coerce float64/int
	cb := make(map[string]interface{}, 0)
	for k, v := range callback {
		switch k {
		case `backend`, `types`:
			continue
		default:
			cb[k] = coerce(k, v)
		}
	}

	logger.Debug(`Sending request to Hyperion: `, cb)
	// Encode and send request
	if err := encoder.Encode(&cb); err != nil {
		logger.Error(`Sending to Hyperion: `, err)
	}
	// Check response and log any failure responses from Hyperion
	Read(response)
	logger.Debug(`Received response from Hyperion: `, response)
	if response.Success == false && response.Error != nil {
		logger.Warn(`Error received from Hyperion: `, response.Error)
	}
}
<commit_msg>Hyperion: `priority` parameter can be larger than `unit8`<commit_after>package hyperion

import (
	`encoding/json`
	`io`
	`net`
	`reflect`
	`strconv`
	`time`
	`github.com/pdf/xbmc-callback-daemon/logger`
)

var (
	connAddress string
	conn        net.Conn
	encoder     *json.Encoder
	decoder     *json.Decoder
)

// Response stores Hyperion results for RPC calls.
type Response struct {
	Success bool    `json:"success"`
	Error   *string `json:"error"`
}

// qtfloat64 is a float64 class to ensure marshalling as floats as QT expects
// them.
// Seriously, fuck you QT.
type qtfloat64 float64

// Custom marshaller for correct float output
func (f qtfloat64) MarshalText() ([]byte, error) {
	v := reflect.ValueOf(f)
	return []byte(strconv.FormatFloat(v.Float(), 'f', 6, 64)), nil
}

// Custom marshaller for correct float output
func (f qtfloat64) MarshalJSON() ([]byte, error) {
	v := reflect.ValueOf(f)
	return []byte(strconv.FormatFloat(v.Float(), 'f', 6, 64)), nil
}

// Connect establishes a TCP connection to the specified address and attaches
// JSON encoders/decoders.
func Connect(address string) {
	if connAddress == `` {
		connAddress = address
	}
	conn, err := net.Dial(`tcp`, address)
	for err != nil {
		logger.Error(`Connecting to Hyperion: `, err)
		logger.Info(`Attempting reconnect...`)
		time.Sleep(time.Second)
		conn, err = net.Dial(`tcp`, address)
	}
	logger.Info(`Connected to Hyperion`)
	encoder = json.NewEncoder(conn)
	decoder = json.NewDecoder(conn)
}

// Close Hyperion connection
func Close() {
	conn.Close()
}

// coerce takes a key/value pair and recurses down the value, replacing any
// float64 values with qtfloat64 conversions and returns the result.
// Some known non-float values are instead converted to integers.
// Seriously, fuck you QT.
func coerce(key string, value interface{}) interface{} {
	switch value.(type) {
	case float64:
		switch key {
		case `priority`:
			return uint16(value.(float64))
		case `color`:
			return uint8(value.(float64))
		default:
			return qtfloat64(value.(float64))
		}
	case []interface{}:
		result, ok := value.([]interface{})
		if ok == false {
			logger.Panic(`Could not parse array, check configuration near `, value)
		}
		for i := range result {
			result[i] = coerce(key, result[i])
		}
		return result
	case map[string]interface{}:
		result, ok := value.(map[string]interface{})
		if ok == false {
			logger.Panic(`Could not parse object, check configuration near `, value)
		}
		for k, v := range result {
			result[k] = coerce(k, v)
		}
		return result
	default:
		return value
	}
}

// Read and decode JSON from the XBMC connection into the notification pointer.
func Read(response *Response) {
	err := decoder.Decode(&response)
	// Kick off the connection again on EOF, eat any decoding errors otherwise.
	// TODO: This probably needs to be more robust.
	if err == io.EOF {
		logger.Error(`Reading from Hyperion: `, err)
		Connect(connAddress)
	} else if err != nil {
		logger.Error(`Decoding response from Hyperion: `, err)
		return
	}
}

// Execute takes the callback and performs a JSON-RPC request over the
// established Hyperion connection.
func Execute(callback map[string]interface{}) {
	response := &Response{}

	// Drop properties that the backend doesn't understand, and coerce float64/int
	cb := make(map[string]interface{}, 0)
	for k, v := range callback {
		switch k {
		case `backend`, `types`:
			continue
		default:
			cb[k] = coerce(k, v)
		}
	}

	logger.Debug(`Sending request to Hyperion: `, cb)
	// Encode and send request
	if err := encoder.Encode(&cb); err != nil {
		logger.Error(`Sending to Hyperion: `, err)
	}
	// Check response and log any failure responses from Hyperion
	Read(response)
	logger.Debug(`Received response from Hyperion: `, response)
	if response.Success == false && response.Error != nil {
		logger.Warn(`Error received from Hyperion: `, response.Error)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package record

import (
	"fmt"
	"math/rand"
	"time"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/clock"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/watch"
	restclient "k8s.io/client-go/rest"
	ref "k8s.io/client-go/tools/reference"

	"net/http"

	"github.com/golang/glog"
)

const maxTriesPerEvent = 12

var defaultSleepDuration = 10 * time.Second

const maxQueuedEvents = 1000

// EventSink knows how to store events (client.Client implements it.)
// EventSink must respect the namespace that will be embedded in 'event'.
// It is assumed that EventSink will return the same sorts of errors as
// pkg/client's REST client.
type EventSink interface {
	Create(event *v1.Event) (*v1.Event, error)
	Update(event *v1.Event) (*v1.Event, error)
	Patch(oldEvent *v1.Event, data []byte) (*v1.Event, error)
}

// EventRecorder knows how to record events on behalf of an EventSource.
type EventRecorder interface {
	// Event constructs an event from the given information and puts it in the queue for sending.
	// 'object' is the object this event is about. Event will make a reference-- or you may also
	// pass a reference to the object directly.
	// 'type' of this event, and can be one of Normal, Warning. New types could be added in future
	// 'reason' is the reason this event is generated. 'reason' should be short and unique; it
	// should be in UpperCamelCase format (starting with a capital letter). "reason" will be used
	// to automate handling of events, so imagine people writing switch statements to handle them.
	// You want to make that easy.
	// 'message' is intended to be human readable.
	//
	// The resulting event will be created in the same namespace as the reference object.
	Event(object runtime.Object, eventtype, reason, message string)

	// Eventf is just like Event, but with Sprintf for the message field.
	Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{})

	// PastEventf is just like Eventf, but with an option to specify the event's 'timestamp' field.
	PastEventf(object runtime.Object, timestamp metav1.Time, eventtype, reason, messageFmt string, args ...interface{})
}

// EventBroadcaster knows how to receive events and send them to any EventSink, watcher, or log.
type EventBroadcaster interface {
	// StartEventWatcher starts sending events received from this EventBroadcaster to the given
	// event handler function. The return value can be ignored or used to stop recording, if
	// desired.
	StartEventWatcher(eventHandler func(*v1.Event)) watch.Interface

	// StartRecordingToSink starts sending events received from this EventBroadcaster to the given
	// sink. The return value can be ignored or used to stop recording, if desired.
	StartRecordingToSink(sink EventSink) watch.Interface

	// StartLogging starts sending events received from this EventBroadcaster to the given logging
	// function. The return value can be ignored or used to stop recording, if desired.
	StartLogging(logf func(format string, args ...interface{})) watch.Interface

	// NewRecorder returns an EventRecorder that can be used to send events to this EventBroadcaster
	// with the event source set to the given event source.
	NewRecorder(scheme *runtime.Scheme, source v1.EventSource) EventRecorder
}

// Creates a new event broadcaster.
func NewBroadcaster() EventBroadcaster {
	return &eventBroadcasterImpl{watch.NewBroadcaster(maxQueuedEvents, watch.DropIfChannelFull), defaultSleepDuration}
}

func NewBroadcasterForTests(sleepDuration time.Duration) EventBroadcaster {
	return &eventBroadcasterImpl{watch.NewBroadcaster(maxQueuedEvents, watch.DropIfChannelFull), sleepDuration}
}

type eventBroadcasterImpl struct {
	*watch.Broadcaster
	sleepDuration time.Duration
}

// StartRecordingToSink starts sending events received from the specified eventBroadcaster to the given sink.
// The return value can be ignored or used to stop recording, if desired.
// TODO: make me an object with parameterizable queue length and retry interval
func (eventBroadcaster *eventBroadcasterImpl) StartRecordingToSink(sink EventSink) watch.Interface {
	// The default math/rand package functions aren't thread safe, so create a
	// new Rand object for each StartRecording call.
	randGen := rand.New(rand.NewSource(time.Now().UnixNano()))
	eventCorrelator := NewEventCorrelator(clock.RealClock{})
	return eventBroadcaster.StartEventWatcher(
		func(event *v1.Event) {
			recordToSink(sink, event, eventCorrelator, randGen, eventBroadcaster.sleepDuration)
		})
}

func recordToSink(sink EventSink, event *v1.Event, eventCorrelator *EventCorrelator, randGen *rand.Rand, sleepDuration time.Duration) {
	// Make a copy before modification, because there could be multiple listeners.
	// Events are safe to copy like this.
	eventCopy := *event
	event = &eventCopy
	result, err := eventCorrelator.EventCorrelate(event)
	if err != nil {
		utilruntime.HandleError(err)
	}
	if result.Skip {
		return
	}
	tries := 0
	for {
		if recordEvent(sink, result.Event, result.Patch, result.Event.Count > 1, eventCorrelator) {
			break
		}
		tries++
		if tries >= maxTriesPerEvent {
			glog.Errorf("Unable to write event '%#v' (retry limit exceeded!)", event)
			break
		}
		// Randomize the first sleep so that various clients won't all be
		// synced up if the master goes down.
		if tries == 1 {
			time.Sleep(time.Duration(float64(sleepDuration) * randGen.Float64()))
		} else {
			time.Sleep(sleepDuration)
		}
	}
}

func isKeyNotFoundError(err error) bool {
	statusErr, _ := err.(*errors.StatusError)

	if statusErr != nil && statusErr.Status().Code == http.StatusNotFound {
		return true
	}

	return false
}

// recordEvent attempts to write event to a sink. It returns true if the event
// was successfully recorded or discarded, false if it should be retried.
// If updateExistingEvent is false, it creates a new event, otherwise it updates
// existing event.
func recordEvent(sink EventSink, event *v1.Event, patch []byte, updateExistingEvent bool, eventCorrelator *EventCorrelator) bool {
	var newEvent *v1.Event
	var err error
	if updateExistingEvent {
		newEvent, err = sink.Patch(event, patch)
	}
	// Update can fail because the event may have been removed and it no longer exists.
	if !updateExistingEvent || (updateExistingEvent && isKeyNotFoundError(err)) {
		// Making sure that ResourceVersion is empty on creation
		event.ResourceVersion = ""
		newEvent, err = sink.Create(event)
	}
	if err == nil {
		// we need to update our event correlator with the server returned state to handle name/resourceversion
		eventCorrelator.UpdateState(newEvent)
		return true
	}

	// If we can't contact the server, then hold everything while we keep trying.
	// Otherwise, something about the event is malformed and we should abandon it.
	switch err.(type) {
	case *restclient.RequestConstructionError:
		// We will construct the request the same next time, so don't keep trying.
		glog.Errorf("Unable to construct event '%#v': '%v' (will not retry!)", event, err)
		return true
	case *errors.StatusError:
		if errors.IsAlreadyExists(err) {
			glog.V(5).Infof("Server rejected event '%#v': '%v' (will not retry!)", event, err)
		} else {
			glog.Errorf("Server rejected event '%#v': '%v' (will not retry!)", event, err)
		}
		return true
	case *errors.UnexpectedObjectError:
		// We don't expect this; it implies the server's response didn't match a
		// known pattern. Go ahead and retry.
	default:
		// This case includes actual http transport errors. Go ahead and retry.
	}
	glog.Errorf("Unable to write event: '%v' (may retry after sleeping)", err)
	return false
}

// StartLogging starts sending events received from this EventBroadcaster to the given logging function.
// The return value can be ignored or used to stop recording, if desired.
func (eventBroadcaster *eventBroadcasterImpl) StartLogging(logf func(format string, args ...interface{})) watch.Interface {
	return eventBroadcaster.StartEventWatcher(
		func(e *v1.Event) {
			logf("Event(%#v): type: '%v' reason: '%v' %v", e.InvolvedObject, e.Type, e.Reason, e.Message)
		})
}

// StartEventWatcher starts sending events received from this EventBroadcaster to the given event handler function.
// The return value can be ignored or used to stop recording, if desired.
func (eventBroadcaster *eventBroadcasterImpl) StartEventWatcher(eventHandler func(*v1.Event)) watch.Interface {
	watcher := eventBroadcaster.Watch()
	go func() {
		defer utilruntime.HandleCrash()
		for {
			watchEvent, open := <-watcher.ResultChan()
			if !open {
				return
			}
			event, ok := watchEvent.Object.(*v1.Event)
			if !ok {
				// This is all local, so there's no reason this should
				// ever happen.
				continue
			}
			eventHandler(event)
		}
	}()
	return watcher
}

// NewRecorder returns an EventRecorder that records events with the given event source.
func (eventBroadcaster *eventBroadcasterImpl) NewRecorder(scheme *runtime.Scheme, source v1.EventSource) EventRecorder {
	return &recorderImpl{scheme, source, eventBroadcaster.Broadcaster, clock.RealClock{}}
}

type recorderImpl struct {
	scheme *runtime.Scheme
	source v1.EventSource
	*watch.Broadcaster
	clock clock.Clock
}

func (recorder *recorderImpl) generateEvent(object runtime.Object, timestamp metav1.Time, eventtype, reason, message string) {
	ref, err := ref.GetReference(recorder.scheme, object)
	if err != nil {
		glog.Errorf("Could not construct reference to: '%#v' due to: '%v'. Will not report event: '%v' '%v' '%v'", object, err, eventtype, reason, message)
		return
	}

	if !validateEventType(eventtype) {
		glog.Errorf("Unsupported event type: '%v'", eventtype)
		return
	}

	event := recorder.makeEvent(ref, eventtype, reason, message)
	event.Source = recorder.source

	go func() {
		// NOTE: events should be a non-blocking operation
		defer utilruntime.HandleCrash()
		recorder.Action(watch.Added, event)
	}()
}

func validateEventType(eventtype string) bool {
	switch eventtype {
	case v1.EventTypeNormal, v1.EventTypeWarning:
		return true
	}
	return false
}

func (recorder *recorderImpl) Event(object runtime.Object, eventtype, reason, message string) {
	recorder.generateEvent(object, metav1.Now(), eventtype, reason, message)
}

func (recorder *recorderImpl) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) {
	recorder.Event(object, eventtype, reason, fmt.Sprintf(messageFmt, args...))
}

func (recorder *recorderImpl) PastEventf(object runtime.Object, timestamp metav1.Time, eventtype, reason, messageFmt string, args ...interface{}) {
	recorder.generateEvent(object, timestamp, eventtype, reason, fmt.Sprintf(messageFmt, args...))
}

func (recorder *recorderImpl) makeEvent(ref *v1.ObjectReference, eventtype, reason, message string) *v1.Event {
	t := metav1.Time{Time: recorder.clock.Now()}
	namespace := ref.Namespace
	if namespace == "" {
		namespace = metav1.NamespaceDefault
	}
	return &v1.Event{
		ObjectMeta: metav1.ObjectMeta{
			Name:      fmt.Sprintf("%v.%x", ref.Name, t.UnixNano()),
			Namespace: namespace,
		},
		InvolvedObject: *ref,
		Reason:         reason,
		Message:        message,
		FirstTimestamp: t,
		LastTimestamp:  t,
		Count:          1,
		Type:           eventtype,
	}
}
<commit_msg>Use range in loops; misc fixes<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package record

import (
	"fmt"
	"math/rand"
	"time"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/clock"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/watch"
	restclient "k8s.io/client-go/rest"
	ref "k8s.io/client-go/tools/reference"

	"net/http"

	"github.com/golang/glog"
)

const maxTriesPerEvent = 12

var defaultSleepDuration = 10 * time.Second

const maxQueuedEvents = 1000

// EventSink knows how to store events (client.Client implements it.)
// EventSink must respect the namespace that will be embedded in 'event'.
// It is assumed that EventSink will return the same sorts of errors as
// pkg/client's REST client.
type EventSink interface {
	Create(event *v1.Event) (*v1.Event, error)
	Update(event *v1.Event) (*v1.Event, error)
	Patch(oldEvent *v1.Event, data []byte) (*v1.Event, error)
}

// EventRecorder knows how to record events on behalf of an EventSource.
type EventRecorder interface {
	// Event constructs an event from the given information and puts it in the queue for sending.
	// 'object' is the object this event is about. Event will make a reference-- or you may also
	// pass a reference to the object directly.
	// 'type' of this event, and can be one of Normal, Warning. New types could be added in future
	// 'reason' is the reason this event is generated. 'reason' should be short and unique; it
	// should be in UpperCamelCase format (starting with a capital letter). "reason" will be used
	// to automate handling of events, so imagine people writing switch statements to handle them.
	// You want to make that easy.
	// 'message' is intended to be human readable.
	//
	// The resulting event will be created in the same namespace as the reference object.
	Event(object runtime.Object, eventtype, reason, message string)

	// Eventf is just like Event, but with Sprintf for the message field.
	Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{})

	// PastEventf is just like Eventf, but with an option to specify the event's 'timestamp' field.
	PastEventf(object runtime.Object, timestamp metav1.Time, eventtype, reason, messageFmt string, args ...interface{})
}

// EventBroadcaster knows how to receive events and send them to any EventSink, watcher, or log.
type EventBroadcaster interface {
	// StartEventWatcher starts sending events received from this EventBroadcaster to the given
	// event handler function. The return value can be ignored or used to stop recording, if
	// desired.
	StartEventWatcher(eventHandler func(*v1.Event)) watch.Interface

	// StartRecordingToSink starts sending events received from this EventBroadcaster to the given
	// sink. The return value can be ignored or used to stop recording, if desired.
	StartRecordingToSink(sink EventSink) watch.Interface

	// StartLogging starts sending events received from this EventBroadcaster to the given logging
	// function. The return value can be ignored or used to stop recording, if desired.
	StartLogging(logf func(format string, args ...interface{})) watch.Interface

	// NewRecorder returns an EventRecorder that can be used to send events to this EventBroadcaster
	// with the event source set to the given event source.
	NewRecorder(scheme *runtime.Scheme, source v1.EventSource) EventRecorder
}

// Creates a new event broadcaster.
func NewBroadcaster() EventBroadcaster {
	return &eventBroadcasterImpl{watch.NewBroadcaster(maxQueuedEvents, watch.DropIfChannelFull), defaultSleepDuration}
}

func NewBroadcasterForTests(sleepDuration time.Duration) EventBroadcaster {
	return &eventBroadcasterImpl{watch.NewBroadcaster(maxQueuedEvents, watch.DropIfChannelFull), sleepDuration}
}

type eventBroadcasterImpl struct {
	*watch.Broadcaster
	sleepDuration time.Duration
}

// StartRecordingToSink starts sending events received from the specified eventBroadcaster to the given sink.
// The return value can be ignored or used to stop recording, if desired.
// TODO: make me an object with parameterizable queue length and retry interval
func (eventBroadcaster *eventBroadcasterImpl) StartRecordingToSink(sink EventSink) watch.Interface {
	// The default math/rand package functions aren't thread safe, so create a
	// new Rand object for each StartRecording call.
	randGen := rand.New(rand.NewSource(time.Now().UnixNano()))
	eventCorrelator := NewEventCorrelator(clock.RealClock{})
	return eventBroadcaster.StartEventWatcher(
		func(event *v1.Event) {
			recordToSink(sink, event, eventCorrelator, randGen, eventBroadcaster.sleepDuration)
		})
}

func recordToSink(sink EventSink, event *v1.Event, eventCorrelator *EventCorrelator, randGen *rand.Rand, sleepDuration time.Duration) {
	// Make a copy before modification, because there could be multiple listeners.
	// Events are safe to copy like this.
	eventCopy := *event
	event = &eventCopy
	result, err := eventCorrelator.EventCorrelate(event)
	if err != nil {
		utilruntime.HandleError(err)
	}
	if result.Skip {
		return
	}
	tries := 0
	for {
		if recordEvent(sink, result.Event, result.Patch, result.Event.Count > 1, eventCorrelator) {
			break
		}
		tries++
		if tries >= maxTriesPerEvent {
			glog.Errorf("Unable to write event '%#v' (retry limit exceeded!)", event)
			break
		}
		// Randomize the first sleep so that various clients won't all be
		// synced up if the master goes down.
		if tries == 1 {
			time.Sleep(time.Duration(float64(sleepDuration) * randGen.Float64()))
		} else {
			time.Sleep(sleepDuration)
		}
	}
}

func isKeyNotFoundError(err error) bool {
	statusErr, _ := err.(*errors.StatusError)

	if statusErr != nil && statusErr.Status().Code == http.StatusNotFound {
		return true
	}

	return false
}

// recordEvent attempts to write event to a sink. It returns true if the event
// was successfully recorded or discarded, false if it should be retried.
// If updateExistingEvent is false, it creates a new event, otherwise it updates
// existing event.
func recordEvent(sink EventSink, event *v1.Event, patch []byte, updateExistingEvent bool, eventCorrelator *EventCorrelator) bool {
	var newEvent *v1.Event
	var err error
	if updateExistingEvent {
		newEvent, err = sink.Patch(event, patch)
	}
	// Update can fail because the event may have been removed and it no longer exists.
	if !updateExistingEvent || (updateExistingEvent && isKeyNotFoundError(err)) {
		// Making sure that ResourceVersion is empty on creation
		event.ResourceVersion = ""
		newEvent, err = sink.Create(event)
	}
	if err == nil {
		// we need to update our event correlator with the server returned state to handle name/resourceversion
		eventCorrelator.UpdateState(newEvent)
		return true
	}

	// If we can't contact the server, then hold everything while we keep trying.
	// Otherwise, something about the event is malformed and we should abandon it.
	switch err.(type) {
	case *restclient.RequestConstructionError:
		// We will construct the request the same next time, so don't keep trying.
		glog.Errorf("Unable to construct event '%#v': '%v' (will not retry!)", event, err)
		return true
	case *errors.StatusError:
		if errors.IsAlreadyExists(err) {
			glog.V(5).Infof("Server rejected event '%#v': '%v' (will not retry!)", event, err)
		} else {
			glog.Errorf("Server rejected event '%#v': '%v' (will not retry!)", event, err)
		}
		return true
	case *errors.UnexpectedObjectError:
		// We don't expect this; it implies the server's response didn't match a
		// known pattern. Go ahead and retry.
	default:
		// This case includes actual http transport errors. Go ahead and retry.
	}
	glog.Errorf("Unable to write event: '%v' (may retry after sleeping)", err)
	return false
}

// StartLogging starts sending events received from this EventBroadcaster to the given logging function.
// The return value can be ignored or used to stop recording, if desired.
func (eventBroadcaster *eventBroadcasterImpl) StartLogging(logf func(format string, args ...interface{})) watch.Interface {
	return eventBroadcaster.StartEventWatcher(
		func(e *v1.Event) {
			logf("Event(%#v): type: '%v' reason: '%v' %v", e.InvolvedObject, e.Type, e.Reason, e.Message)
		})
}

// StartEventWatcher starts sending events received from this EventBroadcaster to the given event handler function.
// The return value can be ignored or used to stop recording, if desired.
func (eventBroadcaster *eventBroadcasterImpl) StartEventWatcher(eventHandler func(*v1.Event)) watch.Interface {
	watcher := eventBroadcaster.Watch()
	go func() {
		defer utilruntime.HandleCrash()
		for watchEvent := range watcher.ResultChan() {
			event, ok := watchEvent.Object.(*v1.Event)
			if !ok {
				// This is all local, so there's no reason this should
				// ever happen.
				continue
			}
			eventHandler(event)
		}
	}()
	return watcher
}

// NewRecorder returns an EventRecorder that records events with the given event source.
func (eventBroadcaster *eventBroadcasterImpl) NewRecorder(scheme *runtime.Scheme, source v1.EventSource) EventRecorder {
	return &recorderImpl{scheme, source, eventBroadcaster.Broadcaster, clock.RealClock{}}
}

type recorderImpl struct {
	scheme *runtime.Scheme
	source v1.EventSource
	*watch.Broadcaster
	clock clock.Clock
}

func (recorder *recorderImpl) generateEvent(object runtime.Object, timestamp metav1.Time, eventtype, reason, message string) {
	ref, err := ref.GetReference(recorder.scheme, object)
	if err != nil {
		glog.Errorf("Could not construct reference to: '%#v' due to: '%v'. Will not report event: '%v' '%v' '%v'", object, err, eventtype, reason, message)
		return
	}

	if !validateEventType(eventtype) {
		glog.Errorf("Unsupported event type: '%v'", eventtype)
		return
	}

	event := recorder.makeEvent(ref, eventtype, reason, message)
	event.Source = recorder.source

	go func() {
		// NOTE: events should be a non-blocking operation
		defer utilruntime.HandleCrash()
		recorder.Action(watch.Added, event)
	}()
}

func validateEventType(eventtype string) bool {
	switch eventtype {
	case v1.EventTypeNormal, v1.EventTypeWarning:
		return true
	}
	return false
}

func (recorder *recorderImpl) Event(object runtime.Object, eventtype, reason, message string) {
	recorder.generateEvent(object, metav1.Now(), eventtype, reason, message)
}

func (recorder *recorderImpl) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) {
	recorder.Event(object, eventtype, reason, fmt.Sprintf(messageFmt, args...))
}

func (recorder *recorderImpl) PastEventf(object runtime.Object, timestamp metav1.Time, eventtype, reason, messageFmt string, args ...interface{}) {
	recorder.generateEvent(object, timestamp, eventtype, reason, fmt.Sprintf(messageFmt, args...))
}

func (recorder *recorderImpl) makeEvent(ref *v1.ObjectReference, eventtype, reason, message string) *v1.Event {
	t := metav1.Time{Time: recorder.clock.Now()}
	namespace := ref.Namespace
	if namespace == "" {
		namespace = metav1.NamespaceDefault
	}
	return &v1.Event{
		ObjectMeta: metav1.ObjectMeta{
			Name:      fmt.Sprintf("%v.%x", ref.Name, t.UnixNano()),
			Namespace: namespace,
		},
		InvolvedObject: *ref,
		Reason:         reason,
		Message:        message,
		FirstTimestamp: t,
		LastTimestamp:  t,
		Count:          1,
		Type:           eventtype,
	}
}
<|endoftext|>