<commit_before>package collector

import (
	"bytes"
	"encoding/binary"
	"encoding/csv"
	"encoding/hex"
	"os/exec"
	"regexp"
	"strconv"
        "strings"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
)

type metric struct {
	metricsname string
	value       float64
	unit        string
	addr        string
}

// Exporter implements the prometheus.Collector interface. It exposes the metrics
// of a ipmi node.
type Exporter struct {
	IPMIBinary string

	namespace string
}

// NewExporter instantiates a new ipmi Exporter.
func NewExporter(ipmiBinary string) *Exporter {
	return &Exporter{
		IPMIBinary: ipmiBinary,
		namespace:  "ipmi",
	}
}

func ipmiOutput(cmd string) ([]byte, error) {
	parts := strings.Fields(cmd)
	out, err := exec.Command(parts[0], parts[1:]...).Output()
	if err != nil {
		log.Errorf("error while calling ipmitool: %v", err)
	}
	return out, err
}

func convertValue(strfloat string, strunit string) (value float64, err error) {
	if strfloat != "na" {
		if strunit == "discrete" {
			strfloat = strings.Replace(strfloat, "0x", "", -1)
			parsedValue, err := strconv.ParseUint(strfloat, 16, 32)
			if err != nil {
				log.Errorf("could not translate hex: %v, %v", parsedValue, err)
			}
			value = float64(parsedValue)
		} else {
			value, err = strconv.ParseFloat(strfloat, 64)
		}
	}
	return value, err
}

func convertOutput(result [][]string) (metrics []metric, err error) {
	for _, res := range result {
		var value float64
		var currentMetric metric

		for n := range res {
			res[n] = strings.TrimSpace(res[n])
		}
		value, err = convertValue(res[1], res[2])
		if err != nil {
			log.Errorf("could not parse ipmi output: %s", err)
		}

		currentMetric.value = value
		currentMetric.unit = res[2]
		currentMetric.metricsname = res[0]

		metrics = append(metrics, currentMetric)
	}
	return metrics, err
}

// Convert raw IPMI tool output to decimal numbers
func convertRawOutput(result [][]string) (metrics []metric, err error) {
	for _, res := range result {
		var value []byte
		var currentMetric metric

		for n := range res {
			res[n] = strings.TrimSpace(res[n])
		}
		value, err := hex.DecodeString(res[1])
		if err != nil {
			log.Errorf("could not parse ipmi output: %s", err)
		}
		r, _ := binary.Uvarint(value)
		currentMetric.value = float64(r)
		currentMetric.unit = res[2]
		currentMetric.metricsname = res[0]

		metrics = append(metrics, currentMetric)
	}
	return metrics, err
}

func splitOutput(impiOutput []byte) ([][]string, error) {
	r := csv.NewReader(bytes.NewReader(impiOutput))
	r.Comma = '|'
	r.Comment = '#'
	result, err := r.ReadAll()
	if err != nil {
		log.Errorf("could not parse ipmi output: %v", err)
        return result, err
	}

        keys := make(map[string]int)
        var res [][]string
        for _, v := range result {
            key := v[0]
            if _, ok := keys[key]; ok {
                keys[key] += 1
                v[0] = strings.TrimSpace(v[0]) + strconv.Itoa(keys[key])
            } else {
                keys[key] = 1
            }
            res = append(res, v)
        }
        return res, err
}

// Describe describes all the registered stats metrics from the ipmi node.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	ch <- temperatures
	ch <- fanspeed
	ch <- voltages
	ch <- intrusion
	ch <- powersupply
        ch <- current
}

// Collect collects all the registered stats metrics from the ipmi node.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	output, err := ipmiOutput(e.IPMIBinary + " sensor")
	if err != nil {
		log.Errorln(err)
	}
	splitted, err := splitOutput(output)
	if err != nil {
		log.Errorln(err)
	}
	convertedOutput, err := convertOutput(splitted)
	if err != nil {
		log.Errorln(err)
	}

	for _, res := range convertedOutput {
		push := func(m *prometheus.Desc) {
			ch <- prometheus.MustNewConstMetric(m, prometheus.GaugeValue, res.value, res.metricsname)
		}
		switch strings.ToLower(res.unit) {
		case "degrees c":
			push(temperatures)
		case "volts":
			push(voltages)
		case "rpm":
			push(fanspeed)
                case "watts":
                        push(powersupply)
                case "amps":
                        push(current)
		}

		if matches, err := regexp.MatchString("PS.* Status", res.metricsname); matches && err == nil {
			push(powersupply)
		} else if strings.HasSuffix(res.metricsname, "Chassis Intru") {
			ch <- prometheus.MustNewConstMetric(intrusion, prometheus.GaugeValue, res.value)
		}
	}

	e.collectRaws(ch)
}

// Collect some Supermicro X8-specific metrics with raw commands
func (e *Exporter) collectRaws(ch chan<- prometheus.Metric) {
	commands := [][]string{
		{"InputPowerPSU1", " raw 0x06 0x52 0x07 0x78 0x01 0x97", "W"},
		{"InputPowerPSU2", " raw 0x06 0x52 0x07 0x7a 0x01 0x97", "W"},
	}
	results := [][]string{}
	for _, command := range commands {
		output, err := ipmiOutput(e.IPMIBinary + command[1])
		if err != nil {
			log.Errorln(err)
		}

		results = append(results, []string{command[0], string(output), command[2]})
	}

	convertedRawOutput, err := convertRawOutput(results)
	if err != nil {
		log.Errorln(err)
	}
	for _, res := range convertedRawOutput {
		push := func(m *prometheus.Desc) {
			ch <- prometheus.MustNewConstMetric(m, prometheus.GaugeValue, res.value, res.metricsname)
		}
		push(powersupply)
	}
}
<commit_msg>Disabling raw sensors on first failed query attempt<commit_after>package collector

import (
	"bytes"
	"encoding/binary"
	"encoding/csv"
	"encoding/hex"
	"os/exec"
	"regexp"
	"strconv"
        "strings"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
)

type metric struct {
	metricsname string
	value       float64
	unit        string
	addr        string
}

// Exporter implements the prometheus.Collector interface. It exposes the metrics
// of a ipmi node.
type Exporter struct {
	IPMIBinary string

	namespace string
}

var rawSensors = [][]string{
	{"InputPowerPSU1", " raw 0x06 0x52 0x07 0x78 0x01 0x97", "W", "enabled"},
	{"InputPowerPSU2", " raw 0x06 0x52 0x07 0x7a 0x01 0x97", "W", "enabled"},
}

// NewExporter instantiates a new ipmi Exporter.
func NewExporter(ipmiBinary string) *Exporter {
	return &Exporter{
		IPMIBinary: ipmiBinary,
		namespace:  "ipmi",
	}
}

func ipmiOutput(cmd string) ([]byte, error) {
	parts := strings.Fields(cmd)
	out, err := exec.Command(parts[0], parts[1:]...).Output()
	if err != nil {
		log.Errorf("error while calling ipmitool: %v", err)
	}
	return out, err
}

func convertValue(strfloat string, strunit string) (value float64, err error) {
	if strfloat != "na" {
		if strunit == "discrete" {
			strfloat = strings.Replace(strfloat, "0x", "", -1)
			parsedValue, err := strconv.ParseUint(strfloat, 16, 32)
			if err != nil {
				log.Errorf("could not translate hex: %v, %v", parsedValue, err)
			}
			value = float64(parsedValue)
		} else {
			value, err = strconv.ParseFloat(strfloat, 64)
		}
	}
	return value, err
}

func convertOutput(result [][]string) (metrics []metric, err error) {
	for _, res := range result {
		var value float64
		var currentMetric metric

		for n := range res {
			res[n] = strings.TrimSpace(res[n])
		}
		value, err = convertValue(res[1], res[2])
		if err != nil {
			log.Errorf("could not parse ipmi output: %s", err)
		}

		currentMetric.value = value
		currentMetric.unit = res[2]
		currentMetric.metricsname = res[0]

		metrics = append(metrics, currentMetric)
	}
	return metrics, err
}

// Convert raw IPMI tool output to decimal numbers
func convertRawOutput(result [][]string) (metrics []metric, err error) {
	for _, res := range result {
		var value []byte
		var currentMetric metric

		for n := range res {
			res[n] = strings.TrimSpace(res[n])
		}
		value, err := hex.DecodeString(res[1])
		if err != nil {
			log.Errorf("could not parse ipmi output: %s", err)
		}
		r, _ := binary.Uvarint(value)
		currentMetric.value = float64(r)
		currentMetric.unit = res[2]
		currentMetric.metricsname = res[0]

		metrics = append(metrics, currentMetric)
	}
	return metrics, err
}

func splitOutput(impiOutput []byte) ([][]string, error) {
	r := csv.NewReader(bytes.NewReader(impiOutput))
	r.Comma = '|'
	r.Comment = '#'
	result, err := r.ReadAll()
	if err != nil {
		log.Errorf("could not parse ipmi output: %v", err)
        return result, err
	}

        keys := make(map[string]int)
        var res [][]string
        for _, v := range result {
            key := v[0]
            if _, ok := keys[key]; ok {
                keys[key] += 1
                v[0] = strings.TrimSpace(v[0]) + strconv.Itoa(keys[key])
            } else {
                keys[key] = 1
            }
            res = append(res, v)
        }
        return res, err
}

// Describe describes all the registered stats metrics from the ipmi node.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	ch <- temperatures
	ch <- fanspeed
	ch <- voltages
	ch <- intrusion
	ch <- powersupply
        ch <- current
}

// Collect collects all the registered stats metrics from the ipmi node.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	output, err := ipmiOutput(e.IPMIBinary + " sensor")
	if err != nil {
		log.Errorln(err)
	}
	splitted, err := splitOutput(output)
	if err != nil {
		log.Errorln(err)
	}
	convertedOutput, err := convertOutput(splitted)
	if err != nil {
		log.Errorln(err)
	}

	for _, res := range convertedOutput {
		push := func(m *prometheus.Desc) {
			ch <- prometheus.MustNewConstMetric(m, prometheus.GaugeValue, res.value, res.metricsname)
		}
		switch strings.ToLower(res.unit) {
		case "degrees c":
			push(temperatures)
		case "volts":
			push(voltages)
		case "rpm":
			push(fanspeed)
                case "watts":
                        push(powersupply)
                case "amps":
                        push(current)
		}

		if matches, err := regexp.MatchString("PS.* Status", res.metricsname); matches && err == nil {
			push(powersupply)
		} else if strings.HasSuffix(res.metricsname, "Chassis Intru") {
			ch <- prometheus.MustNewConstMetric(intrusion, prometheus.GaugeValue, res.value)
		}
	}

	e.collectRaws(ch)
}

// Collect some Supermicro X8-specific metrics with raw commands
func (e *Exporter) collectRaws(ch chan<- prometheus.Metric) {
	results := [][]string{}
	for i, command := range rawSensors {
		if command[3] == "enabled" {
			output, err := ipmiOutput(e.IPMIBinary + command[1])
			if err != nil {
				log.Infof("Error detected on quering %v. Disabling this sensor.", command[1])
				rawSensors[i][3] = "disabled"
				log.Errorln(err)
			}

			results = append(results, []string{command[0], string(output), command[2]})
		}
	}

	convertedRawOutput, err := convertRawOutput(results)
	if err != nil {
		log.Errorln(err)
	}
	for _, res := range convertedRawOutput {
		push := func(m *prometheus.Desc) {
			ch <- prometheus.MustNewConstMetric(m, prometheus.GaugeValue, res.value, res.metricsname)
		}
		push(powersupply)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package collector

import (
	"github.com/globocom/tsuru/app"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"labix.org/v2/mgo/bson"
	"sort"
)

// AppList is a list of apps. It's not thread safe.
type AppList []*app.App

func (l AppList) Search(name string) (*app.App, int) {
	index := sort.Search(len(l), func(i int) bool {
		return l[i].Name >= name
	})
	if index < len(l) && l[index].Name == name {
		return l[index], -1
	} else if index < len(l) {
		return &app.App{Name: name}, index
	}
	return &app.App{Name: name}, len(l)
}

func (l *AppList) Add(a *app.App, index int) {
	length := len(*l)
	*l = append(*l, a)
	if index < length {
		for i := length; i > index; i-- {
			(*l)[i] = (*l)[i-1]
		}
		(*l)[index] = a
	}
}

func update(units []provision.Unit) {
	log.Print("updating status from provisioner")
	var l AppList
	for _, unit := range units {
		a, index := l.Search(unit.AppName)
		if index > -1 {
			err := a.Get()
			if err != nil {
				log.Printf("collector: app %q not found. Skipping.\n", unit.AppName)
				continue
			}
			l.Add(a, index)
		}
		u := app.Unit{}
		u.Name = unit.Name
		u.Type = unit.Type
		u.Machine = unit.Machine
		u.InstanceId = unit.InstanceId
		u.Ip = unit.Ip
		if unit.Status == provision.StatusStarted && a.State == "" {
			a.State = "ready"
		}
		u.State = string(unit.Status)
		a.AddUnit(&u)
	}
	conn, err := db.Conn()
	if err != nil {
		log.Printf("collector failed to connect to the database: %s", err)
		return
	}
	defer conn.Close()
	for _, a := range l {
		a.Ip, _ = app.Provisioner.Addr(a)
		conn.Apps().Update(bson.M{"name": a.Name}, a)
	}
}
<commit_msg>collector: log errors in Addr<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package collector

import (
	"github.com/globocom/tsuru/app"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"labix.org/v2/mgo/bson"
	"sort"
)

// AppList is a list of apps. It's not thread safe.
type AppList []*app.App

func (l AppList) Search(name string) (*app.App, int) {
	index := sort.Search(len(l), func(i int) bool {
		return l[i].Name >= name
	})
	if index < len(l) && l[index].Name == name {
		return l[index], -1
	} else if index < len(l) {
		return &app.App{Name: name}, index
	}
	return &app.App{Name: name}, len(l)
}

func (l *AppList) Add(a *app.App, index int) {
	length := len(*l)
	*l = append(*l, a)
	if index < length {
		for i := length; i > index; i-- {
			(*l)[i] = (*l)[i-1]
		}
		(*l)[index] = a
	}
}

func update(units []provision.Unit) {
	log.Print("updating status from provisioner")
	var l AppList
	for _, unit := range units {
		a, index := l.Search(unit.AppName)
		if index > -1 {
			err := a.Get()
			if err != nil {
				log.Printf("collector: app %q not found. Skipping.\n", unit.AppName)
				continue
			}
			l.Add(a, index)
		}
		u := app.Unit{}
		u.Name = unit.Name
		u.Type = unit.Type
		u.Machine = unit.Machine
		u.InstanceId = unit.InstanceId
		u.Ip = unit.Ip
		if unit.Status == provision.StatusStarted && a.State == "" {
			a.State = "ready"
		}
		u.State = string(unit.Status)
		a.AddUnit(&u)
	}
	conn, err := db.Conn()
	if err != nil {
		log.Printf("collector failed to connect to the database: %s", err)
		return
	}
	defer conn.Close()
	for _, a := range l {
		a.Ip, err = app.Provisioner.Addr(a)
		if err != nil {
			log.Printf("collector failed to get app (%q) address: %s", a.Name, err)
		}
		conn.Apps().Update(bson.M{"name": a.Name}, a)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package collector

import (
	"github.com/globocom/tsuru/app"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"labix.org/v2/mgo/bson"
	"sort"
)

// AppList is a list of apps. It's not thread safe.
type AppList []*app.App

func (l AppList) Search(name string) (*app.App, int) {
	index := sort.Search(len(l), func(i int) bool {
		return l[i].Name >= name
	})
	if index < len(l) && l[index].Name == name {
		return l[index], -1
	} else if index < len(l) {
		return &app.App{Name: name}, index
	}
	return &app.App{Name: name}, len(l)
}

func (l *AppList) Add(a *app.App, index int) {
	length := len(*l)
	*l = append(*l, a)
	if index < length {
		for i := length; i > index; i-- {
			(*l)[i] = (*l)[i-1]
		}
		(*l)[index] = a
	}
}

func update(units []provision.Unit) {
	log.Print("updating status from provisioner")
	var l AppList
	for _, unit := range units {
		a, index := l.Search(unit.AppName)
		if index > -1 {
			err := a.Get()
			if err != nil {
				log.Printf("collector: app %q not found. Skipping.\n", unit.AppName)
				continue
			}
		}
		u := app.Unit{}
		u.Name = unit.Name
		u.Type = unit.Type
		u.Machine = unit.Machine
		u.InstanceId = unit.InstanceId
		u.Ip = unit.Ip
		u.State = string(unit.Status)
		a.AddUnit(&u)
		if index > -1 {
			l.Add(a, index)
		}
	}
	conn, err := db.Conn()
	if err != nil {
		log.Printf("collector failed to connect to the database: %s", err)
		return
	}
	defer conn.Close()
	for _, a := range l {
		a.Ip, _ = app.Provisioner.Addr(a)
		conn.Apps().Update(bson.M{"name": a.Name}, a)
	}
}
<commit_msg>collector: simplify code in the update function<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package collector

import (
	"github.com/globocom/tsuru/app"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"labix.org/v2/mgo/bson"
	"sort"
)

// AppList is a list of apps. It's not thread safe.
type AppList []*app.App

func (l AppList) Search(name string) (*app.App, int) {
	index := sort.Search(len(l), func(i int) bool {
		return l[i].Name >= name
	})
	if index < len(l) && l[index].Name == name {
		return l[index], -1
	} else if index < len(l) {
		return &app.App{Name: name}, index
	}
	return &app.App{Name: name}, len(l)
}

func (l *AppList) Add(a *app.App, index int) {
	length := len(*l)
	*l = append(*l, a)
	if index < length {
		for i := length; i > index; i-- {
			(*l)[i] = (*l)[i-1]
		}
		(*l)[index] = a
	}
}

func update(units []provision.Unit) {
	log.Print("updating status from provisioner")
	var l AppList
	for _, unit := range units {
		a, index := l.Search(unit.AppName)
		if index > -1 {
			err := a.Get()
			if err != nil {
				log.Printf("collector: app %q not found. Skipping.\n", unit.AppName)
				continue
			}
			l.Add(a, index)
		}
		u := app.Unit{}
		u.Name = unit.Name
		u.Type = unit.Type
		u.Machine = unit.Machine
		u.InstanceId = unit.InstanceId
		u.Ip = unit.Ip
		u.State = string(unit.Status)
		a.AddUnit(&u)
	}
	conn, err := db.Conn()
	if err != nil {
		log.Printf("collector failed to connect to the database: %s", err)
		return
	}
	defer conn.Close()
	for _, a := range l {
		a.Ip, _ = app.Provisioner.Addr(a)
		conn.Apps().Update(bson.M{"name": a.Name}, a)
	}
}
<|endoftext|>


<commit_before>package benchlist

import (
	"container/list"
	"math/rand"
	"sync"
	"time"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/snow"
	"github.com/ava-labs/avalanchego/snow/validators"
	"github.com/ava-labs/avalanchego/utils/constants"
	"github.com/ava-labs/avalanchego/utils/timer"

	safemath "github.com/ava-labs/avalanchego/utils/math"
)

// QueryBenchlist ...
type QueryBenchlist interface {
	// RegisterQuery registers a sent query and returns whether the query is subject to benchlist
	RegisterQuery(validatorID ids.ShortID, requestID uint32, msgType constants.MsgType) bool
	// RegisterResponse registers the response to a query message
	RegisterResponse(validatorID ids.ShortID, requstID uint32)
	// QueryFailed registers that a query did not receive a response within our synchrony bound
	QueryFailed(validatorID ids.ShortID, requestID uint32)
}

// If a peer consistently does not respond to queries, it will
// increase latencies on the network whenever that peer is polled.
// If we cannot terminate the poll early, then the poll will wait
// the full timeout before finalizing the poll and making progress.
// This can increase network latencies to an undesirable level.

// Therefore, a benchlist is used as a heurstic to immediately fail
// queries to nodes that are consistently not responding.

type queryBenchlist struct {
	vdrs validators.Set
	// Validator ID --> Request ID --> non-empty iff
	// there is an outstanding request to this validator
	// with the corresponding requestID
	pendingQueries map[[20]byte]map[uint32]pendingQuery
	// Map of consecutive query failures
	consecutiveFailures map[[20]byte]failureStreak

	// Maintain benchlist
	benchlistTimes map[[20]byte]time.Time
	benchlistOrder *list.List
	benchlistSet   ids.ShortSet

	threshold              int
	minimumFailingDuration time.Duration
	duration               time.Duration
	maxPortion             float64

	clock timer.Clock

	metrics *metrics
	ctx     *snow.Context

	lock sync.Mutex
}

type pendingQuery struct {
	registered time.Time
	msgType    constants.MsgType
}

type failureStreak struct {
	firstFailure time.Time
	consecutive  int
}

// NewQueryBenchlist ...
func NewQueryBenchlist(
	validators validators.Set,
	ctx *snow.Context,
	threshold int,
	minimumFailingDuration,
	duration time.Duration,
	maxPortion float64,
	summaryEnabled bool,
	namespace string,
) QueryBenchlist {
	metrics := &metrics{}
	metrics.Initialize(ctx, namespace, summaryEnabled)

	return &queryBenchlist{
		pendingQueries:         make(map[[20]byte]map[uint32]pendingQuery),
		consecutiveFailures:    make(map[[20]byte]failureStreak),
		benchlistTimes:         make(map[[20]byte]time.Time),
		benchlistOrder:         list.New(),
		benchlistSet:           ids.ShortSet{},
		vdrs:                   validators,
		threshold:              threshold,
		minimumFailingDuration: minimumFailingDuration,
		duration:               duration,
		maxPortion:             maxPortion,
		ctx:                    ctx,
		metrics:                metrics,
	}
}

// RegisterQuery attempts to register a query from [validatorID] and returns true
// if that request should be made (not subject to benchlisting)
func (b *queryBenchlist) RegisterQuery(validatorID ids.ShortID, requestID uint32, msgType constants.MsgType) bool {
	b.lock.Lock()
	defer b.lock.Unlock()

	key := validatorID.Key()
	if benched := b.benched(validatorID); benched {
		return false
	}

	validatorRequests, ok := b.pendingQueries[key]
	if !ok {
		validatorRequests = make(map[uint32]pendingQuery)
		b.pendingQueries[key] = validatorRequests
	}

	validatorRequests[requestID] = pendingQuery{
		registered: b.clock.Time(),
		msgType:    msgType,
	}
	return true
}

// RegisterResponse removes the query from pending
func (b *queryBenchlist) RegisterResponse(validatorID ids.ShortID, requestID uint32) {
	b.lock.Lock()
	defer b.lock.Unlock()

	if ok := b.removeQuery(validatorID, requestID); !ok {
		return
	}

	// Reset consecutive failures on success
	delete(b.consecutiveFailures, validatorID.Key())
}

// QueryFailed notes a failure and benchlists [validatorID] if necessary
func (b *queryBenchlist) QueryFailed(validatorID ids.ShortID, requestID uint32) {
	b.lock.Lock()
	defer b.lock.Unlock()

	if ok := b.removeQuery(validatorID, requestID); !ok {
		return
	}

	key := validatorID.Key()
	// Tracks a message failure and benches [validatorID] if it has
	// surpassed the threshold

	// Get the current time
	currentTime := b.clock.Time()
	failureStreak := b.consecutiveFailures[key]
	failureStreak.consecutive++
	if failureStreak.firstFailure.IsZero() {
		failureStreak.firstFailure = currentTime
		b.consecutiveFailures[key] = failureStreak
		// Return early here since a peer will not
		// be benched after its first failure
		return
	}
	b.consecutiveFailures[key] = failureStreak

	if failureStreak.consecutive >= b.threshold && currentTime.After(failureStreak.firstFailure.Add(b.minimumFailingDuration)) {
		b.bench(validatorID)
	}
}

func (b *queryBenchlist) bench(validatorID ids.ShortID) {
	if b.benchlistSet.Contains(validatorID) {
		return
	}

	key := validatorID.Key()

	// Goal:
	// Random end time in the range:
	// [max(lastEndTime,(currentTime + (duration/2)): currentTime + duration]
	// This maintains the invariant that validators in benchlistOrder are
	// ordered by the time that they should be unbenched
	currTime := b.clock.Time()
	minEndTime := currTime.Add(b.duration / 2)
	if elem := b.benchlistOrder.Back(); elem != nil {
		lastValidator := elem.Value.(ids.ShortID)
		lastEndTime := b.benchlistTimes[lastValidator.Key()]
		if lastEndTime.After(minEndTime) {
			minEndTime = lastEndTime
		}
	}
	maxEndTime := currTime.Add(b.duration)
	// Since maxEndTime is at least [duration] in the future and every element
	// added to benchlist was added in the past with an end time at most [duration]
	// in the future, this should never produce a negative duration.
	diff := maxEndTime.Sub(minEndTime)
	randomizedEndTime := minEndTime.Add(time.Duration(rand.Float64() * float64(diff))) // #nosec G404

	// Add to benchlist times with randomized delay
	b.benchlistTimes[key] = randomizedEndTime
	b.benchlistOrder.PushBack(validatorID)
	b.benchlistSet.Add(validatorID)
	delete(b.consecutiveFailures, key)
	b.ctx.Log.Debug(
		"benching validator %s after %d consecutive failed queries for %s",
		validatorID,
		b.threshold,
		randomizedEndTime.Sub(currTime),
	)

	// Note: there could be a memory leak if a large number of
	// validators were added, sampled, benched, and never sampled
	// again. Due to the minimum staking amount and durations this
	// is not a realistic concern.
	b.cleanup()
}

// benched checks if [validatorID] is currently benched
// and calls cleanup if its benching period has elapsed
func (b *queryBenchlist) benched(validatorID ids.ShortID) bool {
	key := validatorID.Key()

	end, ok := b.benchlistTimes[key]
	if !ok {
		return false
	}

	if b.clock.Time().Before(end) {
		return true
	}

	// If a benched item has expired, cleanup the benchlist
	b.cleanup()
	return false
}

// cleanup ensures that we have not benched too much stake
// and removes anything from the benchlist whose time has expired
func (b *queryBenchlist) cleanup() {
	currentWeight, err := b.vdrs.SubsetWeight(b.benchlistSet)
	if err != nil {
		// Add log for this, should never happen
		b.ctx.Log.Error("failed to calculate subset weight due to: %s... Resetting benchlist", err)
		b.reset()
		return
	}

	currentTime := b.clock.Time()

	benchLen := b.benchlistSet.Len()
	updatedWeight := currentWeight
	totalWeight := b.vdrs.Weight()
	maxBenchlistWeight := uint64(float64(totalWeight) * b.maxPortion)

	// Iterate over elements of the benchlist in order of expiration
	for e := b.benchlistOrder.Front(); e != nil; {
		validatorID := e.Value.(ids.ShortID)
		key := validatorID.Key()
		end := b.benchlistTimes[key]
		// Remove elements with the next expiration until the next item has not
		// expired and the bench has less than the maximum weight
		// Note: this creates an edge case where benchlisting a validator
		// with a sufficient stake may clear the benchlist
		if currentTime.Before(end) && updatedWeight < maxBenchlistWeight {
			break
		}

		removeWeight, ok := b.vdrs.GetWeight(validatorID)
		if ok {
			newWeight, err := safemath.Sub64(updatedWeight, removeWeight)
			if err != nil {
				b.ctx.Log.Error("failed to calculate new subset weight due to: %s... Resetting benchlist", err)
				b.reset()
				return
			}
			updatedWeight = newWeight
		}

		next := e.Next()
		b.ctx.Log.Debug("Removed Validator: (%s, %d). EndTime: %s. CurrentTime: %s)", validatorID, removeWeight, end, currentTime)
		b.benchlistOrder.Remove(e)
		delete(b.benchlistTimes, key)
		b.benchlistSet.Remove(validatorID)
		e = next
	}

	updatedBenchLen := b.benchlistSet.Len()
	b.ctx.Log.Debug("Benched Weight: (%d/%d) -> (%d/%d). Benched Validators: %d -> %d.",
		currentWeight,
		totalWeight,
		updatedWeight,
		totalWeight,
		benchLen,
		updatedBenchLen,
	)
	b.metrics.weightBenched.Set(float64(updatedWeight))
	b.metrics.numBenched.Set(float64(updatedBenchLen))
}

func (b *queryBenchlist) reset() {
	b.pendingQueries = make(map[[20]byte]map[uint32]pendingQuery)
	b.consecutiveFailures = make(map[[20]byte]failureStreak)
	b.benchlistTimes = make(map[[20]byte]time.Time)
	b.benchlistOrder.Init()
	b.benchlistSet.Clear()
	b.metrics.weightBenched.Set(0)
	b.metrics.numBenched.Set(0)
}

// removeQuery returns true if the query was present
func (b *queryBenchlist) removeQuery(validatorID ids.ShortID, requestID uint32) bool {
	key := validatorID.Key()

	validatorRequests, ok := b.pendingQueries[key]
	if !ok {
		return false
	}

	query, ok := validatorRequests[requestID]
	if !ok {
		return false
	}

	delete(validatorRequests, requestID)
	if len(validatorRequests) == 0 {
		delete(b.pendingQueries, key)
	}
	b.metrics.observe(validatorID, query.msgType, b.clock.Time().Sub(query.registered))
	return true
}
<commit_msg>cleanup comments<commit_after>package benchlist

import (
	"container/list"
	"math/rand"
	"sync"
	"time"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/snow"
	"github.com/ava-labs/avalanchego/snow/validators"
	"github.com/ava-labs/avalanchego/utils/constants"
	"github.com/ava-labs/avalanchego/utils/timer"

	safemath "github.com/ava-labs/avalanchego/utils/math"
)

// If a peer consistently does not respond to queries, it will
// increase latencies on the network whenever that peer is polled.
// If we cannot terminate the poll early, then the poll will wait
// the full timeout before finalizing the poll and making progress.
// This can increase network latencies to an undesirable level.

// Therefore, nodes that consistently fail are "benched" such that
// queries to that node fail immediately to avoid waiting up to
// the full network timeout for a response.

// QueryBenchlist ...
type QueryBenchlist interface {
	// RegisterQuery registers a sent query and returns whether the query is subject to benchlist
	RegisterQuery(validatorID ids.ShortID, requestID uint32, msgType constants.MsgType) bool
	// RegisterResponse registers the response to a query message
	RegisterResponse(validatorID ids.ShortID, requstID uint32)
	// QueryFailed registers that a query did not receive a response within our synchrony bound
	QueryFailed(validatorID ids.ShortID, requestID uint32)
}

type queryBenchlist struct {
	vdrs validators.Set
	// Validator ID --> Request ID --> non-empty iff
	// there is an outstanding request to this validator
	// with the corresponding requestID
	pendingQueries map[[20]byte]map[uint32]pendingQuery
	// Map of consecutive query failures
	consecutiveFailures map[[20]byte]failureStreak

	// Maintain benchlist
	benchlistTimes map[[20]byte]time.Time
	benchlistOrder *list.List
	benchlistSet   ids.ShortSet

	threshold              int
	minimumFailingDuration time.Duration
	duration               time.Duration
	maxPortion             float64

	clock timer.Clock

	metrics *metrics
	ctx     *snow.Context

	lock sync.Mutex
}

type pendingQuery struct {
	registered time.Time
	msgType    constants.MsgType
}

type failureStreak struct {
	firstFailure time.Time
	consecutive  int
}

// NewQueryBenchlist ...
func NewQueryBenchlist(
	validators validators.Set,
	ctx *snow.Context,
	threshold int,
	minimumFailingDuration,
	duration time.Duration,
	maxPortion float64,
	summaryEnabled bool,
	namespace string,
) QueryBenchlist {
	metrics := &metrics{}
	metrics.Initialize(ctx, namespace, summaryEnabled)

	return &queryBenchlist{
		pendingQueries:         make(map[[20]byte]map[uint32]pendingQuery),
		consecutiveFailures:    make(map[[20]byte]failureStreak),
		benchlistTimes:         make(map[[20]byte]time.Time),
		benchlistOrder:         list.New(),
		benchlistSet:           ids.ShortSet{},
		vdrs:                   validators,
		threshold:              threshold,
		minimumFailingDuration: minimumFailingDuration,
		duration:               duration,
		maxPortion:             maxPortion,
		ctx:                    ctx,
		metrics:                metrics,
	}
}

// RegisterQuery attempts to register a query from [validatorID] and returns true
// if that request should be made (not subject to benchlisting)
func (b *queryBenchlist) RegisterQuery(validatorID ids.ShortID, requestID uint32, msgType constants.MsgType) bool {
	b.lock.Lock()
	defer b.lock.Unlock()

	key := validatorID.Key()
	if benched := b.benched(validatorID); benched {
		return false
	}

	validatorRequests, ok := b.pendingQueries[key]
	if !ok {
		validatorRequests = make(map[uint32]pendingQuery)
		b.pendingQueries[key] = validatorRequests
	}

	validatorRequests[requestID] = pendingQuery{
		registered: b.clock.Time(),
		msgType:    msgType,
	}
	return true
}

// RegisterResponse removes the query from pending
func (b *queryBenchlist) RegisterResponse(validatorID ids.ShortID, requestID uint32) {
	b.lock.Lock()
	defer b.lock.Unlock()

	if ok := b.removeQuery(validatorID, requestID); !ok {
		return
	}

	// Reset consecutive failures on success
	delete(b.consecutiveFailures, validatorID.Key())
}

// QueryFailed notes a failure and benchlists [validatorID] if necessary
func (b *queryBenchlist) QueryFailed(validatorID ids.ShortID, requestID uint32) {
	b.lock.Lock()
	defer b.lock.Unlock()

	if ok := b.removeQuery(validatorID, requestID); !ok {
		return
	}

	// Track the message failure and bench [validatorID] if it has
	// surpassed the threshold
	key := validatorID.Key()
	currentTime := b.clock.Time()
	failureStreak := b.consecutiveFailures[key]
	failureStreak.consecutive++
	if failureStreak.firstFailure.IsZero() {
		failureStreak.firstFailure = currentTime
		b.consecutiveFailures[key] = failureStreak
		// Return early here since a peer will not
		// be benched after its first failure
		return
	}
	b.consecutiveFailures[key] = failureStreak

	if failureStreak.consecutive >= b.threshold && currentTime.After(failureStreak.firstFailure.Add(b.minimumFailingDuration)) {
		b.bench(validatorID)
	}
}

func (b *queryBenchlist) bench(validatorID ids.ShortID) {
	if b.benchlistSet.Contains(validatorID) {
		return
	}

	key := validatorID.Key()

	// Goal:
	// Random end time in the range:
	// [max(lastEndTime, (currentTime + (duration/2)): currentTime + duration]
	// This maintains the invariant that validators in benchlistOrder are
	// ordered by the time that they should be unbenched
	currTime := b.clock.Time()
	minEndTime := currTime.Add(b.duration / 2)
	if elem := b.benchlistOrder.Back(); elem != nil {
		lastValidator := elem.Value.(ids.ShortID)
		lastEndTime := b.benchlistTimes[lastValidator.Key()]
		if lastEndTime.After(minEndTime) {
			minEndTime = lastEndTime
		}
	}
	maxEndTime := currTime.Add(b.duration)
	// Since maxEndTime is at least [duration] in the future and every element
	// added to benchlist was added in the past with an end time at most [duration]
	// in the future, this should never produce a negative duration.
	diff := maxEndTime.Sub(minEndTime)
	randomizedEndTime := minEndTime.Add(time.Duration(rand.Float64() * float64(diff))) // #nosec G404

	// Add to benchlist times with randomized delay
	b.benchlistTimes[key] = randomizedEndTime
	b.benchlistOrder.PushBack(validatorID)
	b.benchlistSet.Add(validatorID)
	delete(b.consecutiveFailures, key)
	b.ctx.Log.Debug(
		"benching validator %s after %d consecutive failed queries for %s",
		validatorID,
		b.threshold,
		randomizedEndTime.Sub(currTime),
	)

	// Note: there could be a memory leak if a large number of
	// validators were added, sampled, benched, and never sampled
	// again. Due to the minimum staking amount and durations this
	// is not a realistic concern.
	b.cleanup()
}

// benched checks if [validatorID] is currently benched
// and calls cleanup if its benching period has elapsed
func (b *queryBenchlist) benched(validatorID ids.ShortID) bool {
	key := validatorID.Key()

	end, ok := b.benchlistTimes[key]
	if !ok {
		return false
	}

	if b.clock.Time().Before(end) {
		return true
	}

	// If a benched item has expired, cleanup the benchlist
	b.cleanup()
	return false
}

// cleanup ensures that we have not benched too much stake
// and removes anything from the benchlist whose time has expired
func (b *queryBenchlist) cleanup() {
	currentWeight, err := b.vdrs.SubsetWeight(b.benchlistSet)
	if err != nil {
		// Add log for this, should never happen
		b.ctx.Log.Error("failed to calculate subset weight due to: %s... Resetting benchlist", err)
		b.reset()
		return
	}

	currentTime := b.clock.Time()

	benchLen := b.benchlistSet.Len()
	updatedWeight := currentWeight
	totalWeight := b.vdrs.Weight()
	maxBenchlistWeight := uint64(float64(totalWeight) * b.maxPortion)

	// Iterate over elements of the benchlist in order of expiration
	for e := b.benchlistOrder.Front(); e != nil; {
		validatorID := e.Value.(ids.ShortID)
		key := validatorID.Key()
		end := b.benchlistTimes[key]
		// Remove elements with the next expiration until the next item has not
		// expired and the bench has less than the maximum weight
		// Note: this creates an edge case where benching a validator
		// with a sufficient stake may clear the bench if the benchlist is
		// not parameterized correctly.
		if currentTime.Before(end) && updatedWeight < maxBenchlistWeight {
			break
		}

		removeWeight, ok := b.vdrs.GetWeight(validatorID)
		if ok {
			newWeight, err := safemath.Sub64(updatedWeight, removeWeight)
			if err != nil {
				b.ctx.Log.Error("failed to calculate new subset weight due to: %s... Resetting benchlist", err)
				b.reset()
				return
			}
			updatedWeight = newWeight
		}

		next := e.Next()
		b.ctx.Log.Debug("Removed Validator: (%s, %d). EndTime: %s. CurrentTime: %s)", validatorID, removeWeight, end, currentTime)
		b.benchlistOrder.Remove(e)
		delete(b.benchlistTimes, key)
		b.benchlistSet.Remove(validatorID)
		e = next
	}

	updatedBenchLen := b.benchlistSet.Len()
	b.ctx.Log.Debug("Benched Weight: (%d/%d) -> (%d/%d). Benched Validators: %d -> %d.",
		currentWeight,
		totalWeight,
		updatedWeight,
		totalWeight,
		benchLen,
		updatedBenchLen,
	)
	b.metrics.weightBenched.Set(float64(updatedWeight))
	b.metrics.numBenched.Set(float64(updatedBenchLen))
}

func (b *queryBenchlist) reset() {
	b.pendingQueries = make(map[[20]byte]map[uint32]pendingQuery)
	b.consecutiveFailures = make(map[[20]byte]failureStreak)
	b.benchlistTimes = make(map[[20]byte]time.Time)
	b.benchlistOrder.Init()
	b.benchlistSet.Clear()
	b.metrics.weightBenched.Set(0)
	b.metrics.numBenched.Set(0)
}

// removeQuery returns true if the query was present
func (b *queryBenchlist) removeQuery(validatorID ids.ShortID, requestID uint32) bool {
	key := validatorID.Key()

	validatorRequests, ok := b.pendingQueries[key]
	if !ok {
		return false
	}

	query, ok := validatorRequests[requestID]
	if !ok {
		return false
	}

	delete(validatorRequests, requestID)
	if len(validatorRequests) == 0 {
		delete(b.pendingQueries, key)
	}
	b.metrics.observe(validatorID, query.msgType, b.clock.Time().Sub(query.registered))
	return true
}
<|endoftext|>


<commit_before>// Copyright 2016 Mesosphere, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package node

import (
	"math"
	"sync"

	"github.com/dcos/dcos-metrics/producers"
	"github.com/shirou/gopsutil/cpu"
)

type cpuCoresMetric struct {
	cpuCores  int32
	cpuTotal  float64
	cpuUser   float64
	cpuSystem float64
	cpuIdle   float64
	cpuWait   float64
	timestamp string
}

func (m *cpuCoresMetric) poll() error {
	ts := thisTime()
	hi, err := getNumCores()
	if err != nil {
		return err
	}

	cpuStatePcts := calculatePcts(getCPUTimes())

	m.cpuCores = hi
	m.timestamp = ts
	m.cpuTotal = cpuStatePcts.User + cpuStatePcts.System
	m.cpuUser = cpuStatePcts.User
	m.cpuSystem = cpuStatePcts.System
	m.cpuIdle = cpuStatePcts.Idle
	m.cpuWait = cpuStatePcts.Iowait

	return nil
}

func (m *cpuCoresMetric) addDatapoints(nc *nodeCollector) error {
	cpuDps := []producers.Datapoint{
		producers.Datapoint{
			Name:      CPU_CORES,
			Unit:      COUNT,
			Value:     m.cpuCores,
			Timestamp: m.timestamp,
		},
		producers.Datapoint{
			Name:      CPU_TOTAL,
			Unit:      COUNT,
			Value:     m.cpuTotal,
			Timestamp: m.timestamp,
		},
		producers.Datapoint{
			Name:      CPU_USER,
			Unit:      COUNT,
			Value:     m.cpuUser,
			Timestamp: m.timestamp,
		},
		producers.Datapoint{
			Name:      CPU_SYSTEM,
			Unit:      COUNT,
			Value:     m.cpuSystem,
			Timestamp: m.timestamp,
		},
		producers.Datapoint{
			Name:      CPU_IDLE,
			Unit:      COUNT,
			Value:     m.cpuIdle,
			Timestamp: m.timestamp,
		},
		producers.Datapoint{
			Name:      CPU_WAIT,
			Unit:      COUNT,
			Value:     m.cpuWait,
			Timestamp: m.timestamp,
		},
	}

	for _, dp := range cpuDps {
		nc.datapoints = append(nc.datapoints, dp)
	}

	return nil
}

// -- helpers
func getNumCores() (int32, error) {
	cores := int32(0)
	cpus, err := cpu.Info()
	if err != nil {
		return cores, err
	}
	for _, c := range cpus {
		cores += c.Cores
	}
	return cores, nil
}

/* CPU Helper Methods */
type lastCPUTimes struct {
	sync.Mutex
	times cpu.TimesStat
}

var lastCPU lastCPUTimes

func init() {
	t, _ := cpu.Times(false) // get totals, not per-cpu stats
	lastCPU.Lock()
	lastCPU.times = t[0]
	lastCPU.Unlock()
}

func getCPUTimes() (cpu.TimesStat, cpu.TimesStat) {
	currentTimes, _ := cpu.Times(false) // get totals, not per-cpu stats
	lastTimes := lastCPU.times

	lastCPU.Lock()
	lastCPU.times = currentTimes[0] // update lastTimes to the currentTimes
	lastCPU.Unlock()

	return currentTimes[0], lastTimes
}

// calculatePct returns the percent utilization for CPU states. 100.00 => 100.00%
func calculatePcts(lastTimes cpu.TimesStat, curTimes cpu.TimesStat) cpu.TimesStat {
	totalDelta := curTimes.Total() - lastTimes.Total()
	if totalDelta == 0 {
		totalDelta = 1 // can't divide by zero
	}
	return cpu.TimesStat{
		User:      round(math.Dim(curTimes.User, lastTimes.User) / totalDelta * 100),
		System:    round(math.Dim(curTimes.System, lastTimes.System) / totalDelta * 100),
		Idle:      round(math.Dim(curTimes.Idle, lastTimes.Idle) / totalDelta * 100),
		Nice:      round(math.Dim(curTimes.Nice, lastTimes.Nice) / totalDelta * 100),
		Iowait:    round(math.Dim(curTimes.Iowait, lastTimes.Iowait) / totalDelta * 100),
		Irq:       round(math.Dim(curTimes.Irq, lastTimes.Irq) / totalDelta * 100),
		Softirq:   round(math.Dim(curTimes.Softirq, lastTimes.Softirq) / totalDelta * 100),
		Steal:     round(math.Dim(curTimes.Steal, lastTimes.Steal) / totalDelta * 100),
		Guest:     round(math.Dim(curTimes.Guest, lastTimes.Guest) / totalDelta * 100),
		GuestNice: round(math.Dim(curTimes.GuestNice, lastTimes.GuestNice) / totalDelta * 100),
		Stolen:    round(math.Dim(curTimes.Stolen, lastTimes.Stolen) / totalDelta * 100),
	}
}

// Helper function for rounding to two decimal places
func round(f float64) float64 {
	shift := math.Pow(10, float64(2))
	return math.Floor(f*shift+.5) / shift
}
<commit_msg>Address PR comments<commit_after>// Copyright 2016 Mesosphere, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package node

import (
	"math"
	"sync"

	"github.com/dcos/dcos-metrics/producers"
	"github.com/shirou/gopsutil/cpu"
)

type cpuCoresMetric struct {
	cpuCores  int32
	cpuTotal  float64
	cpuUser   float64
	cpuSystem float64
	cpuIdle   float64
	cpuWait   float64
	timestamp string
}

func (m *cpuCoresMetric) poll() error {
	ts := thisTime()
	hi, err := getNumCores()
	if err != nil {
		return err
	}

	currentTime, lastTime, err := getCPUTimes()
	if err != nil {
		return err
	}

	cpuStatePcts := calculatePcts(currentTime, lastTime)

	m.cpuCores = hi
	m.timestamp = ts
	m.cpuTotal = cpuStatePcts.User + cpuStatePcts.System
	m.cpuUser = cpuStatePcts.User
	m.cpuSystem = cpuStatePcts.System
	m.cpuIdle = cpuStatePcts.Idle
	m.cpuWait = cpuStatePcts.Iowait

	return nil
}

func (m *cpuCoresMetric) addDatapoints(nc *nodeCollector) error {
	cpuDps := []producers.Datapoint{
		producers.Datapoint{
			Name:      CPU_CORES,
			Unit:      COUNT,
			Value:     m.cpuCores,
			Timestamp: m.timestamp,
		},
		producers.Datapoint{
			Name:      CPU_TOTAL,
			Unit:      COUNT,
			Value:     m.cpuTotal,
			Timestamp: m.timestamp,
		},
		producers.Datapoint{
			Name:      CPU_USER,
			Unit:      COUNT,
			Value:     m.cpuUser,
			Timestamp: m.timestamp,
		},
		producers.Datapoint{
			Name:      CPU_SYSTEM,
			Unit:      COUNT,
			Value:     m.cpuSystem,
			Timestamp: m.timestamp,
		},
		producers.Datapoint{
			Name:      CPU_IDLE,
			Unit:      COUNT,
			Value:     m.cpuIdle,
			Timestamp: m.timestamp,
		},
		producers.Datapoint{
			Name:      CPU_WAIT,
			Unit:      COUNT,
			Value:     m.cpuWait,
			Timestamp: m.timestamp,
		},
	}

	nc.datapoints = append(nc.datapoints, cpuDps...)
	return nil
}

// -- helpers
func getNumCores() (int32, error) {
	cores := int32(0)
	cpus, err := cpu.Info()
	if err != nil {
		return cores, err
	}
	for _, c := range cpus {
		cores += c.Cores
	}
	return cores, nil
}

/* CPU Helper Methods */
type lastCPUTimes struct {
	sync.Mutex
	times cpu.TimesStat
}

var lastCPU lastCPUTimes

func init() {
	t, _ := cpu.Times(false) // get totals, not per-cpu stats
	lastCPU.Lock()
	lastCPU.times = t[0]
	lastCPU.Unlock()
}

func getCPUTimes() (cpu.TimesStat, cpu.TimesStat, error) {
	currentTimes, err := cpu.Times(false) // get totals, not per-cpu stats
	if err != nil {
		return cpu.TimesStat{}, cpu.TimesStat{}, err
	}

	lastTimes := lastCPU.times

	lastCPU.Lock()
	lastCPU.times = currentTimes[0] // update lastTimes to the currentTimes
	lastCPU.Unlock()

	return currentTimes[0], lastTimes, nil
}

// calculatePct returns the percent utilization for CPU states. 100.00 => 100.00%
func calculatePcts(lastTimes cpu.TimesStat, curTimes cpu.TimesStat) cpu.TimesStat {
	totalDelta := curTimes.Total() - lastTimes.Total()
	if totalDelta == 0 {
		totalDelta = 1 // can't divide by zero
	}
	return cpu.TimesStat{
		User:      round(math.Dim(curTimes.User, lastTimes.User) / totalDelta * 100),
		System:    round(math.Dim(curTimes.System, lastTimes.System) / totalDelta * 100),
		Idle:      round(math.Dim(curTimes.Idle, lastTimes.Idle) / totalDelta * 100),
		Nice:      round(math.Dim(curTimes.Nice, lastTimes.Nice) / totalDelta * 100),
		Iowait:    round(math.Dim(curTimes.Iowait, lastTimes.Iowait) / totalDelta * 100),
		Irq:       round(math.Dim(curTimes.Irq, lastTimes.Irq) / totalDelta * 100),
		Softirq:   round(math.Dim(curTimes.Softirq, lastTimes.Softirq) / totalDelta * 100),
		Steal:     round(math.Dim(curTimes.Steal, lastTimes.Steal) / totalDelta * 100),
		Guest:     round(math.Dim(curTimes.Guest, lastTimes.Guest) / totalDelta * 100),
		GuestNice: round(math.Dim(curTimes.GuestNice, lastTimes.GuestNice) / totalDelta * 100),
		Stolen:    round(math.Dim(curTimes.Stolen, lastTimes.Stolen) / totalDelta * 100),
	}
}

// Helper function for rounding to two decimal places
func round(f float64) float64 {
	shift := math.Pow(10, float64(2))
	return math.Floor(f*shift+.5) / shift
}
<|endoftext|>


<commit_before>package agent

import (
	"encoding/json"
	"fmt"
	"github.com/hashicorp/serf/serf"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"
)

// Agent starts and manages a Serf instance, adding some niceties
// on top of Serf such as storing logs that you can later retrieve,
// and invoking EventHandlers when events occur.
type Agent struct {
	// Stores the serf configuration
	conf *serf.Config

	// Stores the agent configuration
	agentConf *Config

	// eventCh is used for Serf to deliver events on
	eventCh chan serf.Event

	// eventHandlers is the registered handlers for events
	eventHandlers     map[EventHandler]struct{}
	eventHandlersLock sync.Mutex

	// logger instance wraps the logOutput
	logger *log.Logger

	// This is the underlying Serf we are wrapping
	serf *serf.Serf

	// shutdownCh is used for shutdowns
	shutdown     bool
	shutdownCh   chan struct{}
	shutdownLock sync.Mutex
}

// Start creates a new agent, potentially returning an error
func Create(agentConf *Config, conf *serf.Config, logOutput io.Writer) (*Agent, error) {
	// Ensure we have a log sink
	if logOutput == nil {
		logOutput = os.Stderr
	}

	// Setup the underlying loggers
	conf.MemberlistConfig.LogOutput = logOutput
	conf.LogOutput = logOutput

	// Create a channel to listen for events from Serf
	eventCh := make(chan serf.Event, 64)
	conf.EventCh = eventCh

	// Setup the agent
	agent := &Agent{
		conf:          conf,
		agentConf:     agentConf,
		eventCh:       eventCh,
		eventHandlers: make(map[EventHandler]struct{}),
		logger:        log.New(logOutput, "", log.LstdFlags),
		shutdownCh:    make(chan struct{}),
	}

	// Restore agent tags from a tags file
	if agentConf.TagsFile != "" {
		if  err := agent.loadTagsFile(agentConf.TagsFile); err != nil {
			return nil, err
		}
	}

	return agent, nil
}

// Start is used to initiate the event listeners. It is seperate from
// create so that there isn't a race condition between creating the
// agent and registering handlers
func (a *Agent) Start() error {
	a.logger.Printf("[INFO] agent: Serf agent starting")

	// Create serf first
	serf, err := serf.Create(a.conf)
	if err != nil {
		return fmt.Errorf("Error creating Serf: %s", err)
	}
	a.serf = serf

	// Start event loop
	go a.eventLoop()
	return nil
}

// Leave prepares for a graceful shutdown of the agent and its processes
func (a *Agent) Leave() error {
	if a.serf == nil {
		return nil
	}

	a.logger.Println("[INFO] agent: requesting graceful leave from Serf")
	return a.serf.Leave()
}

// Shutdown closes this agent and all of its processes. Should be preceeded
// by a Leave for a graceful shutdown.
func (a *Agent) Shutdown() error {
	a.shutdownLock.Lock()
	defer a.shutdownLock.Unlock()

	if a.shutdown {
		return nil
	}

	if a.serf == nil {
		goto EXIT
	}

	a.logger.Println("[INFO] agent: requesting serf shutdown")
	if err := a.serf.Shutdown(); err != nil {
		return err
	}

EXIT:
	a.logger.Println("[INFO] agent: shutdown complete")
	a.shutdown = true
	close(a.shutdownCh)
	return nil
}

// ShutdownCh returns a channel that can be selected to wait
// for the agent to perform a shutdown.
func (a *Agent) ShutdownCh() <-chan struct{} {
	return a.shutdownCh
}

// Returns the Serf agent of the running Agent.
func (a *Agent) Serf() *serf.Serf {
	return a.serf
}

// Returns the Serf config of the running Agent.
func (a *Agent) SerfConfig() *serf.Config {
	return a.conf
}

// Join asks the Serf instance to join. See the Serf.Join function.
func (a *Agent) Join(addrs []string, replay bool) (n int, err error) {
	a.logger.Printf("[INFO] agent: joining: %v replay: %v", addrs, replay)
	ignoreOld := !replay
	n, err = a.serf.Join(addrs, ignoreOld)
	a.logger.Printf("[INFO] agent: joined: %d Err: %v", n, err)
	return
}

// ForceLeave is used to eject a failed node from the cluster
func (a *Agent) ForceLeave(node string) error {
	a.logger.Printf("[INFO] agent: Force leaving node: %s", node)
	err := a.serf.RemoveFailedNode(node)
	if err != nil {
		a.logger.Printf("[WARN] agent: failed to remove node: %v", err)
	}
	return err
}

// UserEvent sends a UserEvent on Serf, see Serf.UserEvent.
func (a *Agent) UserEvent(name string, payload []byte, coalesce bool) error {
	a.logger.Printf("[DEBUG] agent: Requesting user event send: %s. Coalesced: %#v. Payload: %#v",
		name, coalesce, string(payload))
	return a.serf.UserEvent(name, payload, coalesce)
}

// Query sends a Query on Serf, see Serf.Query.
func (a *Agent) Query(name string, payload []byte, params *serf.QueryParam) (*serf.QueryResponse, error) {
	// Prevent the use of the internal prefix
	if strings.HasPrefix(name, serf.InternalQueryPrefix) {
		// Allow the special "ping" query
		if name != serf.InternalQueryPrefix+"ping" || payload != nil {
			return nil, fmt.Errorf("Queries cannot contain the '%s' prefix", serf.InternalQueryPrefix)
		}
	}
	a.logger.Printf("[DEBUG] agent: Requesting query send: %s. Payload: %#v",
		name, string(payload))
	return a.serf.Query(name, payload, params)
}

// RegisterEventHandler adds an event handler to recieve event notifications
func (a *Agent) RegisterEventHandler(eh EventHandler) {
	a.eventHandlersLock.Lock()
	defer a.eventHandlersLock.Unlock()
	a.eventHandlers[eh] = struct{}{}
}

// DeregisterEventHandler removes an EventHandler and prevents more invocations
func (a *Agent) DeregisterEventHandler(eh EventHandler) {
	a.eventHandlersLock.Lock()
	defer a.eventHandlersLock.Unlock()
	delete(a.eventHandlers, eh)
}

// eventLoop listens to events from Serf and fans out to event handlers
func (a *Agent) eventLoop() {
	serfShutdownCh := a.serf.ShutdownCh()
	for {
		select {
		case e := <-a.eventCh:
			a.logger.Printf("[INFO] agent: Received event: %s", e.String())
			a.eventHandlersLock.Lock()
			for eh, _ := range a.eventHandlers {
				eh.HandleEvent(e)
			}
			a.eventHandlersLock.Unlock()

		case <-serfShutdownCh:
			a.logger.Printf("[WARN] agent: Serf shutdown detected, quitting")
			a.Shutdown()
			return

		case <-a.shutdownCh:
			return
		}
	}
}

// SetTags is used to update the tags. The agent will make sure to
// persist tags if necessary before gossiping to the cluster.
func (a *Agent) SetTags(tags map[string]string) error {
	// Update the tags file if we have one
	if a.agentConf.TagsFile != "" {
		if err := a.writeTagsFile(tags); err != nil {
			a.logger.Printf("[ERR] agent: %s", err)
			return err
		}
	}

	// Set the tags in Serf, start gossiping out
	return a.serf.SetTags(tags)
}

// loadTagsFile will load agent tags out of a file and set them in the
// current serf configuration.
func (a *Agent) loadTagsFile(tagsFile string) error {
	// Avoid passing tags and using a tags file at the same time
	if len(a.agentConf.Tags) > 0 {
		return fmt.Errorf("Tags config not allowed while using tag files")
	}

	if _, err := os.Stat(tagsFile); err == nil {
		tagData, err := ioutil.ReadFile(tagsFile)
		if err != nil {
			return fmt.Errorf("Failed to read tags file: %s", err)
		}
		if err := json.Unmarshal(tagData, &a.conf.Tags); err != nil {
			return fmt.Errorf("Failed to decode tags file: %s", err)
		}
		a.logger.Printf("[INFO] agent: Restored %d tag(s) from %s",
			len(a.conf.Tags), tagsFile)
	}

	// Success!
	return nil
}

// writeTagsFile will write the current tags to the configured tags file.
func (a *Agent) writeTagsFile(tags map[string]string) error {
	encoded, err := json.MarshalIndent(tags, "", "  ")
	if err != nil {
		return fmt.Errorf("Failed to encode tags: %s", err)
	}

	// Use 0600 for permissions, in case tag data is sensitive
	if err = ioutil.WriteFile(a.agentConf.TagsFile, encoded, 0600); err != nil {
		return fmt.Errorf("Failed to write tags file: %s", err)
	}

	// Success!
	return nil
}
<commit_msg>agent: Clarify some log messages. Fixes #183.<commit_after>package agent

import (
	"encoding/json"
	"fmt"
	"github.com/hashicorp/serf/serf"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"
)

// Agent starts and manages a Serf instance, adding some niceties
// on top of Serf such as storing logs that you can later retrieve,
// and invoking EventHandlers when events occur.
type Agent struct {
	// Stores the serf configuration
	conf *serf.Config

	// Stores the agent configuration
	agentConf *Config

	// eventCh is used for Serf to deliver events on
	eventCh chan serf.Event

	// eventHandlers is the registered handlers for events
	eventHandlers     map[EventHandler]struct{}
	eventHandlersLock sync.Mutex

	// logger instance wraps the logOutput
	logger *log.Logger

	// This is the underlying Serf we are wrapping
	serf *serf.Serf

	// shutdownCh is used for shutdowns
	shutdown     bool
	shutdownCh   chan struct{}
	shutdownLock sync.Mutex
}

// Start creates a new agent, potentially returning an error
func Create(agentConf *Config, conf *serf.Config, logOutput io.Writer) (*Agent, error) {
	// Ensure we have a log sink
	if logOutput == nil {
		logOutput = os.Stderr
	}

	// Setup the underlying loggers
	conf.MemberlistConfig.LogOutput = logOutput
	conf.LogOutput = logOutput

	// Create a channel to listen for events from Serf
	eventCh := make(chan serf.Event, 64)
	conf.EventCh = eventCh

	// Setup the agent
	agent := &Agent{
		conf:          conf,
		agentConf:     agentConf,
		eventCh:       eventCh,
		eventHandlers: make(map[EventHandler]struct{}),
		logger:        log.New(logOutput, "", log.LstdFlags),
		shutdownCh:    make(chan struct{}),
	}

	// Restore agent tags from a tags file
	if agentConf.TagsFile != "" {
		if err := agent.loadTagsFile(agentConf.TagsFile); err != nil {
			return nil, err
		}
	}

	return agent, nil
}

// Start is used to initiate the event listeners. It is seperate from
// create so that there isn't a race condition between creating the
// agent and registering handlers
func (a *Agent) Start() error {
	a.logger.Printf("[INFO] agent: Serf agent starting")

	// Create serf first
	serf, err := serf.Create(a.conf)
	if err != nil {
		return fmt.Errorf("Error creating Serf: %s", err)
	}
	a.serf = serf

	// Start event loop
	go a.eventLoop()
	return nil
}

// Leave prepares for a graceful shutdown of the agent and its processes
func (a *Agent) Leave() error {
	if a.serf == nil {
		return nil
	}

	a.logger.Println("[INFO] agent: requesting graceful leave from Serf")
	return a.serf.Leave()
}

// Shutdown closes this agent and all of its processes. Should be preceeded
// by a Leave for a graceful shutdown.
func (a *Agent) Shutdown() error {
	a.shutdownLock.Lock()
	defer a.shutdownLock.Unlock()

	if a.shutdown {
		return nil
	}

	if a.serf == nil {
		goto EXIT
	}

	a.logger.Println("[INFO] agent: requesting serf shutdown")
	if err := a.serf.Shutdown(); err != nil {
		return err
	}

EXIT:
	a.logger.Println("[INFO] agent: shutdown complete")
	a.shutdown = true
	close(a.shutdownCh)
	return nil
}

// ShutdownCh returns a channel that can be selected to wait
// for the agent to perform a shutdown.
func (a *Agent) ShutdownCh() <-chan struct{} {
	return a.shutdownCh
}

// Returns the Serf agent of the running Agent.
func (a *Agent) Serf() *serf.Serf {
	return a.serf
}

// Returns the Serf config of the running Agent.
func (a *Agent) SerfConfig() *serf.Config {
	return a.conf
}

// Join asks the Serf instance to join. See the Serf.Join function.
func (a *Agent) Join(addrs []string, replay bool) (n int, err error) {
	a.logger.Printf("[INFO] agent: joining: %v replay: %v", addrs, replay)
	ignoreOld := !replay
	n, err = a.serf.Join(addrs, ignoreOld)
	if n > 0 {
		a.logger.Printf("[INFO] agent: joined: %d nodes", n)
	}
	if err != nil {
		a.logger.Printf("[INFO] agent: error joining: %v", err)
	}
	return
}

// ForceLeave is used to eject a failed node from the cluster
func (a *Agent) ForceLeave(node string) error {
	a.logger.Printf("[INFO] agent: Force leaving node: %s", node)
	err := a.serf.RemoveFailedNode(node)
	if err != nil {
		a.logger.Printf("[WARN] agent: failed to remove node: %v", err)
	}
	return err
}

// UserEvent sends a UserEvent on Serf, see Serf.UserEvent.
func (a *Agent) UserEvent(name string, payload []byte, coalesce bool) error {
	a.logger.Printf("[DEBUG] agent: Requesting user event send: %s. Coalesced: %#v. Payload: %#v",
		name, coalesce, string(payload))
	err := a.serf.UserEvent(name, payload, coalesce)
	if err != nil {
		a.logger.Printf("[WARN] agent: failed to send user event: %v", err)
	}
	return err
}

// Query sends a Query on Serf, see Serf.Query.
func (a *Agent) Query(name string, payload []byte, params *serf.QueryParam) (*serf.QueryResponse, error) {
	// Prevent the use of the internal prefix
	if strings.HasPrefix(name, serf.InternalQueryPrefix) {
		// Allow the special "ping" query
		if name != serf.InternalQueryPrefix+"ping" || payload != nil {
			return nil, fmt.Errorf("Queries cannot contain the '%s' prefix", serf.InternalQueryPrefix)
		}
	}
	a.logger.Printf("[DEBUG] agent: Requesting query send: %s. Payload: %#v",
		name, string(payload))
	resp, err := a.serf.Query(name, payload, params)
	if err != nil {
		a.logger.Printf("[WARN] agent: failed to start user query: %v", err)
	}
	return resp, err
}

// RegisterEventHandler adds an event handler to recieve event notifications
func (a *Agent) RegisterEventHandler(eh EventHandler) {
	a.eventHandlersLock.Lock()
	defer a.eventHandlersLock.Unlock()
	a.eventHandlers[eh] = struct{}{}
}

// DeregisterEventHandler removes an EventHandler and prevents more invocations
func (a *Agent) DeregisterEventHandler(eh EventHandler) {
	a.eventHandlersLock.Lock()
	defer a.eventHandlersLock.Unlock()
	delete(a.eventHandlers, eh)
}

// eventLoop listens to events from Serf and fans out to event handlers
func (a *Agent) eventLoop() {
	serfShutdownCh := a.serf.ShutdownCh()
	for {
		select {
		case e := <-a.eventCh:
			a.logger.Printf("[INFO] agent: Received event: %s", e.String())
			a.eventHandlersLock.Lock()
			for eh, _ := range a.eventHandlers {
				eh.HandleEvent(e)
			}
			a.eventHandlersLock.Unlock()

		case <-serfShutdownCh:
			a.logger.Printf("[WARN] agent: Serf shutdown detected, quitting")
			a.Shutdown()
			return

		case <-a.shutdownCh:
			return
		}
	}
}

// SetTags is used to update the tags. The agent will make sure to
// persist tags if necessary before gossiping to the cluster.
func (a *Agent) SetTags(tags map[string]string) error {
	// Update the tags file if we have one
	if a.agentConf.TagsFile != "" {
		if err := a.writeTagsFile(tags); err != nil {
			a.logger.Printf("[ERR] agent: %s", err)
			return err
		}
	}

	// Set the tags in Serf, start gossiping out
	return a.serf.SetTags(tags)
}

// loadTagsFile will load agent tags out of a file and set them in the
// current serf configuration.
func (a *Agent) loadTagsFile(tagsFile string) error {
	// Avoid passing tags and using a tags file at the same time
	if len(a.agentConf.Tags) > 0 {
		return fmt.Errorf("Tags config not allowed while using tag files")
	}

	if _, err := os.Stat(tagsFile); err == nil {
		tagData, err := ioutil.ReadFile(tagsFile)
		if err != nil {
			return fmt.Errorf("Failed to read tags file: %s", err)
		}
		if err := json.Unmarshal(tagData, &a.conf.Tags); err != nil {
			return fmt.Errorf("Failed to decode tags file: %s", err)
		}
		a.logger.Printf("[INFO] agent: Restored %d tag(s) from %s",
			len(a.conf.Tags), tagsFile)
	}

	// Success!
	return nil
}

// writeTagsFile will write the current tags to the configured tags file.
func (a *Agent) writeTagsFile(tags map[string]string) error {
	encoded, err := json.MarshalIndent(tags, "", "  ")
	if err != nil {
		return fmt.Errorf("Failed to encode tags: %s", err)
	}

	// Use 0600 for permissions, in case tag data is sensitive
	if err = ioutil.WriteFile(a.agentConf.TagsFile, encoded, 0600); err != nil {
		return fmt.Errorf("Failed to write tags file: %s", err)
	}

	// Success!
	return nil
}
<|endoftext|>


<commit_before>package command

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"text/template"
)

//DataFormatter is a transformer of the data.
type DataFormatter interface {
	// TransformData should return transformed string data.
	TransformData(interface{}) (string, error)
}

// DataFormat returns the data formatter specified format.
func DataFormat(format, tmpl string) (DataFormatter, error) {
	switch format {
	case "json":
		return &JSONFormat{}, nil
	case "template":
		return &TemplateFormat{tmpl}, nil
	}
	return nil, fmt.Errorf("Unsupported format is specified.")
}

type JSONFormat struct {
}

// TransformData returns JSON format string data.
func (p *JSONFormat) TransformData(data interface{}) (string, error) {
	out, err := json.MarshalIndent(&data, "", "    ")
	if err != nil {
		return "", err
	}

	return string(out), nil
}

type TemplateFormat struct {
	tmpl string
}

// TransformData returns template format string data.
func (p *TemplateFormat) TransformData(data interface{}) (string, error) {
	var out io.Writer = new(bytes.Buffer)
	if len(p.tmpl) == 0 {
		return "", fmt.Errorf("template needs to be specified the golang templates.")
	}

	t := template.Must(template.New("format").Parse(p.tmpl))
	err := t.Execute(out, data)
	if err != nil {
		return "", err
	}
	return fmt.Sprint(out), nil
}
<commit_msg>fix go panic<commit_after>package command

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"text/template"
)

//DataFormatter is a transformer of the data.
type DataFormatter interface {
	// TransformData should return transformed string data.
	TransformData(interface{}) (string, error)
}

// DataFormat returns the data formatter specified format.
func DataFormat(format, tmpl string) (DataFormatter, error) {
	switch format {
	case "json":
		return &JSONFormat{}, nil
	case "template":
		return &TemplateFormat{tmpl}, nil
	}
	return nil, fmt.Errorf("Unsupported format is specified.")
}

type JSONFormat struct {
}

// TransformData returns JSON format string data.
func (p *JSONFormat) TransformData(data interface{}) (string, error) {
	out, err := json.MarshalIndent(&data, "", "    ")
	if err != nil {
		return "", err
	}

	return string(out), nil
}

type TemplateFormat struct {
	tmpl string
}

// TransformData returns template format string data.
func (p *TemplateFormat) TransformData(data interface{}) (string, error) {
	var out io.Writer = new(bytes.Buffer)
	if len(p.tmpl) == 0 {
		return "", fmt.Errorf("template needs to be specified the golang templates.")
	}

	t, err := template.New("format").Parse(p.tmpl)
	if err != nil {
		return "", err
	}

	err = t.Execute(out, data)
	if err != nil {
		return "", err
	}
	return fmt.Sprint(out), nil
}
<|endoftext|>


<commit_before>package file

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"syscall"

	"path"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/term"
)

var (
	re           = regexp.MustCompile("[^a-zA-Z0-9]")
	ErrSkipBuild = errors.New("skip build")
)

type Dapperfile struct {
	File   string
	Mode   string
	docker string
	env    Context
	Socket bool
	NoOut  bool
	Args   []string
	From   string
}

func Lookup(file string) (*Dapperfile, error) {
	if _, err := os.Stat(file); err != nil {
		return nil, err
	}

	d := &Dapperfile{
		File: file,
	}

	return d, d.init()
}

func (d *Dapperfile) init() error {
	docker, err := exec.LookPath("docker")
	if err != nil {
		return err
	}
	d.docker = docker
	if d.Args, err = d.argsFromEnv(d.File); err != nil {
		return err
	}
	return nil
}

func (d *Dapperfile) argsFromEnv(dockerfile string) ([]string, error) {
	file, err := os.Open(dockerfile)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	r := []string{}
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		fields := strings.Fields(line)
		if len(fields) <= 1 {
			continue
		}

		command := fields[0]
		if command != "ARG" {
			continue
		}

		key := strings.Split(fields[1], "=")[0]
		value := os.Getenv(key)

		if key == "DAPPER_HOST_ARCH" && value == "" {
			value = d.hostArch()
		}

		if value != "" {
			r = append(r, fmt.Sprint("%s=%s", key, value))
		}
	}

	return r, nil
}

func (d *Dapperfile) Run(commandArgs []string) error {
	tag, err := d.build()
	if err != nil {
		return err
	}

	logrus.Debugf("Running build in %s", tag)
	name, args := d.runArgs(tag, "", commandArgs)
	defer func() {
		logrus.Debugf("Deleting temp container %s", name)
		d.execWithOutput("rm", "-fv", name)
	}()

	if err := d.run(args...); err != nil {
		return err
	}

	source := d.env.Source()
	output := d.env.Output()
	if !d.IsBind() && !d.NoOut {
		for _, i := range output {
			p := i
			if !strings.HasPrefix(p, "/") {
				p = path.Join(source, i)
			}
			targetDir := path.Dir(i)
			if err := os.MkdirAll(targetDir, 0755); err != nil {
				return err
			}
			logrus.Infof("docker cp %s %s", p, targetDir)
			if err := d.exec("cp", name+":"+p, targetDir); err != nil {
				logrus.Debugf("Error copying back '%s': %s", i, err)
			}
		}
	}

	return nil
}

func (d *Dapperfile) Shell(commandArgs []string) error {
	tag, err := d.build()
	if err != nil {
		return err
	}

	logrus.Debugf("Running shell in %s", tag)
	_, args := d.runArgs(tag, d.env.Shell(), nil)
	args = append([]string{"--rm"}, args...)

	return d.runExec(args...)
}

func (d *Dapperfile) runArgs(tag, shell string, commandArgs []string) (string, []string) {
	name := fmt.Sprintf("%s-%s", strings.Split(tag, ":")[0], randString())

	args := []string{"-i", "--name", name}

	if term.IsTerminal(0) {
		args = append(args, "-t")
	}

	if d.env.Socket() || d.Socket {
		args = append(args, "-v", "/var/run/docker.sock:/var/run/docker.sock")
	}

	if d.IsBind() {
		wd, err := os.Getwd()
		if err == nil {
			args = append(args, "-v", fmt.Sprintf("%s:%s", fmt.Sprintf("%s/%s", wd, d.env.Cp()), d.env.Source()))
		}
	}

	args = append(args, "-e", fmt.Sprintf("DAPPER_UID=%d", os.Getuid()))
	args = append(args, "-e", fmt.Sprintf("DAPPER_GID=%d", os.Getgid()))

	for _, env := range d.env.Env() {
		args = append(args, "-e", env)
	}

	if shell != "" {
		args = append(args, "--entrypoint", shell)
		args = append(args, "-e", "TERM")
	}

	args = append(args, d.env.RunArgs()...)
	args = append(args, tag)

	if shell != "" && len(commandArgs) == 0 {
		args = append(args, "-")
	} else {
		args = append(args, commandArgs...)
	}

	return name, args
}

func (d *Dapperfile) prebuild() error {
	f, err := os.Open(d.File)
	if err != nil {
		return err
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	target := ""

	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "FROM ") {
			parts := strings.Fields(line)
			if len(parts) <= 1 {
				return nil
			}

			target = strings.TrimSpace(parts[1])
			continue
		} else if target == "" || !strings.HasPrefix(line, "# FROM ") {
			continue
		}

		baseImage, ok := toMap(line)[d.hostArch()]
		if !ok {
			return nil
		}

		if baseImage == "skip" {
			return ErrSkipBuild
		}

		_, err := exec.Command(d.docker, "inspect", baseImage).CombinedOutput()
		if err != nil {
			if err := d.exec("pull", baseImage); err != nil {
				return err
			}
		}

		return d.exec("tag", baseImage, target)
	}

	return scanner.Err()
}

func (d *Dapperfile) hostArch() string {
	return runtime.GOARCH
}

func (d *Dapperfile) Build(args []string) error {
	if err := d.prebuild(); err != nil {
		return err
	}

	buildArgs := []string{"build", "-f", d.File}
	for _, v := range d.Args {
		buildArgs = append(buildArgs, "--build-arg", v)
	}
	buildArgs = append(buildArgs, args...)

	return d.exec(buildArgs...)
}

func (d *Dapperfile) build() (string, error) {
	if err := d.prebuild(); err != nil {
		return "", err
	}

	tag := d.tag()
	logrus.Debugf("Building %s using %s", tag, d.File)
	buildArgs := []string{"build", "-t", tag, "-f", d.File}
	for _, v := range d.Args {
		buildArgs = append(buildArgs, "--build-arg", v)
	}
	buildArgs = append(buildArgs, ".")
	if err := d.exec(buildArgs...); err != nil {
		return "", err
	}

	if err := d.readEnv(tag); err != nil {
		return "", err
	}

	if !d.IsBind() {
		text := fmt.Sprintf("FROM %s\nCOPY %s %s", tag, d.env.Cp(), d.env.Source())
		if err := d.buildWithContent(tag, text); err != nil {
			return "", err
		}
	}

	return tag, nil
}

func (d *Dapperfile) buildWithContent(tag, content string) error {
	tempfile, err := ioutil.TempFile(".", d.File)
	if err != nil {
		return err
	}

	logrus.Debugf("Created tempfile %s", tempfile.Name())
	defer func() {
		logrus.Debugf("Deleting tempfile %s", tempfile.Name())
		if err := os.Remove(tempfile.Name()); err != nil {
			logrus.Errorf("Failed to delete tempfile %s: %v", tempfile.Name(), err)
		}
	}()

	ioutil.WriteFile(tempfile.Name(), []byte(content), 0600)

	return d.exec("build", "-t", tag, "-f", tempfile.Name(), ".")
}

func (d *Dapperfile) readEnv(tag string) error {
	var envList []string

	args := []string{"inspect", "-f", "{{json .ContainerConfig.Env}}", tag}

	cmd := exec.Command(d.docker, args...)
	output, err := cmd.CombinedOutput()
	if err != nil {
		logrus.Errorf("Failed to run docker %v: %v", args, err)
		return err
	}

	if err := json.Unmarshal(output, &envList); err != nil {
		return err
	}

	d.env = map[string]string{}

	for _, item := range envList {
		parts := strings.SplitN(item, "=", 2)
		k, v := parts[0], parts[1]
		logrus.Debugf("Reading Env: %s=%s", k, v)
		d.env[k] = v
	}

	logrus.Debugf("Source: %s", d.env.Source())
	logrus.Debugf("Cp: %s", d.env.Cp())
	logrus.Debugf("Socket: %t", d.env.Socket())
	logrus.Debugf("Mode: %s", d.env.Mode(d.Mode))
	logrus.Debugf("Env: %v", d.env.Env())
	logrus.Debugf("Output: %v", d.env.Output())

	return nil
}

func (d *Dapperfile) tag() string {
	cwd, err := os.Getwd()
	if err == nil {
		cwd = filepath.Base(cwd)
	} else {
		cwd = "dapper-unknown"
	}

	output, _ := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").Output()
	tag := strings.TrimSpace(string(output))
	if tag == "" {
		tag = randString()
	}
	tag = re.ReplaceAllLiteralString(tag, "-")

	return fmt.Sprintf("%s:%s", cwd, tag)
}

func (d *Dapperfile) run(args ...string) error {
	return d.exec(append([]string{"run"}, args...)...)
}

func (d *Dapperfile) exec(args ...string) error {
	logrus.Debugf("Running %s %v", d.docker, args)
	cmd := exec.Command(d.docker, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	err := cmd.Run()
	if err != nil {
		logrus.Debugf("Failed running %s %v: %v", d.docker, args, err)
	}
	return err
}

func (d *Dapperfile) runExec(args ...string) error {
	logrus.Debugf("Exec %s run %v", d.docker, args)
	return syscall.Exec(d.docker, append([]string{"docker", "run"}, args...), os.Environ())
}

func (d *Dapperfile) execWithOutput(args ...string) ([]byte, error) {
	cmd := exec.Command(d.docker, args...)
	return cmd.CombinedOutput()
}

func (d *Dapperfile) IsBind() bool {
	return d.env.Mode(d.Mode) == "bind"
}
<commit_msg>Fix broken args<commit_after>package file

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"syscall"

	"path"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/term"
)

var (
	re           = regexp.MustCompile("[^a-zA-Z0-9]")
	ErrSkipBuild = errors.New("skip build")
)

type Dapperfile struct {
	File   string
	Mode   string
	docker string
	env    Context
	Socket bool
	NoOut  bool
	Args   []string
	From   string
}

func Lookup(file string) (*Dapperfile, error) {
	if _, err := os.Stat(file); err != nil {
		return nil, err
	}

	d := &Dapperfile{
		File: file,
	}

	return d, d.init()
}

func (d *Dapperfile) init() error {
	docker, err := exec.LookPath("docker")
	if err != nil {
		return err
	}
	d.docker = docker
	if d.Args, err = d.argsFromEnv(d.File); err != nil {
		return err
	}
	return nil
}

func (d *Dapperfile) argsFromEnv(dockerfile string) ([]string, error) {
	file, err := os.Open(dockerfile)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	r := []string{}
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		fields := strings.Fields(line)
		if len(fields) <= 1 {
			continue
		}

		command := fields[0]
		if command != "ARG" {
			continue
		}

		key := strings.Split(fields[1], "=")[0]
		value := os.Getenv(key)

		if key == "DAPPER_HOST_ARCH" && value == "" {
			value = d.hostArch()
		}

		if value != "" {
			r = append(r, fmt.Sprintf("%s=%s", key, value))
		}
	}

	return r, nil
}

func (d *Dapperfile) Run(commandArgs []string) error {
	tag, err := d.build()
	if err != nil {
		return err
	}

	logrus.Debugf("Running build in %s", tag)
	name, args := d.runArgs(tag, "", commandArgs)
	defer func() {
		logrus.Debugf("Deleting temp container %s", name)
		d.execWithOutput("rm", "-fv", name)
	}()

	if err := d.run(args...); err != nil {
		return err
	}

	source := d.env.Source()
	output := d.env.Output()
	if !d.IsBind() && !d.NoOut {
		for _, i := range output {
			p := i
			if !strings.HasPrefix(p, "/") {
				p = path.Join(source, i)
			}
			targetDir := path.Dir(i)
			if err := os.MkdirAll(targetDir, 0755); err != nil {
				return err
			}
			logrus.Infof("docker cp %s %s", p, targetDir)
			if err := d.exec("cp", name+":"+p, targetDir); err != nil {
				logrus.Debugf("Error copying back '%s': %s", i, err)
			}
		}
	}

	return nil
}

func (d *Dapperfile) Shell(commandArgs []string) error {
	tag, err := d.build()
	if err != nil {
		return err
	}

	logrus.Debugf("Running shell in %s", tag)
	_, args := d.runArgs(tag, d.env.Shell(), nil)
	args = append([]string{"--rm"}, args...)

	return d.runExec(args...)
}

func (d *Dapperfile) runArgs(tag, shell string, commandArgs []string) (string, []string) {
	name := fmt.Sprintf("%s-%s", strings.Split(tag, ":")[0], randString())

	args := []string{"-i", "--name", name}

	if term.IsTerminal(0) {
		args = append(args, "-t")
	}

	if d.env.Socket() || d.Socket {
		args = append(args, "-v", "/var/run/docker.sock:/var/run/docker.sock")
	}

	if d.IsBind() {
		wd, err := os.Getwd()
		if err == nil {
			args = append(args, "-v", fmt.Sprintf("%s:%s", fmt.Sprintf("%s/%s", wd, d.env.Cp()), d.env.Source()))
		}
	}

	args = append(args, "-e", fmt.Sprintf("DAPPER_UID=%d", os.Getuid()))
	args = append(args, "-e", fmt.Sprintf("DAPPER_GID=%d", os.Getgid()))

	for _, env := range d.env.Env() {
		args = append(args, "-e", env)
	}

	if shell != "" {
		args = append(args, "--entrypoint", shell)
		args = append(args, "-e", "TERM")
	}

	args = append(args, d.env.RunArgs()...)
	args = append(args, tag)

	if shell != "" && len(commandArgs) == 0 {
		args = append(args, "-")
	} else {
		args = append(args, commandArgs...)
	}

	return name, args
}

func (d *Dapperfile) prebuild() error {
	f, err := os.Open(d.File)
	if err != nil {
		return err
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	target := ""

	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "FROM ") {
			parts := strings.Fields(line)
			if len(parts) <= 1 {
				return nil
			}

			target = strings.TrimSpace(parts[1])
			continue
		} else if target == "" || !strings.HasPrefix(line, "# FROM ") {
			continue
		}

		baseImage, ok := toMap(line)[d.hostArch()]
		if !ok {
			return nil
		}

		if baseImage == "skip" {
			return ErrSkipBuild
		}

		_, err := exec.Command(d.docker, "inspect", baseImage).CombinedOutput()
		if err != nil {
			if err := d.exec("pull", baseImage); err != nil {
				return err
			}
		}

		return d.exec("tag", baseImage, target)
	}

	return scanner.Err()
}

func (d *Dapperfile) hostArch() string {
	return runtime.GOARCH
}

func (d *Dapperfile) Build(args []string) error {
	if err := d.prebuild(); err != nil {
		return err
	}

	buildArgs := []string{"build", "-f", d.File}
	for _, v := range d.Args {
		buildArgs = append(buildArgs, "--build-arg", v)
	}
	buildArgs = append(buildArgs, args...)

	return d.exec(buildArgs...)
}

func (d *Dapperfile) build() (string, error) {
	if err := d.prebuild(); err != nil {
		return "", err
	}

	tag := d.tag()
	logrus.Debugf("Building %s using %s", tag, d.File)
	buildArgs := []string{"build", "-t", tag, "-f", d.File}
	for _, v := range d.Args {
		buildArgs = append(buildArgs, "--build-arg", v)
	}
	buildArgs = append(buildArgs, ".")
	if err := d.exec(buildArgs...); err != nil {
		return "", err
	}

	if err := d.readEnv(tag); err != nil {
		return "", err
	}

	if !d.IsBind() {
		text := fmt.Sprintf("FROM %s\nCOPY %s %s", tag, d.env.Cp(), d.env.Source())
		if err := d.buildWithContent(tag, text); err != nil {
			return "", err
		}
	}

	return tag, nil
}

func (d *Dapperfile) buildWithContent(tag, content string) error {
	tempfile, err := ioutil.TempFile(".", d.File)
	if err != nil {
		return err
	}

	logrus.Debugf("Created tempfile %s", tempfile.Name())
	defer func() {
		logrus.Debugf("Deleting tempfile %s", tempfile.Name())
		if err := os.Remove(tempfile.Name()); err != nil {
			logrus.Errorf("Failed to delete tempfile %s: %v", tempfile.Name(), err)
		}
	}()

	ioutil.WriteFile(tempfile.Name(), []byte(content), 0600)

	return d.exec("build", "-t", tag, "-f", tempfile.Name(), ".")
}

func (d *Dapperfile) readEnv(tag string) error {
	var envList []string

	args := []string{"inspect", "-f", "{{json .ContainerConfig.Env}}", tag}

	cmd := exec.Command(d.docker, args...)
	output, err := cmd.CombinedOutput()
	if err != nil {
		logrus.Errorf("Failed to run docker %v: %v", args, err)
		return err
	}

	if err := json.Unmarshal(output, &envList); err != nil {
		return err
	}

	d.env = map[string]string{}

	for _, item := range envList {
		parts := strings.SplitN(item, "=", 2)
		k, v := parts[0], parts[1]
		logrus.Debugf("Reading Env: %s=%s", k, v)
		d.env[k] = v
	}

	logrus.Debugf("Source: %s", d.env.Source())
	logrus.Debugf("Cp: %s", d.env.Cp())
	logrus.Debugf("Socket: %t", d.env.Socket())
	logrus.Debugf("Mode: %s", d.env.Mode(d.Mode))
	logrus.Debugf("Env: %v", d.env.Env())
	logrus.Debugf("Output: %v", d.env.Output())

	return nil
}

func (d *Dapperfile) tag() string {
	cwd, err := os.Getwd()
	if err == nil {
		cwd = filepath.Base(cwd)
	} else {
		cwd = "dapper-unknown"
	}

	output, _ := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").Output()
	tag := strings.TrimSpace(string(output))
	if tag == "" {
		tag = randString()
	}
	tag = re.ReplaceAllLiteralString(tag, "-")

	return fmt.Sprintf("%s:%s", cwd, tag)
}

func (d *Dapperfile) run(args ...string) error {
	return d.exec(append([]string{"run"}, args...)...)
}

func (d *Dapperfile) exec(args ...string) error {
	logrus.Debugf("Running %s %v", d.docker, args)
	cmd := exec.Command(d.docker, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	err := cmd.Run()
	if err != nil {
		logrus.Debugf("Failed running %s %v: %v", d.docker, args, err)
	}
	return err
}

func (d *Dapperfile) runExec(args ...string) error {
	logrus.Debugf("Exec %s run %v", d.docker, args)
	return syscall.Exec(d.docker, append([]string{"docker", "run"}, args...), os.Environ())
}

func (d *Dapperfile) execWithOutput(args ...string) ([]byte, error) {
	cmd := exec.Command(d.docker, args...)
	return cmd.CombinedOutput()
}

func (d *Dapperfile) IsBind() bool {
	return d.env.Mode(d.Mode) == "bind"
}
<|endoftext|>


<commit_before>package zip

import (
	"archive/zip"
	"errors"
	"fmt"
	"github.com/Sirupsen/logrus"
	"io"
	"os"
	"path/filepath"
	"strings"
)

// AddToZip creates a source object (either a file, or a directory that will be recursively
// added) to a previously opened zip.Writer.  The archive path of `source` is relative to the
// `rootSource` parameter.
func AddToZip(zipWriter *zip.Writer, source string, rootSource string, logger *logrus.Logger) error {
	fullPathSource, err := filepath.Abs(source)
	if nil != err {
		return err
	}
	appendFile := func(sourceFile string) error {
		// Get the relative path
		var name = filepath.Base(sourceFile)
		if sourceFile != rootSource {
			name = strings.TrimPrefix(strings.TrimPrefix(sourceFile, rootSource), string(os.PathSeparator))

			// Normalize the name s.t. path delimiters are AWS Linux friendly when
			// unpacking the archive.
			name = strings.Replace(name, "\\", "/", -1)
		}
		binaryWriter, errCreate := zipWriter.Create(name)
		if errCreate != nil {
			return fmt.Errorf("Failed to create ZIP entry: %s", filepath.Base(sourceFile))
		}
		reader, errOpen := os.Open(sourceFile)
		if errOpen != nil {
			return fmt.Errorf("Failed to open file: %s", sourceFile)
		}
		defer reader.Close()
		io.Copy(binaryWriter, reader)
		logger.WithFields(logrus.Fields{
			"Path": sourceFile,
		}).Debug("Archiving file")

		return nil
	}

	directoryWalker := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		header, err := zip.FileInfoHeader(info)
		if err != nil {
			return err
		}
		header.Name = strings.TrimPrefix(strings.TrimPrefix(path, rootSource), string(os.PathSeparator))
		if info.IsDir() {
			header.Name += "/"
		} else {
			header.Method = zip.Deflate
		}
		writer, err := zipWriter.CreateHeader(header)
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()
		_, err = io.Copy(writer, file)
		return err
	}

	fileInfo, err := os.Stat(fullPathSource)
	if nil != err {
		return err
	}
	switch mode := fileInfo.Mode(); {
	case mode.IsDir():
		err = filepath.Walk(fullPathSource, directoryWalker)
	case mode.IsRegular():
		err = appendFile(fullPathSource)
	default:
		err = errors.New("Inavlid source type")
	}
	zipWriter.Close()
	return err
}
<commit_msg>Minor refactor of file code path for AddToZip<commit_after>package zip

import (
	"archive/zip"
	"errors"
	"github.com/Sirupsen/logrus"
	"io"
	"os"
	"path"
	"path/filepath"
	"strings"
)

// AddToZip creates a source object (either a file, or a directory that will be recursively
// added) to a previously opened zip.Writer.  The archive path of `source` is relative to the
// `rootSource` parameter.
func AddToZip(zipWriter *zip.Writer, source string, rootSource string, logger *logrus.Logger) error {
	fullPathSource, err := filepath.Abs(source)
	if nil != err {
		return err
	}

	appendFile := func(info os.FileInfo) error {
		zipEntryName := source
		if "" != rootSource {
			zipEntryName = path.Join(strings.Replace(rootSource, "\\", "/", -1), info.Name())
		}
		// File info for the binary executable
		binaryWriter, binaryWriterErr := zipWriter.Create(zipEntryName)
		if binaryWriterErr != nil {
			return binaryWriterErr
		}
		reader, readerErr := os.Open(fullPathSource)
		if readerErr != nil {
			return readerErr
		}
		written, copyErr := io.Copy(binaryWriter, reader)
		reader.Close()

		logger.WithFields(logrus.Fields{
			"WrittenBytes": written,
			"SourcePath":   fullPathSource,
			"ZipName":      zipEntryName,
		}).Debug("Archiving file")
		return copyErr
	}

	directoryWalker := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		header, err := zip.FileInfoHeader(info)
		if err != nil {
			return err
		}
		header.Name = strings.TrimPrefix(strings.TrimPrefix(path, rootSource), string(os.PathSeparator))
		if info.IsDir() {
			header.Name += "/"
		} else {
			header.Method = zip.Deflate
		}
		writer, err := zipWriter.CreateHeader(header)
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()
		_, err = io.Copy(writer, file)
		return err
	}

	fileInfo, err := os.Stat(fullPathSource)
	if nil != err {
		return err
	}
	switch mode := fileInfo.Mode(); {
	case mode.IsDir():
		err = filepath.Walk(fullPathSource, directoryWalker)
	case mode.IsRegular():
		err = appendFile(fileInfo)
	default:
		err = errors.New("Inavlid source type")
	}
	zipWriter.Close()
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"os"

	"github.com/anacrolix/torrent"
)

// SeekableContent describes an io.ReadSeeker that can be closed as well.
type SeekableContent interface {
	io.ReadSeeker
	io.Closer
}

// FileEntry helps reading a torrent file.
type FileEntry struct {
	File   *torrent.File
	Reader *torrent.Reader
}

func (f FileEntry) Read(p []byte) (n int, err error) {
	return f.Reader.Read(p)
}

func (f FileEntry) Seek(offset int64, whence int) (int64, error) {
	return f.Reader.Seek(offset+f.File.Offset(), whence)
}

func (f FileEntry) Close() error {
	return f.Reader.Close()
}

// NewFileReader sets up a torrent file for streaming reading.
func NewFileReader(f torrent.File) (SeekableContent, error) {
	// We read ahead 1% of the file continuously.
	var readahead = f.Length() / 100

	// We begin by prioritizing 5% of the beginning of the file.
	f.PrioritizeRegion(f.Offset(), readahead*5)

	reader := t.NewReader()
	reader.SetReadahead(readahead)
	reader.SetResponsive()
	_, err := reader.Seek(f.Offset(), os.SEEK_SET)

	return &FileEntry{
		File:   &f,
		Reader: reader,
	}, err
}
<commit_msg>Extend documentation of fileEntry<commit_after>package main

import (
	"io"
	"os"

	"github.com/anacrolix/torrent"
)

// SeekableContent describes an io.ReadSeeker that can be closed as well.
type SeekableContent interface {
	io.ReadSeeker
	io.Closer
}

// FileEntry helps reading a torrent file.
type FileEntry struct {
	File   *torrent.File
	Reader *torrent.Reader
}

func (f FileEntry) Read(p []byte) (n int, err error) {
	return f.Reader.Read(p)
}

// Seek seeks to the correct file position, paying attention to the offset.
func (f FileEntry) Seek(offset int64, whence int) (int64, error) {
	return f.Reader.Seek(offset+f.File.Offset(), whence)
}

// Close closes the file reader.
func (f FileEntry) Close() error {
	return f.Reader.Close()
}

// NewFileReader sets up a torrent file for streaming reading.
func NewFileReader(f torrent.File) (SeekableContent, error) {
	// We read ahead 1% of the file continuously.
	var readahead = f.Length() / 100

	// We begin by prioritizing 5% of the beginning of the file.
	f.PrioritizeRegion(f.Offset(), readahead*5)

	reader := t.NewReader()
	reader.SetReadahead(readahead)
	reader.SetResponsive()
	_, err := reader.Seek(f.Offset(), os.SEEK_SET)

	return &FileEntry{
		File:   &f,
		Reader: reader,
	}, err
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	irc "github.com/darkliquid/goirc/client"
	"github.com/fluffle/golog/logging"
	"html"
	"net"
	"strings"
)

func URLTitler(conn *irc.Conn, line *irc.Line, target string, url string) {
	// Don't expand our own URLs
	if line.Nick == conn.Me.Nick {
		return
	}

	// use custom client from utils.go because otherwise risk infinite-waits/hangs
	client := newHttpTimeoutClient()
	resp, err := client.Get(url)

	if err != nil {
		if neterr, ok := err.(net.Error); ok && neterr.Timeout() {
			conn.Privmsg(target, fmt.Sprintf("[Link] request timed out"))
		}
		logging.Warn(fmt.Sprintf("Failed to GET %s due to %s", url, err.Error()))
		return
	}

	// Make sure we close our response reader like a good citizen
	defer resp.Body.Close()

	var title string

	// No point in parsing response if it wasn't a success
	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		// Read 10kb of the body
		body := make([]byte, 10240)
		count, err := resp.Body.Read(body)

		// Couldn't read the page for some reason
		if err != nil {
			logging.Warn(fmt.Sprintf("Failed to read page response for %s due to %s", url, err.Error()))
			return
		}

		pageData := string(body[:count])

		start := strings.Index(strings.ToLower(pageData), "<title")
		end := strings.Index(strings.ToLower(pageData), "</title>")

		switch {
		case start > -1 && end > -1: // Found a title tag, get it's contents
			title = pageData[start:end]
			title = pageData[start+strings.Index(title, ">")+1 : end]
		case start > -1: // If for some reason within 10kb we get a chopped off <title>, use the remainder
			title = pageData[start:]
			title = pageData[start+strings.Index(title, ">")+1:]
		default: // No title tags to speak off, use the mime type instead to be helpful
			title = fmt.Sprintf("unknown [%s]", resp.Header.Get("Content-Type"))
		}
	} else {
		title = resp.Status // Lets just return the status text for non-successful responses
	}

	// Post back to channel
	conn.Privmsg(target, fmt.Sprintf("[Link] %s", html.UnescapeString(title)))
}
<commit_msg>Made URLTitler read/process page responses properly. Fixes #4<commit_after>package commands

import (
	"fmt"
	irc "github.com/darkliquid/goirc/client"
	"github.com/fluffle/golog/logging"
	"html"
	"net"
	"strings"
)

func URLTitler(conn *irc.Conn, line *irc.Line, target string, url string) {
	// Don't expand our own URLs
	if line.Nick == conn.Me.Nick {
		return
	}

	// use custom client from utils.go because otherwise risk infinite-waits/hangs
	client := newHttpTimeoutClient()
	resp, err := client.Get(url)

	if err != nil {
		if neterr, ok := err.(net.Error); ok && neterr.Timeout() {
			conn.Privmsg(target, fmt.Sprintf("[Link] request timed out"))
		}
		logging.Warn(fmt.Sprintf("Failed to GET %s due to %s", url, err.Error()))
		return
	}

	// Make sure we close our response reader like a good citizen
	defer resp.Body.Close()

	var title string

	// No point in parsing response if it wasn't a success
	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		// Read 10kb of the body
		body := make([]byte, 0)
		var count int

		for {
			more_body := make([]byte, 10240) // Read in up to 10kb increments
			count, err = resp.Body.Read(more_body)
			body = append(body, more_body[:count]...)
			if len(body) >= 10240 { // Got 10kb, so break out
				break
			}

			// Couldn't read the page for some reason, maybe EOF
			if err != nil {
				if len(body) == 0 {
					fmt.Printf("Failed to read page response for %s due to %s", url, err.Error())
					return
				}
				break
			}
		}

		pageData := string(body[:count])

		start := strings.Index(strings.ToLower(pageData), "<title")
		end := strings.Index(strings.ToLower(pageData), "</title>")

		switch {
		case start > -1 && end > -1: // Found a title tag, get it's contents
			title = pageData[start:end]
			title = pageData[start+strings.Index(title, ">")+1 : end]
		case start > -1: // If for some reason within 10kb we get a chopped off <title>, use the remainder
			title = pageData[start:]
			title = pageData[start+strings.Index(title, ">")+1:]
		default: // No title tags to speak off, use the mime type instead to be helpful
			title = fmt.Sprintf("unknown [%s]", resp.Header.Get("Content-Type"))
		}
	} else {
		title = resp.Status // Lets just return the status text for non-successful responses
	}

	// Post back to channel
	conn.Privmsg(target, fmt.Sprintf("[Link] %s", html.UnescapeString(title)))
}
<|endoftext|>