<commit_before>package signalfx

import (
	"sync"
	"time"

	"github.com/coreos/fleet/log"
	"github.com/zvelo/go-signalfx/sfxproto"
	"golang.org/x/net/context"
)

// DataPointCallback is a functional callback that can be passed to
// DataPointCallback as a way to have the caller calculate and return
// their own datapoints
type DataPointCallback func(defaultDims map[string]string) *DataPoints

// Reporter is an object that tracks DataPoints and manages a Client. It is the
// recommended way to send data to SignalFX.
type Reporter struct {
	client             *Client
	defaultDimensions  map[string]string
	datapoints         *DataPoints
	buckets            map[*Bucket]interface{}
	preReportCallbacks []func()
	datapointCallbacks []DataPointCallback
	mu                 sync.Mutex
	oneShots           []*sfxproto.DataPoint
}

// NewReporter returns a new Reporter object. Any dimensions supplied will be
// appended to all DataPoints sent to SignalFX. config is copied, so future
// changes to the external config object are not reflected within the reporter.
func NewReporter(config *Config,
	defaultDimensions map[string]string) *Reporter {
	return &Reporter{
		client:            NewClient(config),
		defaultDimensions: defaultDimensions,
		datapoints:        NewDataPoints(0),
		buckets:           map[*Bucket]interface{}{},
	}
}

func (r *Reporter) lock() {
	r.mu.Lock()
}

func (r *Reporter) unlock() {
	r.mu.Unlock()
}

// NewBucket creates a new Bucket object that is tracked by the Reporter.
// Buckets are goroutine safe.
func (r *Reporter) NewBucket(metric string, dimensions map[string]string) *Bucket {
	ret := NewBucket(metric, dimensions)

	r.lock()
	defer r.unlock()

	r.buckets[ret] = nil
	return ret
}

// NewCumulative returns a new DataPoint object with type CUMULATIVE_COUNTER.
// val can be any type of int, float, string, nil, pointer to those types, or a
// Getter that returns any of those types. Literal pointers are copied by value.
// Getters that return pointer types should not have their value changed, unless
// atomically, when in a Reporter, except within a PreReportCallback, for
// goroutine safety.
func (r *Reporter) NewCumulative(metric string, val interface{}, dims map[string]string) *DataPoint {
	dp, _ := NewCumulative(metric, val, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	r.datapoints.Add(dp)
	return dp
}

// NewGauge returns a new DataPoint object with type GAUGE. val can be any type
// of int, float, string, nil, pointer to those types, or a Getter that returns
// any of those types. Literal pointers are copied by value. Getters that return
// pointer types should not have their value changed, unless atomically, when in
// a Reporter, except within a PreReportCallback, for goroutine safety.
func (r *Reporter) NewGauge(metric string, val interface{}, dims map[string]string) *DataPoint {
	dp, _ := NewGauge(metric, val, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	r.datapoints.Add(dp)
	return dp
}

// NewCounter returns a new DataPoint object with type COUNTER. val can be any
// type of int, float, string, nil, pointer to those types, or a Getter that
// returns any of those types. Literal pointers are copied by value. Getters
// that return pointer types should not have their value changed, unless
// atomically, when in a Reporter, except within a PreReportCallback, for
// goroutine safety.
func (r *Reporter) NewCounter(metric string, val interface{}, dims map[string]string) *DataPoint {
	dp, _ := NewCounter(metric, val, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	r.datapoints.Add(dp)
	return dp
}

// NewInt32 creates a new DataPoint object with type COUNTER whose value is
// bound to an Int32. All methods on Int32 are goroutine safe and it may also be
// modified in atomic operations as an int32.
func (r *Reporter) NewInt32(metric string, dims map[string]string) (*Int32, *DataPoint) {
	ret := Int32(0)
	dp, _ := NewCounter(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// NewCumulativeInt32 creates a new DataPoint object with type
// CUMULATIVE_COUNTER whose value is bound to an Int32. All methods on Int32 are
// goroutine safe and it may also be modified in atomic operations as an int32.
func (r *Reporter) NewCumulativeInt32(metric string, dims map[string]string) (*Int32, *DataPoint) {
	ret := Int32(0)
	dp, _ := NewCumulative(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// NewInt64 creates a new DataPoint object with type COUNTER whose value is
// bound to an Int64. All methods on Int64 are goroutine safe and it may also be
// modified in atomic operations as an int64.
func (r *Reporter) NewInt64(metric string, dims map[string]string) (*Int64, *DataPoint) {
	ret := Int64(0)
	dp, _ := NewCounter(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// NewCumulativeInt64 creates a new DataPoint object with type
// CUMULATIVE_COUNTER whose value is bound to an Int64. All methods on Int64 are
// goroutine safe and it may also be modified in atomic operations as an int64.
func (r *Reporter) NewCumulativeInt64(metric string, dims map[string]string) (*Int64, *DataPoint) {
	ret := Int64(0)
	dp, _ := NewCumulative(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// NewUint32 creates a new DataPoint object with type COUNTER whose value is
// bound to an Uint32. All methods on Uint32 are goroutine safe and it may also
// be modified in atomic operations as an uint32.
func (r *Reporter) NewUint32(metric string, dims map[string]string) (*Uint32, *DataPoint) {
	ret := Uint32(0)
	dp, _ := NewCounter(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// NewCumulativeUint32 creates a new DataPoint object with type
// CUMULATIVE_COUNTER whose value is bound to an Uint32. All methods on Uint32
// are goroutine safe and it may also be modified in atomic operations as an
// uint32.
func (r *Reporter) NewCumulativeUint32(metric string, dims map[string]string) (*Uint32, *DataPoint) {
	ret := Uint32(0)
	dp, _ := NewCumulative(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// NewUint64 creates a new DataPoint object with type COUNTER whose value is
// bound to an Uint64. All methods on Uint64 are goroutine safe and it may also be
// modified in atomic operations as an uint64.
func (r *Reporter) NewUint64(metric string, dims map[string]string) (*Uint64, *DataPoint) {
	ret := Uint64(0)
	dp, _ := NewCounter(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// NewCumulativeUint64 creates a new DataPoint object with type
// CUMULATIVE_COUNTER whose value is bound to an Uint64. All methods on Uint64 are
// goroutine safe and it may also be modified in atomic operations as an uint64.
func (r *Reporter) NewCumulativeUint64(metric string, dims map[string]string) (*Uint64, *DataPoint) {
	ret := Uint64(0)
	dp, _ := NewCumulative(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// AddDataPoint provides a way to manually add DataPoint(s) to be tracked by the
// Reporter
func (r *Reporter) AddDataPoint(vals ...*DataPoint) {
	r.datapoints.Add(vals...)
}

// AddDataPoints provides a way to manually add DataPoint to be tracked by the
// Reporter
func (r *Reporter) AddDataPoints(dps *DataPoints) {
	r.datapoints.Append(dps)
}

// RemoveDataPoint takes DataPoint(s)out of the set being tracked by the
// Reporter
func (r *Reporter) RemoveDataPoint(dps ...*DataPoint) {
	r.datapoints.Remove(dps...)
}

// RemoveDataPoints takes DataPoints out of the set being tracked by the
// Reporter
func (r *Reporter) RemoveDataPoints(dps *DataPoints) {
	r.datapoints.RemoveDataPoints(dps)
}

// RemoveBucket takes Bucket(s) out of the set being tracked by the Reporter
func (r *Reporter) RemoveBucket(bs ...*Bucket) {
	r.lock()
	defer r.unlock()

	for _, b := range bs {
		delete(r.buckets, b)
	}
}

// AddPreReportCallback adds a function that is called before Report().  This is useful for refetching
// things like runtime.Memstats() so they are only fetched once per report() call. If a DataPoint
func (r *Reporter) AddPreReportCallback(f func()) {
	r.lock()
	defer r.unlock()
	r.preReportCallbacks = append(r.preReportCallbacks, f)
}

// AddDataPointsCallback adds a callback that itself will generate datapoints to report
func (r *Reporter) AddDataPointsCallback(f DataPointCallback) {
	r.lock()
	defer r.unlock()
	r.datapointCallbacks = append(r.datapointCallbacks, f)
}

// Report sends all tracked DataPoints to SignalFX. PreReportCallbacks will be
// run before building the dataset to send. DataPoint callbacks will be executed
// and added to the dataset, but do not become tracked by the Reporter.
func (r *Reporter) Report(ctx context.Context) (*DataPoints, error) {
	if ctx == nil {
		ctx = context.Background()
	} else if ctx.Err() != nil {
		return nil, ctx.Err()
	}

	r.lock()
	defer r.unlock()

	for _, f := range r.preReportCallbacks {
		f()
	}

	ret := r.datapoints.Clone()

	for _, f := range r.datapointCallbacks {
		ret.Append(f(r.defaultDimensions))
	}

	for b := range r.buckets {
		ret.Append(b.DataPoints(r.defaultDimensions))
	}

	var (
		counters           []*DataPoint
		cumulativeCounters []*DataPoint
	)
	ret = ret.filter(func(dp *DataPoint) bool {
		if err := dp.update(); err != nil {
			return false
		}

		switch *dp.pdp.MetricType {
		case sfxproto.MetricType_COUNTER:
			if dp.pdp.Value.IntValue != nil && *dp.pdp.Value.IntValue != 0 {
				counters = append(counters, dp)
				return true
			}
		case sfxproto.MetricType_CUMULATIVE_COUNTER:
			if !dp.pdp.Equal(dp.previous) {
				cumulativeCounters = append(cumulativeCounters, dp)
				return true
			}
		default:
			return true
		}
		return false
	})
	pdps := ret.ProtoDataPoints()

	// append all of the one-shots
	for _, pdp := range r.oneShots {
		pdps.Add(pdp)
	}

	if err := r.client.Submit(ctx, pdps); err != nil {
		return nil, err
	}

	// set submitted counters to zero
	for _, counter := range counters {
		// TODO: what should be done if this fails?
		counter.Set(0)
	}
	// remember submitted cumulative counter values
	for _, counter := range cumulativeCounters {
		counter.previous = counter.pdp
	}

	// and clear the one-shots
	r.oneShots = nil

	return ret, nil
}

func (r *Reporter) Inc(metric string, dimensions map[string]string, delta int64) error {
	r.lock()
	defer r.unlock()

	var protoDims []*sfxproto.Dimension
	for k, v := range r.defaultDimensions {
		// have to copy the values, since these are stored as
		// pointers…
		var dk, dv string
		dk = k
		dv = v
		protoDims = append(protoDims, &sfxproto.Dimension{Key: &dk, Value: &dv})
	}
	for k, v := range dimensions {
		var dk, dv string
		dk = k
		dv = v
		protoDims = append(protoDims, &sfxproto.Dimension{Key: &dk, Value: &dv})
	}
	timestamp := time.Now().UnixNano() / 1000000
	metricType := sfxproto.MetricType_COUNTER
	dp := &sfxproto.DataPoint{
		Metric:     &metric,
		Timestamp:  &timestamp,
		MetricType: &metricType,
		Dimensions: protoDims,
		Value:      &sfxproto.Datum{IntValue: &delta},
	}
	r.oneShots = append(r.oneShots, dp)
	return nil
}

// RunInBackground starts a goroutine which calls Reporter.Report on
// the specified interval.  It returns a function which may be used to
// cancel the backgrounding.
func (r *Reporter) RunInBackground(interval time.Duration) (cancel func()) {
	done := make(chan struct{})
	go func() {
		ticker := time.NewTicker(interval)
		for {
			select {
			case <-ticker.C:
				dps, err := r.Report(context.Background())
				if err != nil {
					log.Error(err)
				} else if dps != nil {
					log.Infof("reported %d datapoints", dps.Len())
				}
			case <-done:
				return
			}
		}
	}()
	return func() {
		done <- struct{}{}
	}
}
<commit_msg>Set time for continually-reported metrics<commit_after>package signalfx

import (
	"sync"
	"time"

	"github.com/coreos/fleet/log"
	"github.com/zvelo/go-signalfx/sfxproto"
	"golang.org/x/net/context"
)

// DataPointCallback is a functional callback that can be passed to
// DataPointCallback as a way to have the caller calculate and return
// their own datapoints
type DataPointCallback func(defaultDims map[string]string) *DataPoints

// Reporter is an object that tracks DataPoints and manages a Client. It is the
// recommended way to send data to SignalFX.
type Reporter struct {
	client             *Client
	defaultDimensions  map[string]string
	datapoints         *DataPoints
	buckets            map[*Bucket]interface{}
	preReportCallbacks []func()
	datapointCallbacks []DataPointCallback
	mu                 sync.Mutex
	oneShots           []*sfxproto.DataPoint
}

// NewReporter returns a new Reporter object. Any dimensions supplied will be
// appended to all DataPoints sent to SignalFX. config is copied, so future
// changes to the external config object are not reflected within the reporter.
func NewReporter(config *Config,
	defaultDimensions map[string]string) *Reporter {
	return &Reporter{
		client:            NewClient(config),
		defaultDimensions: defaultDimensions,
		datapoints:        NewDataPoints(0),
		buckets:           map[*Bucket]interface{}{},
	}
}

func (r *Reporter) lock() {
	r.mu.Lock()
}

func (r *Reporter) unlock() {
	r.mu.Unlock()
}

// NewBucket creates a new Bucket object that is tracked by the Reporter.
// Buckets are goroutine safe.
func (r *Reporter) NewBucket(metric string, dimensions map[string]string) *Bucket {
	ret := NewBucket(metric, dimensions)

	r.lock()
	defer r.unlock()

	r.buckets[ret] = nil
	return ret
}

// NewCumulative returns a new DataPoint object with type CUMULATIVE_COUNTER.
// val can be any type of int, float, string, nil, pointer to those types, or a
// Getter that returns any of those types. Literal pointers are copied by value.
// Getters that return pointer types should not have their value changed, unless
// atomically, when in a Reporter, except within a PreReportCallback, for
// goroutine safety.
func (r *Reporter) NewCumulative(metric string, val interface{}, dims map[string]string) *DataPoint {
	dp, _ := NewCumulative(metric, val, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	r.datapoints.Add(dp)
	return dp
}

// NewGauge returns a new DataPoint object with type GAUGE. val can be any type
// of int, float, string, nil, pointer to those types, or a Getter that returns
// any of those types. Literal pointers are copied by value. Getters that return
// pointer types should not have their value changed, unless atomically, when in
// a Reporter, except within a PreReportCallback, for goroutine safety.
func (r *Reporter) NewGauge(metric string, val interface{}, dims map[string]string) *DataPoint {
	dp, _ := NewGauge(metric, val, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	r.datapoints.Add(dp)
	return dp
}

// NewCounter returns a new DataPoint object with type COUNTER. val can be any
// type of int, float, string, nil, pointer to those types, or a Getter that
// returns any of those types. Literal pointers are copied by value. Getters
// that return pointer types should not have their value changed, unless
// atomically, when in a Reporter, except within a PreReportCallback, for
// goroutine safety.
func (r *Reporter) NewCounter(metric string, val interface{}, dims map[string]string) *DataPoint {
	dp, _ := NewCounter(metric, val, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	r.datapoints.Add(dp)
	return dp
}

// NewInt32 creates a new DataPoint object with type COUNTER whose value is
// bound to an Int32. All methods on Int32 are goroutine safe and it may also be
// modified in atomic operations as an int32.
func (r *Reporter) NewInt32(metric string, dims map[string]string) (*Int32, *DataPoint) {
	ret := Int32(0)
	dp, _ := NewCounter(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// NewCumulativeInt32 creates a new DataPoint object with type
// CUMULATIVE_COUNTER whose value is bound to an Int32. All methods on Int32 are
// goroutine safe and it may also be modified in atomic operations as an int32.
func (r *Reporter) NewCumulativeInt32(metric string, dims map[string]string) (*Int32, *DataPoint) {
	ret := Int32(0)
	dp, _ := NewCumulative(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// NewInt64 creates a new DataPoint object with type COUNTER whose value is
// bound to an Int64. All methods on Int64 are goroutine safe and it may also be
// modified in atomic operations as an int64.
func (r *Reporter) NewInt64(metric string, dims map[string]string) (*Int64, *DataPoint) {
	ret := Int64(0)
	dp, _ := NewCounter(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// NewCumulativeInt64 creates a new DataPoint object with type
// CUMULATIVE_COUNTER whose value is bound to an Int64. All methods on Int64 are
// goroutine safe and it may also be modified in atomic operations as an int64.
func (r *Reporter) NewCumulativeInt64(metric string, dims map[string]string) (*Int64, *DataPoint) {
	ret := Int64(0)
	dp, _ := NewCumulative(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// NewUint32 creates a new DataPoint object with type COUNTER whose value is
// bound to an Uint32. All methods on Uint32 are goroutine safe and it may also
// be modified in atomic operations as an uint32.
func (r *Reporter) NewUint32(metric string, dims map[string]string) (*Uint32, *DataPoint) {
	ret := Uint32(0)
	dp, _ := NewCounter(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// NewCumulativeUint32 creates a new DataPoint object with type
// CUMULATIVE_COUNTER whose value is bound to an Uint32. All methods on Uint32
// are goroutine safe and it may also be modified in atomic operations as an
// uint32.
func (r *Reporter) NewCumulativeUint32(metric string, dims map[string]string) (*Uint32, *DataPoint) {
	ret := Uint32(0)
	dp, _ := NewCumulative(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// NewUint64 creates a new DataPoint object with type COUNTER whose value is
// bound to an Uint64. All methods on Uint64 are goroutine safe and it may also be
// modified in atomic operations as an uint64.
func (r *Reporter) NewUint64(metric string, dims map[string]string) (*Uint64, *DataPoint) {
	ret := Uint64(0)
	dp, _ := NewCounter(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// NewCumulativeUint64 creates a new DataPoint object with type
// CUMULATIVE_COUNTER whose value is bound to an Uint64. All methods on Uint64 are
// goroutine safe and it may also be modified in atomic operations as an uint64.
func (r *Reporter) NewCumulativeUint64(metric string, dims map[string]string) (*Uint64, *DataPoint) {
	ret := Uint64(0)
	dp, _ := NewCumulative(metric, &ret, sfxproto.Dimensions(r.defaultDimensions).Append(dims))
	if dp == nil {
		return nil, nil
	}
	r.datapoints.Add(dp)
	return &ret, dp
}

// AddDataPoint provides a way to manually add DataPoint(s) to be tracked by the
// Reporter
func (r *Reporter) AddDataPoint(vals ...*DataPoint) {
	r.datapoints.Add(vals...)
}

// AddDataPoints provides a way to manually add DataPoint to be tracked by the
// Reporter
func (r *Reporter) AddDataPoints(dps *DataPoints) {
	r.datapoints.Append(dps)
}

// RemoveDataPoint takes DataPoint(s)out of the set being tracked by the
// Reporter
func (r *Reporter) RemoveDataPoint(dps ...*DataPoint) {
	r.datapoints.Remove(dps...)
}

// RemoveDataPoints takes DataPoints out of the set being tracked by the
// Reporter
func (r *Reporter) RemoveDataPoints(dps *DataPoints) {
	r.datapoints.RemoveDataPoints(dps)
}

// RemoveBucket takes Bucket(s) out of the set being tracked by the Reporter
func (r *Reporter) RemoveBucket(bs ...*Bucket) {
	r.lock()
	defer r.unlock()

	for _, b := range bs {
		delete(r.buckets, b)
	}
}

// AddPreReportCallback adds a function that is called before Report().  This is useful for refetching
// things like runtime.Memstats() so they are only fetched once per report() call. If a DataPoint
func (r *Reporter) AddPreReportCallback(f func()) {
	r.lock()
	defer r.unlock()
	r.preReportCallbacks = append(r.preReportCallbacks, f)
}

// AddDataPointsCallback adds a callback that itself will generate datapoints to report
func (r *Reporter) AddDataPointsCallback(f DataPointCallback) {
	r.lock()
	defer r.unlock()
	r.datapointCallbacks = append(r.datapointCallbacks, f)
}

// Report sends all tracked DataPoints to SignalFX. PreReportCallbacks will be
// run before building the dataset to send. DataPoint callbacks will be executed
// and added to the dataset, but do not become tracked by the Reporter.
func (r *Reporter) Report(ctx context.Context) (*DataPoints, error) {
	if ctx == nil {
		ctx = context.Background()
	} else if ctx.Err() != nil {
		return nil, ctx.Err()
	}

	r.lock()
	defer r.unlock()

	for _, f := range r.preReportCallbacks {
		f()
	}

	ret := r.datapoints.Clone()

	for _, f := range r.datapointCallbacks {
		ret.Append(f(r.defaultDimensions))
	}

	for b := range r.buckets {
		ret.Append(b.DataPoints(r.defaultDimensions))
	}

	var (
		counters           []*DataPoint
		cumulativeCounters []*DataPoint
	)
	ret = ret.filter(func(dp *DataPoint) bool {
		if err := dp.update(); err != nil {
			return false
		}

		switch *dp.pdp.MetricType {
		case sfxproto.MetricType_COUNTER:
			if dp.pdp.Value.IntValue != nil && *dp.pdp.Value.IntValue != 0 {
				counters = append(counters, dp)
				dp.SetTime(time.Now())
				return true
			}
		case sfxproto.MetricType_CUMULATIVE_COUNTER:
			if !dp.pdp.Equal(dp.previous) {
				cumulativeCounters = append(cumulativeCounters, dp)
				dp.SetTime(time.Now())
				return true
			}
		default:
			dp.SetTime(time.Now())
			return true
		}
		return false
	})
	pdps := ret.ProtoDataPoints()

	// append all of the one-shots
	for _, pdp := range r.oneShots {
		pdps.Add(pdp)
	}

	if err := r.client.Submit(ctx, pdps); err != nil {
		return nil, err
	}

	// set submitted counters to zero
	for _, counter := range counters {
		// TODO: what should be done if this fails?
		counter.Set(0)
	}
	// remember submitted cumulative counter values
	for _, counter := range cumulativeCounters {
		counter.previous = counter.pdp
	}

	// and clear the one-shots
	r.oneShots = nil

	return ret, nil
}

func (r *Reporter) Inc(metric string, dimensions map[string]string, delta int64) error {
	r.lock()
	defer r.unlock()

	var protoDims []*sfxproto.Dimension
	for k, v := range r.defaultDimensions {
		// have to copy the values, since these are stored as
		// pointers…
		var dk, dv string
		dk = k
		dv = v
		protoDims = append(protoDims, &sfxproto.Dimension{Key: &dk, Value: &dv})
	}
	for k, v := range dimensions {
		var dk, dv string
		dk = k
		dv = v
		protoDims = append(protoDims, &sfxproto.Dimension{Key: &dk, Value: &dv})
	}
	timestamp := time.Now().UnixNano() / 1000000
	metricType := sfxproto.MetricType_COUNTER
	dp := &sfxproto.DataPoint{
		Metric:     &metric,
		Timestamp:  &timestamp,
		MetricType: &metricType,
		Dimensions: protoDims,
		Value:      &sfxproto.Datum{IntValue: &delta},
	}
	r.oneShots = append(r.oneShots, dp)
	return nil
}

// RunInBackground starts a goroutine which calls Reporter.Report on
// the specified interval.  It returns a function which may be used to
// cancel the backgrounding.
func (r *Reporter) RunInBackground(interval time.Duration) (cancel func()) {
	done := make(chan struct{})
	go func() {
		ticker := time.NewTicker(interval)
		for {
			select {
			case <-ticker.C:
				dps, err := r.Report(context.Background())
				if err != nil {
					log.Error(err)
				} else if dps != nil {
					log.Infof("reported %d datapoints", dps.Len())
				}
			case <-done:
				return
			}
		}
	}()
	return func() {
		done <- struct{}{}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"github.com/aarondl/pack"
	"log"
)

const (
	initialStackSize        = 20
	kidOffset               = 32
	stackIndexMask   uint64 = 0xFFFFFFFF
)

// bitFilter is a type used to filter versions based on their index in an array
// and an easy way to do cumulative versioning.
type bitFilter uint64

// Set sets a bit in the filter.
func (b bitFilter) Set(index uint) bitFilter {
	return bitFilter((1 << uint64(index)) | uint64(b))
}

// IsSet checks if a a bit in the filter is set.
func (b bitFilter) IsSet(index uint) bool {
	return 0 != ((1 << uint64(index)) & uint64(b))
}

// Clear turns off a bit in the filter.
func (b bitFilter) Clear(index uint) bitFilter {
	return bitFilter(^(1 << uint64(index)) & uint64(b))
}

// Add is the union of two bitFilters.
func (b bitFilter) Add(a bitFilter) bitFilter {
	return bitFilter(uint64(a) | uint64(b))
}

// versionProvider allows us to look up available versions for each package
// the array returned must be in sorted order for the best result from the
// solver as it assumes [0] is the latest version, and [1]... is less than that.
//
// GetVersions: Get the versions only from the dependency information.
// GetGraphs: Get a list of graphs showing dependency information for each
// version of the package.
type versionProvider interface {
	GetVersions(string) []*pack.Version
	GetGraph(string, *pack.Version) *depgraph
}

// stacknode helps to emulate recursion and perform safejumps.
type stacknode struct {
	kid     int
	version int
	ai      int
	current *depnode
	parent  *depnode
}

// savestate is the state of the algorithm at an activation point.
type savestate struct {
	kid     int
	version int
	si      int
	ai      int
	current *depnode
	parent  *depnode
	stack   []stacknode
}

// activation is the details of a packages activation.
type activation struct {
	name    string
	version *pack.Version
	state   *savestate
	filter  bitFilter
}

// String is used to debug activations.
func (a activation) String() string {
	var buf bytes.Buffer
	buf.WriteString(a.name)
	if a.version != nil {
		buf.WriteRune(space)
		buf.WriteString(a.version.String())
	}
	return buf.String()
}

/*
solve a dependency graph. This algorithm is a depth first search with
backjumping to resolve conflicts.
*/
func (g *depgraph) solve(vp versionProvider) error {
	if len(g.head.kids) == 0 {
		return nil
	}

	var current, parent *depnode = g.head, nil
	var stack = make([]stacknode, 0, initialStackSize) // Avoid allocations
	var si, ai, kid = -1, -1, 0
	var activations []*activation
	var active *activation
	var versions = make(map[string][]*pack.Version)
	var version *pack.Version
	var vs []*pack.Version
	var vi int
	var filter bitFilter
	var excluded int
	var ok bool
	var conflicts = make([]string, 0)
	var conflict bool

	var verbose = true // Replace by flag.

	for i := 0; i < 20; i++ {
		name := current.d.Name
		if verbose {
			log.Println("Current:", current.d)
		}

		if current == g.head {
			if kid >= len(current.kids) {
				if verbose {
					log.Println("Success!")
					log.Println(activations)
				}
				return nil
			} else {
				goto NEXT
			}
		}

		// Skip Activation if we're on any child other than 0.
		if kid != 0 {
			goto NEXT
		}

		// Fetch Versions for current.
		vs = nil
		if vs, ok = versions[name]; !ok {
			vstmp := vp.GetVersions(current.d.Name)
			vs = make([]*pack.Version, len(vstmp))
			copy(vs, vstmp)
			versions[name] = vs
			if verbose {
				log.Printf("Fetched Versions")
			}
		}

		if verbose {
			log.Println("Versions:", vs)
		}

		// Weed out versions.
		filter = 0
		excluded = 0
		for j := 0; j < len(vs); j++ {
			for _, con := range current.d.Constraints {
				if !vs[j].Satisfies(con.Operator, con.Version) {
					if verbose {
						log.Println("Removing unacceptable version:", vs[j])
					}
					filter.Set(uint(j))
					excluded++
				}
			}
		}

		if parent == g.head && len(vs) == excluded {
			return fmt.Errorf("No versions to satisfy root dependency: %v",
				current.d)
		}

		// Check for activeness. The first activation will always serve as the
		// main activation point, with the others simply being save points.
		active = nil
		for j := 0; j < len(activations); j++ {
			if activations[j].name == name {
				active = activations[j]
				break
			}
		}

		version = nil
		conflict = false
		if active != nil {
			if verbose {
				log.Println("Found activation:", active)
			}

			// Check that we comply with the currently active.
			for _, con := range current.d.Constraints {
				if !active.version.Satisfies(con.Operator, con.Version) {
					// We've found a problem.
					log.Printf("Conflict (%v): %v fails new constraint: %v%v",
						name, active.version, con.Operator.String(),
						con.Version)

					conflict = true
					break
				}
			}

			// Add the current filter to the primary activation.
			version = active.version
			active.filter = active.filter.Add(filter)
		} else {
			if verbose {
				log.Println("Not activated:", name)
			}
			// Find a suitable version
			for ; version == nil && vi < len(vs); vi++ {
				if active != nil && active.filter.IsSet(uint(vi)) {
					continue
				}
				if len(current.d.Constraints) == 0 {
					version = vs[vi]
					break
				}
				for _, con := range current.d.Constraints {
					if vs[vi].Satisfies(con.Operator, con.Version) {
						version = vs[vi]
						break
					}
				}
			}

			if version == nil {
				if verbose {
					log.Printf("Conflict (%v): has no usable versions %v",
						name, vs)
				}
				// No version could be found, this is a conflict of sorts.
				conflict = true
			}
		}

		if conflict {
			conflicts = append(conflicts, name)
			// If we cannot climb the stack any further, go back to a save
			// point if one exists.
			if parent == g.head {
				if len(conflicts) == 0 {
					// No conflicts exist to jump back to.
					return fmt.Errorf("We've tried everything mate: %v",
						conflicts)
				}
				name = conflicts[0]
				conflicts = conflicts[1:]
				var st *savestate
				for i := 0; i < len(activations); i++ {
					if activations[i].name == name {
						st = activations[i].state
						break
					}
				}
				if st == nil {
					return fmt.Errorf("Conflict's activation not found: %v %v",
						name, activations)
				}
				current = st.current
				parent = st.parent
				if verbose {
					log.Println("Conflict! Restoring:", current.d.Name)
				}
				ai = st.ai
				vi = st.version
				vi++
				stack = st.stack
				si = st.si
				kid = 0
				activations = activations[:ai]
				if verbose {
					log.Println("Activations:", activations)
				}
				continue
			}

			// We can still climb the stack, try it.
			sn := stack[si]
			parent = sn.parent
			current = sn.current
			if verbose {
				log.Println("Conflict! Popping:", current.d.Name)
			}
			ai = sn.ai
			vi = sn.version
			vi++
			stack = stack[:len(stack)-1]
			si--
			kid = 0
			activations = activations[:ai]
			if verbose {
				log.Println("Activations:", activations)
			}
			continue
		}

		// Add ourselves to the list of activators.
		ai++
		activations = append(activations, &activation{
			name:    name,
			version: version,
			filter:  filter,
			state: &savestate{
				kid:     kid,
				version: vi,
				si:      si,
				ai:      ai,
				current: current,
				parent:  parent,
				stack:   make([]stacknode, len(stack)),
			},
		})
		copy(activations[len(activations)-1].state.stack, stack)
		current.v = version

		if verbose {
			log.Printf("Added: %v %v to activations", name, version)
			log.Println("Activations:", activations)
		}

		if verbose {
			log.Println("Fetching Dependencies for:", name)
		}

		current.kids = vp.GetGraph(name, version).head.kids
		if verbose {
			var b bytes.Buffer
			for _, dep := range current.kids {
				b.WriteString(dep.d.String())
				b.WriteRune(space)
			}
			log.Println("Got dependencies:", b.String())
		}

	NEXT:
		// Push current on to stack, go into child.
		if kid < len(current.kids) {
			if verbose {
				log.Println("Pushing:", name, kid)
			}
			stack = append(stack, stacknode{
				kid:     kid,
				version: vi,
				ai:      ai,
				current: current,
				parent:  parent,
			})
			parent = current
			current = current.kids[kid]
			ai = len(activations) - 1
			kid = 0
			vi = 0
			si++
			continue
		}

		// Pop off the stack back to parent.
		sn := stack[si]
		parent = sn.parent
		current = sn.current
		if verbose {
			log.Println("Popping:", current.d.Name)
		}
		kid = sn.kid
		vi = sn.version
		ai = sn.ai
		stack = stack[:len(stack)-1]
		kid++
		si--
	}

	return nil
}
<commit_msg>Fix some comments.<commit_after>package main

import (
	"bytes"
	"fmt"
	"github.com/aarondl/pack"
	"log"
)

const (
	initialStackSize        = 20
	kidOffset               = 32
	stackIndexMask   uint64 = 0xFFFFFFFF
)

// bitFilter is a type used to filter versions based on their index in an array
type bitFilter uint64

// Set sets a bit in the filter.
func (b bitFilter) Set(index uint) bitFilter {
	return bitFilter((1 << uint64(index)) | uint64(b))
}

// IsSet checks if a a bit in the filter is set.
func (b bitFilter) IsSet(index uint) bool {
	return 0 != ((1 << uint64(index)) & uint64(b))
}

// Clear turns off a bit in the filter.
func (b bitFilter) Clear(index uint) bitFilter {
	return bitFilter(^(1 << uint64(index)) & uint64(b))
}

// Add is the union of two bitFilters.
func (b bitFilter) Add(a bitFilter) bitFilter {
	return bitFilter(uint64(a) | uint64(b))
}

// versionProvider allows us to look up available versions for each package
// the array returned must be in reverse sorted order for the best result from
// the solver as it assumes [0] > [1] > [2]...
type versionProvider interface {
	// GetVersions gets the versions only from the dependency information.
	GetVersions(string) []*pack.Version
	// GetGraphs gets a list of graphs showing dependency information for each
	// version of the package.
	GetGraph(string, *pack.Version) *depgraph
}

// stacknode helps to emulate recursion and perform safejumps.
type stacknode struct {
	kid     int
	version int
	ai      int
	current *depnode
	parent  *depnode
}

// savestate is the state of the algorithm at an activation point.
type savestate struct {
	kid     int
	version int
	si      int
	ai      int
	current *depnode
	parent  *depnode
	stack   []stacknode
}

// activation is the details of a packages activation.
type activation struct {
	name    string
	version *pack.Version
	state   *savestate
	filter  bitFilter
}

// String is used to debug activations.
func (a activation) String() string {
	var buf bytes.Buffer
	buf.WriteString(a.name)
	if a.version != nil {
		buf.WriteRune(space)
		buf.WriteString(a.version.String())
	}
	return buf.String()
}

/*
solve a dependency graph. This algorithm is a depth first search with
backjumping to resolve conflicts.
*/
func (g *depgraph) solve(vp versionProvider) error {
	if len(g.head.kids) == 0 {
		return nil
	}

	var current, parent *depnode = g.head, nil
	var stack = make([]stacknode, 0, initialStackSize) // Avoid allocations
	var si, ai, kid = -1, -1, 0
	var activations []*activation
	var active *activation
	var versions = make(map[string][]*pack.Version)
	var version *pack.Version
	var vs []*pack.Version
	var vi int
	var filter bitFilter
	var excluded int
	var ok bool
	var conflicts = make([]string, 0)
	var conflict bool

	var verbose = false // Replace by flag.

	for i := 0; i < 20; i++ {
		name := current.d.Name
		if verbose {
			log.Println("Current:", current.d)
		}

		if current == g.head {
			if kid >= len(current.kids) {
				if verbose {
					log.Println("Success!")
					log.Println(activations)
				}
				return nil
			} else {
				goto NEXT
			}
		}

		// Skip Activation if we're on any child other than 0.
		if kid != 0 {
			goto NEXT
		}

		// Fetch Versions for current.
		vs = nil
		if vs, ok = versions[name]; !ok {
			vstmp := vp.GetVersions(current.d.Name)
			vs = make([]*pack.Version, len(vstmp))
			copy(vs, vstmp)
			versions[name] = vs
			if verbose {
				log.Printf("Fetched Versions")
			}
		}

		if verbose {
			log.Println("Versions:", vs)
		}

		// Weed out versions.
		filter = 0
		excluded = 0
		for j := 0; j < len(vs); j++ {
			for _, con := range current.d.Constraints {
				if !vs[j].Satisfies(con.Operator, con.Version) {
					if verbose {
						log.Println("Removing unacceptable version:", vs[j])
					}
					filter.Set(uint(j))
					excluded++
				}
			}
		}

		if parent == g.head && len(vs) == excluded {
			return fmt.Errorf("No versions to satisfy root dependency: %v",
				current.d)
		}

		// Check for activeness. The first activation will always serve as the
		// main activation point, with the others simply being save points.
		active = nil
		for j := 0; j < len(activations); j++ {
			if activations[j].name == name {
				active = activations[j]
				break
			}
		}

		version = nil
		conflict = false
		if active != nil {
			if verbose {
				log.Println("Found activation:", active)
			}

			// Check that we comply with the currently active.
			for _, con := range current.d.Constraints {
				if !active.version.Satisfies(con.Operator, con.Version) {
					// We've found a problem.
					if verbose {
						log.Printf("Conflict (%v): %v fails constraint: %v%v",
							name, active.version, con.Operator.String(),
							con.Version)
					}

					conflict = true
					break
				}
			}

			// Add the current filter to the primary activation.
			version = active.version
			active.filter = active.filter.Add(filter)
		} else {
			if verbose {
				log.Println("Not activated:", name)
			}
			// Find a suitable version
			for ; version == nil && vi < len(vs); vi++ {
				if active != nil && active.filter.IsSet(uint(vi)) {
					continue
				}
				if len(current.d.Constraints) == 0 {
					version = vs[vi]
					break
				}
				for _, con := range current.d.Constraints {
					if vs[vi].Satisfies(con.Operator, con.Version) {
						version = vs[vi]
						break
					}
				}
			}

			if version == nil {
				if verbose {
					log.Printf("Conflict (%v): has no usable versions %v",
						name, vs)
				}
				// No version could be found, this is a conflict of sorts.
				conflict = true
			}
		}

		if conflict {
			conflicts = append(conflicts, name)
			// If we cannot climb the stack any further, go back to a save
			// point if one exists.
			if parent == g.head {
				if len(conflicts) == 0 {
					// No conflicts exist to jump back to.
					return fmt.Errorf("We've tried everything mate: %v",
						conflicts)
				}
				name = conflicts[0]
				conflicts = conflicts[1:]
				var st *savestate
				for i := 0; i < len(activations); i++ {
					if activations[i].name == name {
						st = activations[i].state
						break
					}
				}
				if st == nil {
					return fmt.Errorf("Conflict's activation not found: %v %v",
						name, activations)
				}
				current = st.current
				parent = st.parent
				if verbose {
					log.Println("Conflict! Restoring:", current.d.Name)
				}
				ai = st.ai
				vi = st.version
				vi++
				stack = st.stack
				si = st.si
				kid = 0
				activations = activations[:ai]
				if verbose {
					log.Println("Activations:", activations)
				}
				continue
			}

			// We can still climb the stack, try it.
			sn := stack[si]
			parent = sn.parent
			current = sn.current
			if verbose {
				log.Println("Conflict! Popping:", current.d.Name)
			}
			ai = sn.ai
			vi = sn.version
			vi++
			stack = stack[:len(stack)-1]
			si--
			kid = 0
			activations = activations[:ai]
			if verbose {
				log.Println("Activations:", activations)
			}
			continue
		}

		// Add ourselves to the list of activators.
		ai++
		activations = append(activations, &activation{
			name:    name,
			version: version,
			filter:  filter,
			state: &savestate{
				kid:     kid,
				version: vi,
				si:      si,
				ai:      ai,
				current: current,
				parent:  parent,
				stack:   make([]stacknode, len(stack)),
			},
		})
		copy(activations[len(activations)-1].state.stack, stack)
		current.v = version

		if verbose {
			log.Printf("Added: %v %v to activations", name, version)
			log.Println("Activations:", activations)
		}

		if verbose {
			log.Println("Fetching Dependencies for:", name)
		}

		current.kids = vp.GetGraph(name, version).head.kids
		if verbose {
			var b bytes.Buffer
			for _, dep := range current.kids {
				b.WriteString(dep.d.String())
				b.WriteRune(space)
			}
			log.Println("Got dependencies:", b.String())
		}

	NEXT:
		// Push current on to stack, go into child.
		if kid < len(current.kids) {
			if verbose {
				log.Println("Pushing:", name, kid)
			}
			stack = append(stack, stacknode{
				kid:     kid,
				version: vi,
				ai:      ai,
				current: current,
				parent:  parent,
			})
			parent = current
			current = current.kids[kid]
			ai = len(activations) - 1
			kid = 0
			vi = 0
			si++
			continue
		}

		// Pop off the stack back to parent.
		sn := stack[si]
		parent = sn.parent
		current = sn.current
		if verbose {
			log.Println("Popping:", current.d.Name)
		}
		kid = sn.kid
		vi = sn.version
		ai = sn.ai
		stack = stack[:len(stack)-1]
		kid++
		si--
	}

	return nil
}
<|endoftext|>


<commit_before>package whois

import (
	"fmt"
	"mime"
	"net/http"
	"time"
)

// Response represents a whois response from a server.
type Response struct {
	Query       string
	Host        string
	FetchedAt   time.Time
	ContentType string
	Charset     string
	Body        []byte
}

// NewResponse initializes a new whois response.
func NewResponse(query, host string) *Response {
	return &Response{
		Query:     query,
		Host:      host,
		FetchedAt: time.Now(),
	}
}

// String returns the response body.
func (res *Response) String() string {
	return string(res.Body)
}

// DetectContentType detects and sets the response content type and charset.
func (res *Response) DetectContentType(mt string) {
	// Sensible defaults
	res.ContentType = "text/plain"
	res.Charset = "utf-8"

	// Autodetect if not passed a Content-Type header
	if mt == "" {
		mt = http.DetectContentType(res.Body)
	}

	// Content type (e.g. text/plan or text/html)
	ct, mh, err := mime.ParseMediaType(mt)
	if err != nil {
		return
	}
	res.ContentType = ct

	// Character set (e.g. utf-8)
	cs, ok := mh["charset"]
	if !ok {
		return
	}
	res.Charset = cs
}

// Header returns a stringproto header representing the response.
func (res *Response) Header() http.Header {
	h := make(http.Header)
	h.Set("Query", res.Query)
	h.Set("Host", res.Host)
	h.Set("Fetched-At", res.FetchedAt.Format(time.RFC3339))
	h.Set("Content-Type", res.contentType())
	fmt.Printf("Content-Type: %s\n\n", res.ContentType)
	return h
}

func (res *Response) contentType() string {
	return mime.FormatMediaType(res.ContentType, map[string]string{"charset": res.Charset})
}
<commit_msg>Remove debug line.<commit_after>package whois

import (
	"mime"
	"net/http"
	"time"
)

// Response represents a whois response from a server.
type Response struct {
	Query       string
	Host        string
	FetchedAt   time.Time
	ContentType string
	Charset     string
	Body        []byte
}

// NewResponse initializes a new whois response.
func NewResponse(query, host string) *Response {
	return &Response{
		Query:     query,
		Host:      host,
		FetchedAt: time.Now(),
	}
}

// String returns the response body.
func (res *Response) String() string {
	return string(res.Body)
}

// DetectContentType detects and sets the response content type and charset.
func (res *Response) DetectContentType(mt string) {
	// Sensible defaults
	res.ContentType = "text/plain"
	res.Charset = "utf-8"

	// Autodetect if not passed a Content-Type header
	if mt == "" {
		mt = http.DetectContentType(res.Body)
	}

	// Content type (e.g. text/plan or text/html)
	ct, mh, err := mime.ParseMediaType(mt)
	if err != nil {
		return
	}
	res.ContentType = ct

	// Character set (e.g. utf-8)
	cs, ok := mh["charset"]
	if !ok {
		return
	}
	res.Charset = cs
}

// Header returns a stringproto header representing the response.
func (res *Response) Header() http.Header {
	h := make(http.Header)
	h.Set("Query", res.Query)
	h.Set("Host", res.Host)
	h.Set("Fetched-At", res.FetchedAt.Format(time.RFC3339))
	h.Set("Content-Type", res.contentType())
	return h
}

func (res *Response) contentType() string {
	return mime.FormatMediaType(res.ContentType, map[string]string{"charset": res.Charset})
}
<|endoftext|>


<commit_before>package elastic

import (
//"encoding/json"
)

/*
 * Elasticsearch failure representation
 * e.g.:{"error":{"root_cause":[{"type":"no_shard_available_action_exception","reason":"No shard available for [org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest@74508901]"}],"type":"no_shard_available_action_exception","reason":"No shard available for [org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest@74508901]"},"status":503}
 * e.g.:{"error":{"root_cause":[{"type":"index_already_exists_exception","reason":"already exists","index":"my_index"}],"type":"index_already_exists_exception","reason":"already exists","index":"my_index"},"status":400}
 */
type Failure struct {
	err Error `json:"error"`
}
type Error struct {
	rootCause []Dict `json:"root_cause"`
	kind      string `json:"type"`
	reason    string `json:"reason"`
	status    int    `json:"status"`
}

/*
 * Elasticsearch success representation
 * e.g.: {"acknowledged":true}
 */
type Success struct {
	Acknowledged bool `json:"acknowledged"`
}

/*
 * Elasticsearch unvalid representation
 * e.g.: {"valid":false,"_shards":{"total":1,"successful":1,"failed":0},"explanations":[{"index":"gb","valid":false,"error":"org.elasticsearch.index.query.QueryParsingException: No query registered for [tweet]"}]}
 */
type Unvalid struct {
	Valid       bool   `json:"valid"`
	Shards      Dict   `json:"_shards"`
	Explanation []Dict `json:"explanations"`
}

/////////////////////////////////// Search Query

/*
 * Elasticsearch shard response representation
 */
type Shard struct {
	Total      int `json:"total"`
	Successful int `json:"successful"`
	Failed     int `json:"failed"`
}

/*
 * Elasticsearch search hits representation
 */
type Hits struct {
	Total    int         `json:"total"`
	MaxScore interface{} `json:"max_score"`
	Hits     []Dict      `json:"hits"`
}

/*
 * Elasticsearch explain result
 * e.g. {"valid":true,"_shards":{"total":1,"successful":1,"failed":0},"explanations":[{"index":"my_index","valid":true,"explanation":"+((name:b name:br name:bro name:brow name:brown) (name:f name:fo)) #ConstantScore(+ConstantScore(_type:my_type))"}]}
 */
/*
 * Elastisearch search result representation
 * e.g. {"took":1,"timed_out":false,"_shards":{"total":5,"successful":5,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}
 * e.g. {"took":3,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":1,"max_score":0.50741017,"hits":[{"_index":"my_index","_type":"my_type","_id":"1","_score":0.50741017,"_source":{"name":"Brown foxes"}}]}}
 */
type SearchResult struct {
	Took     int   `json:"took"`
	TimedOut bool  `json:"timed_out"`
	Shards   Shard `json:"_shards"`
	Hits     Hits  `json:"hits"`
}

/////////////////////////////////// Analyze Query

/*
 * Elasticsearch analyze result representation
 * e.g. {"tokens":[{"token":"quick","start_offset":0,"end_offset":5,"type":"<ALPHANUM>","position":0},{"token":"brown","start_offset":6,"end_offset":11,"type":"<ALPHANUM>","position":1},{"token":"fox","start_offset":12,"end_offset":15,"type":"<ALPHANUM>","position":2}]}
 */
type AnalyzeResult struct {
	tokens []AnalyzeToken `json:"tokens"`
}

type AnalyzeToken struct {
	token       string `json:"token"`
	startOffset int    `json:"start_offset"`
	endOffset   int    `json:"end_offset"`
	tokenType   string `json:"type"`
	position    int    `json:"position"`
}

/////////////////////////////////// Insert Query

/*
 * Elasticsearch insert result representation
 * e.g. {"_index":"my_index","_type":"groups","_id":"1","_version":1,"_shards":{"total":2,"successful":1,"failed":0},"created":true}
 */
type InsertResult struct {
	index   string `json:"_index"`
	doctype string `json:"_type"`
	id      string `json:"_id"`
	version int    `json:"version"`
	shards  Shard  `json:"_shards"`
	created bool   `json:"created"`
	status  int    `json:"status"`
}

/////////////////////////////////// Bulk Query
/*
 * Elasticsearch bulk result representation
 * e.g. {"took":118,"errors":false,"items":[{"index":{"_index":"my_index","_type":"my_type","_id":"1","_version":1,"_shards":{"total":2,"successful":1,"failed":0},"status":201}},{"index":{"_index":"my_index","_type":"my_type","_id":"2","_version":1,"_shards":{"total":2,"successful":1,"failed":0},"status":201}}]}
 */
type BulkResult struct {
	took   int            `josn:"took"`
	errors bool           `json:"errors"`
	items  []InsertResult `json:"items"`
}
<commit_msg>fix for golint<commit_after>package elastic

import (
//"encoding/json"
)

// Failure is a structure representing the Elasticsearch failure response
// e.g.:{"error":{"root_cause":[{"type":"no_shard_available_action_exception","reason":"No shard available for [org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest@74508901]"}],"type":"no_shard_available_action_exception","reason":"No shard available for [org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest@74508901]"},"status":503}
// e.g.:{"error":{"root_cause":[{"type":"index_already_exists_exception","reason":"already exists","index":"my_index"}],"type":"index_already_exists_exception","reason":"already exists","index":"my_index"},"status":400}
type Failure struct {
	err Error `json:"error"`
}

// Error is a structure representing the Elasticsearch error response
type Error struct {
	rootCause []Dict `json:"root_cause"`
	kind      string `json:"type"`
	reason    string `json:"reason"`
	status    int    `json:"status"`
}

// Success is a structure representing an Elasticsearch success response
// e.g.: {"acknowledged":true}
type Success struct {
	Acknowledged bool `json:"acknowledged"`
}

// Unvalid is a structure representing an Elasticsearch unvalid response
// e.g.: {"valid":false,"_shards":{"total":1,"successful":1,"failed":0},"explanations":[{"index":"gb","valid":false,"error":"org.elasticsearch.index.query.QueryParsingException: No query registered for [tweet]"}]}
type Unvalid struct {
	Valid       bool   `json:"valid"`
	Shards      Dict   `json:"_shards"`
	Explanation []Dict `json:"explanations"`
}

/////////////////////////////////// Search Query

// Shard is a structure representing the Elasticsearch shard part of Search query response
type Shard struct {
	Total      int `json:"total"`
	Successful int `json:"successful"`
	Failed     int `json:"failed"`
}

// Hits is a structure representing the Elasticsearch hits part of Search query response
type Hits struct {
	Total    int         `json:"total"`
	MaxScore interface{} `json:"max_score"`
	Hits     []Dict      `json:"hits"`
}

/*
 * Elasticsearch explain result
 * e.g. {"valid":true,"_shards":{"total":1,"successful":1,"failed":0},"explanations":[{"index":"my_index","valid":true,"explanation":"+((name:b name:br name:bro name:brow name:brown) (name:f name:fo)) #ConstantScore(+ConstantScore(_type:my_type))"}]}
 */

// SearchResult is a structure representing the Elastisearch search result
// e.g. {"took":1,"timed_out":false,"_shards":{"total":5,"successful":5,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}
// e.g. {"took":3,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":1,"max_score":0.50741017,"hits":[{"_index":"my_index","_type":"my_type","_id":"1","_score":0.50741017,"_source":{"name":"Brown foxes"}}]}}
type SearchResult struct {
	Took     int   `json:"took"`
	TimedOut bool  `json:"timed_out"`
	Shards   Shard `json:"_shards"`
	Hits     Hits  `json:"hits"`
}

/////////////////////////////////// Analyze Query

// AnalyzeResult is a structure representing the Elasticsearch analyze query result
// e.g. {"tokens":[{"token":"quick","start_offset":0,"end_offset":5,"type":"<ALPHANUM>","position":0},{"token":"brown","start_offset":6,"end_offset":11,"type":"<ALPHANUM>","position":1},{"token":"fox","start_offset":12,"end_offset":15,"type":"<ALPHANUM>","position":2}]}
type AnalyzeResult struct {
	tokens []AnalyzeToken `json:"tokens"`
}

// AnalyzeToken is a structure representing part of the Elasticsearch analyze query response
type AnalyzeToken struct {
	token       string `json:"token"`
	startOffset int    `json:"start_offset"`
	endOffset   int    `json:"end_offset"`
	tokenType   string `json:"type"`
	position    int    `json:"position"`
}

/////////////////////////////////// Insert Query

// InsertResult is a strucuture representing the Elasticsearch insert query result
// e.g. {"_index":"my_index","_type":"groups","_id":"1","_version":1,"_shards":{"total":2,"successful":1,"failed":0},"created":true}
type InsertResult struct {
	index   string `json:"_index"`
	doctype string `json:"_type"`
	id      string `json:"_id"`
	version int    `json:"version"`
	shards  Shard  `json:"_shards"`
	created bool   `json:"created"`
	status  int    `json:"status"`
}

/////////////////////////////////// Bulk Query

// BulkResult is a structure representing the Elasticsearch bulk query result
// e.g. {"took":118,"errors":false,"items":[{"index":{"_index":"my_index","_type":"my_type","_id":"1","_version":1,"_shards":{"total":2,"successful":1,"failed":0},"status":201}},{"index":{"_index":"my_index","_type":"my_type","_id":"2","_version":1,"_shards":{"total":2,"successful":1,"failed":0},"status":201}}]}
type BulkResult struct {
	took   int            `josn:"took"`
	errors bool           `json:"errors"`
	items  []InsertResult `json:"items"`
}
<|endoftext|>


<commit_before>package restless

import (
    "encoding/json"
    "fmt"
    "github.com/gorilla/mux"
    "labix.org/v2/mgo"
    "labix.org/v2/mgo/bson"
    "net/http"
)

func GetAll(c *mgo.Collection, ip interface{}) {
    c.Find(nil).All(ip)
}

func Insert(c *mgo.Collection, i interface{}) (string, error) {
    info, err := c.Upsert(bson.M{"_id": nil}, i)
    id := info.UpsertedId.(bson.ObjectId)
    return id.Hex(), err
}

func RemoveId(c *mgo.Collection, id bson.ObjectId) error {
    return c.RemoveId(id)
}

func GetId(c *mgo.Collection, i interface{}, id bson.ObjectId) error {
    return c.FindId(id).One(i)
}

func UpdateId(c *mgo.Collection, i interface{}, id bson.ObjectId) error {
    return c.UpdateId(id, i)
}

func GetGenHandler(s *mgo.Session, dbName string, colName string, cns Constructor) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {

        var jdata []byte
        var err error

        ns := s.Clone()
        defer ns.Close()

        col := ns.DB(dbName).C(colName)

        switch r.Method {
        //TODO: Add ability to queary specifics
        case "GET":
            i := cns.Slice()
            GetAll(col, i)
            jdata, err = json.Marshal(i)

            w.Header().Add("Content-Type", "application/json")
            fmt.Fprintf(w, "%s", jdata)

        case "POST":
            var lastId string

            i := cns.Single()
            if err = r.ParseForm(); err != nil {
                http.Error(w, "Unable to parse form", http.StatusBadRequest)
                Log.Errorf("Parsing form : %s", err)
            }

            jString := []byte(r.PostForm.Get("json"))
            if err = json.Unmarshal(jString, i); err != nil {
                http.Error(w, "Unable to unmarshal data", http.StatusBadRequest)
                Log.Errorf("UnMarshal error : %s", err)
                return
            }

            if lastId, err = Insert(col, i); err != nil {
                http.Error(w, "Unable to unmarshal data", http.StatusInternalServerError)
                Log.Errorf("Insert Error : %#v", err)
                return
            }

            if jdata, err = json.Marshal(i); err != nil {
                http.Error(w, "Marshal error", http.StatusInternalServerError)
            }
            w.Header().Add("Location", fmt.Sprintf("%s/%s", r.URL, lastId))
            w.WriteHeader(http.StatusCreated)
        }
        return
    }
}

func GetIdHandler(s *mgo.Session, dbName string, colName string, cns Constructor) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {

        var jdata []byte
        var err error
        var ids string

        ns := s.Clone()
        defer ns.Close()

        vars := mux.Vars(r)
        ids = vars["id"]

        col := ns.DB(dbName).C(colName)

        if !bson.IsObjectIdHex(ids) {
            http.Error(w, "Provided ID is unknown", http.StatusNotFound)
            return
        }

        id := bson.ObjectIdHex(ids)
        i := cns.Single()

        if err = GetId(col, i, id); err != nil {
            http.Error(w, "Provided ID is unknown", http.StatusNotFound)
            return
        }

        if jdata, err = json.Marshal(i); err != nil {
            http.Error(w, "", http.StatusBadRequest)
        }

        switch r.Method {
        case "GET":
            w.Header().Add("Content-Type", "application/json")
            fmt.Fprintf(w, "%s", jdata)

        case "PUT":
            if r.ParseForm(); err != nil {
                http.Error(w, "", http.StatusBadRequest)
            }

            if err = json.Unmarshal([]byte(r.PostForm.Get("json")), i); err != nil {
                http.Error(w, "", http.StatusBadRequest)
                Log.Errorf("UnMarshal error : %s", err)
            }

            if err = UpdateId(col, i, id); err != nil {
                http.Error(w, "Failed to update provided ID", http.StatusInternalServerError)
                Log.Errorf("UnMarshal error : %s", err)
            }

        case "DELETE":
            if err = RemoveId(col, id); err != nil {
                http.Error(w, "Failed to remove provided ID", http.StatusInternalServerError)
                Log.Errorf("Failed to remove id %s; error : %s", id, err)
            }
        }
        return
    }
}
<commit_msg>Fixed labeling<commit_after>package restless

import (
    "encoding/json"
    "fmt"
    "github.com/gorilla/mux"
    "labix.org/v2/mgo"
    "labix.org/v2/mgo/bson"
    "net/http"
)

func GetAll(c *mgo.Collection, ip interface{}) {
    c.Find(nil).All(ip)
}

func Insert(c *mgo.Collection, i interface{}) (string, error) {
    info, err := c.Upsert(bson.M{"_id": nil}, i)
    id := info.UpsertedId.(bson.ObjectId)
    return id.Hex(), err
}

func RemoveId(c *mgo.Collection, id bson.ObjectId) error {
    return c.RemoveId(id)
}

func GetId(c *mgo.Collection, i interface{}, id bson.ObjectId) error {
    return c.FindId(id).One(i)
}

func UpdateId(c *mgo.Collection, i interface{}, id bson.ObjectId) error {
    return c.UpdateId(id, i)
}

func GetGenHandler(s *mgo.Session, dbName string, colName string, cns Constructor) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {

        var jdata []byte
        var err error

        ns := s.Clone()
        defer ns.Close()

        col := ns.DB(dbName).C(colName)

        switch r.Method {
        //TODO: Add ability to queary specifics
        case "GET":
            i := cns.Slice()
            GetAll(col, i)
            jdata, err = json.Marshal(i)

            w.Header().Add("Content-Type", "application/json")
            fmt.Fprintf(w, "[]%s=%s", colName, jdata)

        case "POST":
            var lastId string

            i := cns.Single()
            if err = r.ParseForm(); err != nil {
                http.Error(w, "Unable to parse form", http.StatusBadRequest)
                Log.Errorf("Parsing form : %s", err)
            }

            jString := []byte(r.PostForm.Get("json"))
            if err = json.Unmarshal(jString, i); err != nil {
                http.Error(w, "Unable to unmarshal data", http.StatusBadRequest)
                Log.Errorf("UnMarshal error : %s", err)
                return
            }

            if lastId, err = Insert(col, i); err != nil {
                http.Error(w, "Unable to unmarshal data", http.StatusInternalServerError)
                Log.Errorf("Insert Error : %#v", err)
                return
            }

            if jdata, err = json.Marshal(i); err != nil {
                http.Error(w, "Marshal error", http.StatusInternalServerError)
            }
            w.Header().Add("Location", fmt.Sprintf("%s/%s", r.URL, lastId))
            w.WriteHeader(http.StatusCreated)
        }
        return
    }
}

func GetIdHandler(s *mgo.Session, dbName string, colName string, cns Constructor) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {

        var jdata []byte
        var err error
        var ids string

        ns := s.Clone()
        defer ns.Close()

        vars := mux.Vars(r)
        ids = vars["id"]

        col := ns.DB(dbName).C(colName)

        if !bson.IsObjectIdHex(ids) {
            http.Error(w, "Provided ID is unknown", http.StatusNotFound)
            return
        }

        id := bson.ObjectIdHex(ids)
        i := cns.Single()

        if err = GetId(col, i, id); err != nil {
            http.Error(w, "Provided ID is unknown", http.StatusNotFound)
            return
        }

        if jdata, err = json.Marshal(i); err != nil {
            http.Error(w, "", http.StatusBadRequest)
        }

        switch r.Method {
        case "GET":
            w.Header().Add("Content-Type", "application/json")
            fmt.Fprintf(w, "%s=%s", colName, jdata)

        case "PUT":
            if r.ParseForm(); err != nil {
                http.Error(w, "", http.StatusBadRequest)
            }

            if err = json.Unmarshal([]byte(r.PostForm.Get("json")), i); err != nil {
                http.Error(w, "", http.StatusBadRequest)
                Log.Errorf("UnMarshal error : %s", err)
            }

            if err = UpdateId(col, i, id); err != nil {
                http.Error(w, "Failed to update provided ID", http.StatusInternalServerError)
                Log.Errorf("UnMarshal error : %s", err)
            }

        case "DELETE":
            if err = RemoveId(col, id); err != nil {
                http.Error(w, "Failed to remove provided ID", http.StatusInternalServerError)
                Log.Errorf("Failed to remove id %s; error : %s", id, err)
            }
        }
        return
    }
}
<|endoftext|>


<commit_before>package gosql

import (
	"database/sql"
	"log"
	"os"
	"testing"

	. "github.com/smartystreets/goconvey/convey"

	_ "github.com/lib/pq"
)

func testDb(t *testing.T) (db *sql.DB) {
	var e error
	e = func() error {
		db, e = sql.Open("postgres", "postgres://tobias@localhost/gosql_test?sslmode=disable")
		if e != nil {
			t.Fatal(e)
		}
		return db.Ping()
	}()

	if e != nil {
		t.Fatal(e)
	}
	return db

}

func TestMigrations(t *testing.T) {
	Convey("Migrations", t, func() {
		db := testDb(t)
		tx, e := db.Begin()
		So(e, ShouldBeNil)
		defer tx.Rollback()
		m := NewMigrator("CREATE TABLE users (id SERIAL PRIMARY KEY)", "SELECT 2")
		m.Logger = log.New(os.Stdout, "", 0)
		So(m, ShouldNotBeNil)

		migs, e := m.migrations(tx)
		So(e, ShouldBeNil)
		So(len(migs), ShouldEqual, 2)

		So(migs[0].Statement, ShouldEqual, "CREATE TABLE users (id SERIAL PRIMARY KEY)")
		So(migs[0].Idx, ShouldEqual, 1)
		So(migs[0].Executed, ShouldEqual, false)

		So(migs[1].Statement, ShouldEqual, "SELECT 2")
		So(migs[1].Idx, ShouldEqual, 2)
		So(migs[1].Executed, ShouldEqual, false)

		cnt, e := migs.ExecuteUntil(tx, 1)
		So(e, ShouldBeNil)
		So(cnt, ShouldEqual, 1)

		cnt, e = migs.ExecuteUntil(tx, 1)
		So(e, ShouldBeNil)
		So(cnt, ShouldEqual, 0)

		migs, e = m.migrations(tx)
		So(len(migs), ShouldEqual, 2)
		So(migs[0].Executed, ShouldEqual, true)
		So(migs[1].Executed, ShouldEqual, false)

		cnt, e = migs.Execute(tx)
		So(e, ShouldBeNil)
		So(cnt, ShouldEqual, 1)
		So(migs[1].Executed, ShouldEqual, true)

		cnt, e = migs.Execute(tx)
		So(e, ShouldBeNil)
		So(cnt, ShouldEqual, 0)

		_, e = tx.Exec("INSERT INTO users (id) VALUES ($1)", 77)
		So(e, ShouldBeNil)

		cnt = 0
		e = tx.QueryRow("SELECT id FROM users LIMIT 1").Scan(&cnt)
		So(e, ShouldBeNil)
		So(cnt, ShouldEqual, 77)
	})
}
<commit_msg>move test db url into constant<commit_after>package gosql

import (
	"database/sql"
	"log"
	"os"
	"testing"

	. "github.com/smartystreets/goconvey/convey"

	_ "github.com/lib/pq"
)

const dbUrl = "postgres://tobias@localhost/gosql_test?sslmode=disable"

func testDb(t *testing.T) (db *sql.DB) {
	var e error
	e = func() error {
		db, e = sql.Open("postgres", dbUrl)
		if e != nil {
			t.Fatal(e)
		}
		return db.Ping()
	}()

	if e != nil {
		t.Fatal(e)
	}
	return db

}

func TestMigrations(t *testing.T) {
	Convey("Migrations", t, func() {
		db := testDb(t)
		tx, e := db.Begin()
		So(e, ShouldBeNil)
		defer tx.Rollback()
		m := NewMigrator("CREATE TABLE users (id SERIAL PRIMARY KEY)", "SELECT 2")
		m.Logger = log.New(os.Stdout, "", 0)
		So(m, ShouldNotBeNil)

		migs, e := m.migrations(tx)
		So(e, ShouldBeNil)
		So(len(migs), ShouldEqual, 2)

		So(migs[0].Statement, ShouldEqual, "CREATE TABLE users (id SERIAL PRIMARY KEY)")
		So(migs[0].Idx, ShouldEqual, 1)
		So(migs[0].Executed, ShouldEqual, false)

		So(migs[1].Statement, ShouldEqual, "SELECT 2")
		So(migs[1].Idx, ShouldEqual, 2)
		So(migs[1].Executed, ShouldEqual, false)

		cnt, e := migs.ExecuteUntil(tx, 1)
		So(e, ShouldBeNil)
		So(cnt, ShouldEqual, 1)

		cnt, e = migs.ExecuteUntil(tx, 1)
		So(e, ShouldBeNil)
		So(cnt, ShouldEqual, 0)

		migs, e = m.migrations(tx)
		So(len(migs), ShouldEqual, 2)
		So(migs[0].Executed, ShouldEqual, true)
		So(migs[1].Executed, ShouldEqual, false)

		cnt, e = migs.Execute(tx)
		So(e, ShouldBeNil)
		So(cnt, ShouldEqual, 1)
		So(migs[1].Executed, ShouldEqual, true)

		cnt, e = migs.Execute(tx)
		So(e, ShouldBeNil)
		So(cnt, ShouldEqual, 0)

		_, e = tx.Exec("INSERT INTO users (id) VALUES ($1)", 77)
		So(e, ShouldBeNil)

		cnt = 0
		e = tx.QueryRow("SELECT id FROM users LIMIT 1").Scan(&cnt)
		So(e, ShouldBeNil)
		So(cnt, ShouldEqual, 77)
	})
}
<|endoftext|>


<commit_before>package metadata

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/rancher/external-dns/config"
	"github.com/rancher/external-dns/utils"
	"github.com/rancher/go-rancher-metadata/metadata"
	"time"
)

const (
	metadataUrl = "http://rancher-metadata/2015-12-19"
)

type MetadataClient struct {
	MetadataClient  metadata.Client
	EnvironmentName string
	EnvironmentUUID string
}

func getEnvironment(m metadata.Client) (string, string, error) {
	timeout := 30 * time.Second
	var err error
	var stack metadata.Stack
	for i := 1 * time.Second; i < timeout; i *= time.Duration(2) {
		stack, err = m.GetSelfStack()
		if err != nil {
			logrus.Errorf("Error reading stack info: %v...will retry", err)
			time.Sleep(i)
		} else {
			return stack.EnvironmentName, stack.EnvironmentUUID, nil
		}
	}
	return "", "", fmt.Errorf("Error reading stack info: %v", err)
}

func NewMetadataClient() (*MetadataClient, error) {
	m, err := metadata.NewClientAndWait(metadataUrl)
	if err != nil {
		logrus.Fatalf("Failed to configure rancher-metadata: %v", err)
	}

	envName, envUUID, err := getEnvironment(m)
	if err != nil {
		logrus.Fatalf("Error reading stack info: %v", err)
	}

	return &MetadataClient{
		MetadataClient:  m,
		EnvironmentName: envName,
		EnvironmentUUID: envUUID,
	}, nil
}

func (m *MetadataClient) GetVersion() (string, error) {
	return m.MetadataClient.GetVersion()
}

func (m *MetadataClient) GetMetadataDnsRecords() (map[string]utils.DnsRecord, error) {
	dnsEntries := make(map[string]utils.DnsRecord)
	err := m.getContainersDnsRecords(dnsEntries)
	if err != nil {
		return dnsEntries, err
	}
	return dnsEntries, nil
}

func (m *MetadataClient) getContainersDnsRecords(dnsEntries map[string]utils.DnsRecord) error {
	services, err := m.MetadataClient.GetServices()
	if err != nil {
		return err
	}

	ourFqdns := make(map[string]struct{})
	hostMeta := make(map[string]metadata.Host)
	for _, service := range services {

		// Check for Service Label: io.rancher.service.external_dns
		// Accepts 'always', 'auto' (default), or 'never'
		policy, ok := service.Labels["io.rancher.service.external_dns"]
		if !ok {
			policy = "auto"
		} else if policy == "never" {
			logrus.Debugf("Service %v is Disabled", service.Name)
			continue
		}

		if service.Kind != "service" && service.Kind != "loadBalancerService" {
			continue
		}

		for _, container := range service.Containers {

			if (len(container.Ports) == 0 && policy != "always") || !containerStateOK(container) {
				continue
			}

			hostUUID := container.HostUUID
			if len(hostUUID) == 0 {
				logrus.Debugf("Container's %v host_uuid is empty", container.Name)
				continue
			}

			var host metadata.Host
			if _, ok := hostMeta[hostUUID]; ok {
				host = hostMeta[hostUUID]
			} else {
				host, err = m.MetadataClient.GetHost(hostUUID)
				if err != nil {
					logrus.Warnf("Failed to get host metadata: %v", err)
					continue
				}
				hostMeta[hostUUID] = host
			}

			// Check for Host Label: io.rancher.host.external_dns
			// Accepts 'true' (default) or 'false'
			if label, ok := host.Labels["io.rancher.host.external_dns"]; ok {
				if label == "false" {
					logrus.Debugf("Container %v Host %s is Disabled", container.Name, host.Name)
					continue
				}
			}

			ip, ok := host.Labels["io.rancher.host.external_dns_ip"]
			if !ok || ip == "" {
				ip = host.AgentIP
			}

			fqdn := utils.FqdnFromTemplate(config.NameTemplate, container.ServiceName, container.StackName,
				m.EnvironmentName, config.RootDomainName)

			addToDnsEntries(fqdn, ip, dnsEntries)
			ourFqdns[fqdn] = struct{}{}
		}
	}

	if len(ourFqdns) > 0 {
		stateFqdn := utils.StateFqdn(m.EnvironmentUUID, config.RootDomainName)
		stateRec := utils.StateRecord(stateFqdn, config.TTL, ourFqdns)
		dnsEntries[stateFqdn] = stateRec
	}

	return nil
}

func addToDnsEntries(fqdn, ip string, dnsEntries map[string]utils.DnsRecord) {
	var records []string
	if _, ok := dnsEntries[fqdn]; !ok {
		records = []string{ip}
	} else {
		records = dnsEntries[fqdn].Records
		// skip if the records already have that IP
		for _, val := range records {
			if val == ip {
				return
			}
		}
		records = append(records, ip)
	}

	dnsEntries[fqdn] = utils.DnsRecord{fqdn, records, "A", config.TTL}
}

func containerStateOK(container metadata.Container) bool {
	switch container.State {
	case "running":
	default:
		return false
	}

	switch container.HealthState {
	case "healthy":
	case "updating-healthy":
	case "":
	default:
		return false
	}

	return true
}
<commit_msg>Adds support for multi-IP per host allocation<commit_after>package metadata

import (
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/rancher/external-dns/config"
	"github.com/rancher/external-dns/utils"
	"github.com/rancher/go-rancher-metadata/metadata"
)

const (
	metadataUrl = "http://rancher-metadata/2015-12-19"
)

type MetadataClient struct {
	MetadataClient  metadata.Client
	EnvironmentName string
	EnvironmentUUID string
}

func getEnvironment(m metadata.Client) (string, string, error) {
	timeout := 30 * time.Second
	var err error
	var stack metadata.Stack
	for i := 1 * time.Second; i < timeout; i *= time.Duration(2) {
		stack, err = m.GetSelfStack()
		if err != nil {
			logrus.Errorf("Error reading stack info: %v...will retry", err)
			time.Sleep(i)
		} else {
			return stack.EnvironmentName, stack.EnvironmentUUID, nil
		}
	}
	return "", "", fmt.Errorf("Error reading stack info: %v", err)
}

func NewMetadataClient() (*MetadataClient, error) {
	m, err := metadata.NewClientAndWait(metadataUrl)
	if err != nil {
		logrus.Fatalf("Failed to configure rancher-metadata: %v", err)
	}

	envName, envUUID, err := getEnvironment(m)
	if err != nil {
		logrus.Fatalf("Error reading stack info: %v", err)
	}

	return &MetadataClient{
		MetadataClient:  m,
		EnvironmentName: envName,
		EnvironmentUUID: envUUID,
	}, nil
}

func (m *MetadataClient) GetVersion() (string, error) {
	return m.MetadataClient.GetVersion()
}

func (m *MetadataClient) GetMetadataDnsRecords() (map[string]utils.DnsRecord, error) {
	dnsEntries := make(map[string]utils.DnsRecord)
	err := m.getContainersDnsRecords(dnsEntries)
	if err != nil {
		return dnsEntries, err
	}
	return dnsEntries, nil
}

func (m *MetadataClient) getContainersDnsRecords(dnsEntries map[string]utils.DnsRecord) error {
	services, err := m.MetadataClient.GetServices()
	if err != nil {
		return err
	}

	ourFqdns := make(map[string]struct{})
	hostMeta := make(map[string]metadata.Host)
	for _, service := range services {

		// Check for Service Label: io.rancher.service.external_dns
		// Accepts 'always', 'auto' (default), or 'never'
		policy, ok := service.Labels["io.rancher.service.external_dns"]
		if !ok {
			policy = "auto"
		} else if policy == "never" {
			logrus.Debugf("Service %v is Disabled", service.Name)
			continue
		}

		if service.Kind != "service" && service.Kind != "loadBalancerService" {
			continue
		}

		for _, container := range service.Containers {

			if (len(container.Ports) == 0 && policy != "always") || !containerStateOK(container) {
				continue
			}

			hostUUID := container.HostUUID
			if len(hostUUID) == 0 {
				logrus.Debugf("Container's %v host_uuid is empty", container.Name)
				continue
			}

			var host metadata.Host
			if _, ok := hostMeta[hostUUID]; ok {
				host = hostMeta[hostUUID]
			} else {
				host, err = m.MetadataClient.GetHost(hostUUID)
				if err != nil {
					logrus.Warnf("Failed to get host metadata: %v", err)
					continue
				}
				hostMeta[hostUUID] = host
			}

			// Check for Host Label: io.rancher.host.external_dns
			// Accepts 'true' (default) or 'false'
			if label, ok := host.Labels["io.rancher.host.external_dns"]; ok {
				if label == "false" {
					logrus.Debugf("Container %v Host %s is Disabled", container.Name, host.Name)
					continue
				}
			}

			var externalIP string
			if ip, ok := host.Labels["io.rancher.host.external_dns_ip"]; ok && len(ip) > 0 {
				externalIP = ip
			} else if len(container.Ports) > 0 {
				if ip, ok := parsePortToIP(container.Ports[0]); ok {
					externalIP = ip
				}
			}

			// fallback to host agent IP
			if len(externalIP) == 0 {
				logrus.Debugf("Fallback to host.AgentIP %s for container %s", host.AgentIP, container.Name)
				externalIP = host.AgentIP
			}

			if net.ParseIP(externalIP) == nil {
				logrus.Errorf("Skipping container %s: Invalid IP address %s", container.Name, externalIP)
			}

			fqdn := utils.FqdnFromTemplate(config.NameTemplate, container.ServiceName, container.StackName,
				m.EnvironmentName, config.RootDomainName)

			addToDnsEntries(fqdn, externalIP, dnsEntries)
			ourFqdns[fqdn] = struct{}{}
		}
	}

	if len(ourFqdns) > 0 {
		stateFqdn := utils.StateFqdn(m.EnvironmentUUID, config.RootDomainName)
		stateRec := utils.StateRecord(stateFqdn, config.TTL, ourFqdns)
		dnsEntries[stateFqdn] = stateRec
	}

	return nil
}

func addToDnsEntries(fqdn, ip string, dnsEntries map[string]utils.DnsRecord) {
	var records []string
	if _, ok := dnsEntries[fqdn]; !ok {
		records = []string{ip}
	} else {
		records = dnsEntries[fqdn].Records
		// skip if the records already have that IP
		for _, val := range records {
			if val == ip {
				return
			}
		}
		records = append(records, ip)
	}

	dnsEntries[fqdn] = utils.DnsRecord{fqdn, records, "A", config.TTL}
}

func containerStateOK(container metadata.Container) bool {
	switch container.State {
	case "running":
	default:
		return false
	}

	switch container.HealthState {
	case "healthy":
	case "updating-healthy":
	case "":
	default:
		return false
	}

	return true
}

// expects port string as 'ip:publicPort:privatePort'
// returns usable ip address
func parsePortToIP(port string) (string, bool) {
	parts := strings.Split(port, ":")
	if len(parts) == 3 {
		ip := parts[0]
		if len(ip) > 0 && ip != "0.0.0.0" {
			return ip, true
		}
	}

	return "", false
}
<|endoftext|>


<commit_before>package parse

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
)

type Tree struct {
	Roots []Node

	// Config
	includeNonSemantic bool

	// Parser state
	tok       token // single-item lookahead
	peekCount int
	lex       *lexer
	inLambda  bool
}

// String pretty-prints the tree recursively using each Node's String().
func (t *Tree) String() string { return nodesToString(t.Roots, 0) }

func nodesToString(nodes []Node, depth int) string {
	var buf bytes.Buffer
	for _, node := range nodes {
		buf.WriteString(strings.Repeat("  ", depth))
		buf.WriteString(node.String())
		buf.WriteString("\n")
		buf.WriteString(nodesToString(node.Children(), depth+1))
	}
	return buf.String()
}

func (t *Tree) Parse() (err error) {
	defer t.recover(&err)
	for {
		node := t.parse()
		if node == nil {
			break
		}
		if t.includeNonSemantic || isSemantic(node) {
			t.Roots = append(t.Roots, node)
		}
	}
	return nil
}

type lexError struct{ err error }
type parseError struct{ err error }

func (t *Tree) recover(err *error) {
	if e := recover(); e != nil {
		switch e := e.(type) {
		case lexError:
			*err = e.err
		case parseError:
			*err = e.err
		default:
			panic(e)
		}
	}
}

func (t *Tree) nextToken() token {
	tok := t.lex.nextToken()
	if tok.typ == tokError {
		panic(lexError{tok.AsError()})
	}
	return tok
}

func (t *Tree) next() token {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.tok = t.nextToken()
	}
	return t.tok
}

func (t *Tree) backup() {
	t.peekCount++
	if t.peekCount > 1 {
		panic("backup() called twice consecutively")
	}
}

func (t *Tree) peek() token {
	if t.peekCount == 0 {
		t.peekCount++
		t.tok = t.nextToken()
	}
	return t.tok
}

func (t *Tree) errorf(pos *Pos, format string, args ...interface{}) {
	panic(parseError{pos.FormatError("parse", fmt.Sprintf(format, args...))})
}

func (t *Tree) unexpected(tok token) { t.errorf(tok.pos, "unexpected token %q", tok.val) }

func (t *Tree) unexpectedEOF(tok token) { t.errorf(tok.pos, "unexpected EOF") }

// ParseOpts is a bitset of parsing options for Reader and File.
type ParseOpts uint

const (
	// IncludeNonSemantic makes the parser include non-semantic nodes:
	// CommentNodes and NewlineNodes.
	IncludeNonSemantic ParseOpts = 1 << iota
)

func Reader(r io.Reader, filename string, opts ParseOpts) (*Tree, error) {
	t := &Tree{
		includeNonSemantic: opts&IncludeNonSemantic != 0,
		lex:                lex(filename, bufio.NewReader(r)),
	}
	if err := t.Parse(); err != nil {
		return nil, err
	}
	return t, nil
}

func File(filename string, opts ParseOpts) (*Tree, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return Reader(f, filename, opts)
}

// parse parses the next top-level item from the token stream.
// It returns nil if there are no non-EOF tokens left in the stream.
func (t *Tree) parse() Node {
	for {
		switch tok := t.next(); tok.typ {
		case tokSymbol:
			switch val := tok.val; val {
			case "nil":
				return &NilNode{tok.pos}
			case "true", "false":
				return &BoolNode{tok.pos, val == "true"}
			default:
				return &SymbolNode{tok.pos, tok.val}
			}
		case tokCharLiteral:
			return t.parseCharLiteral(tok)
		case tokComment:
			return &CommentNode{tok.pos, tok.val}
		case tokAtSign:
			return &DerefNode{tok.pos, t.parse()}
		case tokKeyword:
			return &KeywordNode{tok.pos, tok.val}
		case tokLeftParen:
			return t.parseList(tok)
		case tokLeftBrace:
			return t.parseMap(tok)
		case tokCircumflex:
			return t.parseMetadata(tok)
		case tokNewline:
			return &NewlineNode{tok.pos}
		case tokNumber:
			// TODO: need to parse the number here; a number token may not be valid.
			return &NumberNode{tok.pos, tok.val}
		case tokApostrophe:
			return &QuoteNode{tok.pos, t.parse()}
		case tokString:
			return &StringNode{tok.pos, tok.val[1 : len(tok.val)-1]}
		case tokBacktick:
			return &SyntaxQuoteNode{tok.pos, t.parse()}
		case tokTilde:
			next := t.next()
			switch next.typ {
			case tokAtSign:
				return &UnquoteSpliceNode{tok.pos, t.parse()}
			case tokEOF:
				t.unexpectedEOF(next)
			}
			t.backup()
			return &UnquoteNode{tok.pos, t.parse()}
		case tokLeftBracket:
			return t.parseVector(tok)
		case tokDispatch:
			return t.parseDispatch(tok)
		case tokOctothorpe:
			return t.parseTag(tok)
		case tokEOF:
			return nil
		default:
			t.unexpected(tok)
		}
		panic("unreached")
	}
}

func (t *Tree) parseCharLiteral(tok token) Node {
	var r rune
	val := tok.val[1:]
	runes := []rune(val)
	switch len(runes) {
	case 1:
		r = runes[0]
	case 0:
		t.errorf(tok.pos, "invalid character literal")
	default:
		switch val {
		case "newline":
			r = '\n'
		case "space":
			r = ' '
		case "tab":
			r = '\t'
		case "formfeed":
			r = '\f'
		case "backspace":
			r = 0x7f
		case "return":
			r = '\r'
		default:
			switch runes[0] {
			case 'o':
				n, err := strconv.ParseInt(val[1:], 8, 32)
				if len(val) != 4 || err != nil || n < 0 {
					t.errorf(tok.pos, "invalid octal literal")
				}
				r = rune(n)
			case 'u':
				n, err := strconv.ParseInt(val[1:], 16, 32)
				if len(val) != 5 || err != nil || n < 0 {
					t.errorf(tok.pos, "invalid unicode literal")
				}
				r = rune(n)
			default:
				t.errorf(tok.pos, "invalid character literal")
			}
		}
	}
	return &CharacterNode{tok.pos, r, tok.val}
}

func (t *Tree) parseList(start token) Node {
	var nodes []Node
	for {
		switch tok := t.next(); tok.typ {
		case tokRightParen:
			return &ListNode{start.pos, nodes}
		case tokEOF:
			t.unexpectedEOF(tok)
		}
		t.backup()
		node := t.parse()
		if t.includeNonSemantic || isSemantic(node) {
			nodes = append(nodes, node)
		}
	}
}

func (t *Tree) parseMap(start token) Node {
	var nodes []Node
	for {
		switch tok := t.next(); tok.typ {
		case tokRightBrace:
			return &MapNode{start.pos, nodes}
		case tokEOF:
			t.unexpectedEOF(tok)
		}
		t.backup()
		node := t.parse()
		if t.includeNonSemantic || isSemantic(node) {
			nodes = append(nodes, node)
		}
	}
}

func (t *Tree) parseVector(start token) Node {
	var nodes []Node
	for {
		switch tok := t.next(); tok.typ {
		case tokRightBracket:
			return &VectorNode{start.pos, nodes}
		case tokEOF:
			t.unexpectedEOF(tok)
		}
		t.backup()
		node := t.parse()
		if t.includeNonSemantic || isSemantic(node) {
			nodes = append(nodes, node)
		}
	}
}

func (t *Tree) parseDispatch(tok token) Node {
	switch tok.val {
	case "#(":
		return t.parseFnLiteral(tok)
	case "#_":
		return t.parseReaderDiscard(tok)
	case "#=":
		return t.parseReaderEval(tok)
	case `#"`:
		return t.parseRegex(tok)
	case "#{":
		return t.parseSet(tok)
	case "#'":
		return t.parseVarQuote(tok)
	case "#^":
		return t.parseMetadata(tok)
	case "#!":
	case "#<":
	default:
		t.unexpected(tok)
	}
	panic("unreached")
}

func (t *Tree) parseTag(start token) Node {
	tok := t.next()
	switch tok.typ {
	case tokSymbol:
		return &TagNode{start.pos, tok.val}
	case tokEOF:
		t.unexpectedEOF(tok)
	default:
		t.unexpected(tok)
	}
	panic("not reached")
}

func (t *Tree) parseFnLiteral(start token) Node {
	if t.inLambda {
		t.errorf(start.pos, "cannot nest fn literals")
	}
	tok := t.next()
	if tok.typ != tokLeftParen {
		panic("should not happen")
	}
	t.inLambda = true
	var nodes []Node
	for {
		switch tok = t.next(); tok.typ {
		case tokRightParen:
			t.inLambda = false
			return &FnLiteralNode{start.pos, nodes}
		case tokEOF:
			t.unexpectedEOF(tok)
		}
		t.backup()
		node := t.parse()
		if t.includeNonSemantic || isSemantic(node) {
			nodes = append(nodes, node)
		}
	}
}

func (t *Tree) parseMetadata(start token) Node {
	tok := t.next()
	if tok.typ == tokEOF {
		t.unexpectedEOF(tok)
	}
	t.backup()
	return &MetadataNode{start.pos, t.parse()}
}

func (t *Tree) parseReaderDiscard(start token) Node {
	tok := t.next()
	if tok.typ == tokEOF {
		t.unexpectedEOF(tok)
	}
	t.backup()
	return &ReaderDiscardNode{start.pos, t.parse()}
}

func (t *Tree) parseReaderEval(start token) Node {
	tok := t.next()
	if tok.typ == tokEOF {
		t.unexpectedEOF(tok)
	}
	t.backup()
	return &ReaderEvalNode{start.pos, t.parse()}
}

func (t *Tree) parseRegex(start token) Node {
	tok := t.next()
	if tok.typ != tokString {
		panic("should not happen")
	}
	return &RegexNode{start.pos, tok.val[1 : len(tok.val)-1]}
}

func (t *Tree) parseSet(start token) Node {
	tok := t.next()
	if tok.typ != tokLeftBrace {
		panic("should not happen")
	}
	var nodes []Node
	for {
		switch tok := t.next(); tok.typ {
		case tokRightBrace:
			return &SetNode{start.pos, nodes}
		case tokEOF:
			t.unexpectedEOF(tok)
		}
		t.backup()
		node := t.parse()
		if t.includeNonSemantic || isSemantic(node) {
			nodes = append(nodes, node)
		}
	}
}

func (t *Tree) parseVarQuote(start token) Node {
	switch tok := t.next(); tok.typ {
	case tokSymbol:
		return &VarQuoteNode{start.pos, tok.val}
	case tokEOF:
		t.unexpectedEOF(tok)
	default:
		t.unexpected(tok)
	}
	panic("unreached")
}
<commit_msg>[parse] Unexport (*Tree).Parse<commit_after>package parse

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
)

type Tree struct {
	Roots []Node

	// Config
	includeNonSemantic bool

	// Parser state
	tok       token // single-item lookahead
	peekCount int
	lex       *lexer
	inLambda  bool
}

// String pretty-prints the tree recursively using each Node's String().
func (t *Tree) String() string { return nodesToString(t.Roots, 0) }

func nodesToString(nodes []Node, depth int) string {
	var buf bytes.Buffer
	for _, node := range nodes {
		buf.WriteString(strings.Repeat("  ", depth))
		buf.WriteString(node.String())
		buf.WriteString("\n")
		buf.WriteString(nodesToString(node.Children(), depth+1))
	}
	return buf.String()
}

func (t *Tree) parse() (err error) {
	defer t.recover(&err)
	for {
		node := t.parseNext()
		if node == nil {
			break
		}
		if t.includeNonSemantic || isSemantic(node) {
			t.Roots = append(t.Roots, node)
		}
	}
	return nil
}

type lexError struct{ err error }
type parseError struct{ err error }

func (t *Tree) recover(err *error) {
	if e := recover(); e != nil {
		switch e := e.(type) {
		case lexError:
			*err = e.err
		case parseError:
			*err = e.err
		default:
			panic(e)
		}
	}
}

func (t *Tree) nextToken() token {
	tok := t.lex.nextToken()
	if tok.typ == tokError {
		panic(lexError{tok.AsError()})
	}
	return tok
}

func (t *Tree) next() token {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.tok = t.nextToken()
	}
	return t.tok
}

func (t *Tree) backup() {
	t.peekCount++
	if t.peekCount > 1 {
		panic("backup() called twice consecutively")
	}
}

func (t *Tree) peek() token {
	if t.peekCount == 0 {
		t.peekCount++
		t.tok = t.nextToken()
	}
	return t.tok
}

func (t *Tree) errorf(pos *Pos, format string, args ...interface{}) {
	panic(parseError{pos.FormatError("parse", fmt.Sprintf(format, args...))})
}

func (t *Tree) unexpected(tok token) { t.errorf(tok.pos, "unexpected token %q", tok.val) }

func (t *Tree) unexpectedEOF(tok token) { t.errorf(tok.pos, "unexpected EOF") }

// ParseOpts is a bitset of parsing options for Reader and File.
type ParseOpts uint

const (
	// IncludeNonSemantic makes the parser include non-semantic nodes:
	// CommentNodes and NewlineNodes.
	IncludeNonSemantic ParseOpts = 1 << iota
)

func Reader(r io.Reader, filename string, opts ParseOpts) (*Tree, error) {
	t := &Tree{
		includeNonSemantic: opts&IncludeNonSemantic != 0,
		lex:                lex(filename, bufio.NewReader(r)),
	}
	if err := t.parse(); err != nil {
		return nil, err
	}
	return t, nil
}

func File(filename string, opts ParseOpts) (*Tree, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return Reader(f, filename, opts)
}

// parseNext parses the next top-level item from the token stream.
// It returns nil if there are no non-EOF tokens left in the stream.
func (t *Tree) parseNext() Node {
	for {
		switch tok := t.next(); tok.typ {
		case tokSymbol:
			switch val := tok.val; val {
			case "nil":
				return &NilNode{tok.pos}
			case "true", "false":
				return &BoolNode{tok.pos, val == "true"}
			default:
				return &SymbolNode{tok.pos, tok.val}
			}
		case tokCharLiteral:
			return t.parseCharLiteral(tok)
		case tokComment:
			return &CommentNode{tok.pos, tok.val}
		case tokAtSign:
			return &DerefNode{tok.pos, t.parseNext()}
		case tokKeyword:
			return &KeywordNode{tok.pos, tok.val}
		case tokLeftParen:
			return t.parseList(tok)
		case tokLeftBrace:
			return t.parseMap(tok)
		case tokCircumflex:
			return t.parseMetadata(tok)
		case tokNewline:
			return &NewlineNode{tok.pos}
		case tokNumber:
			// TODO: need to parse the number here; a number token may not be valid.
			return &NumberNode{tok.pos, tok.val}
		case tokApostrophe:
			return &QuoteNode{tok.pos, t.parseNext()}
		case tokString:
			return &StringNode{tok.pos, tok.val[1 : len(tok.val)-1]}
		case tokBacktick:
			return &SyntaxQuoteNode{tok.pos, t.parseNext()}
		case tokTilde:
			next := t.next()
			switch next.typ {
			case tokAtSign:
				return &UnquoteSpliceNode{tok.pos, t.parseNext()}
			case tokEOF:
				t.unexpectedEOF(next)
			}
			t.backup()
			return &UnquoteNode{tok.pos, t.parseNext()}
		case tokLeftBracket:
			return t.parseVector(tok)
		case tokDispatch:
			return t.parseDispatch(tok)
		case tokOctothorpe:
			return t.parseTag(tok)
		case tokEOF:
			return nil
		default:
			t.unexpected(tok)
		}
		panic("unreached")
	}
}

func (t *Tree) parseCharLiteral(tok token) Node {
	var r rune
	val := tok.val[1:]
	runes := []rune(val)
	switch len(runes) {
	case 1:
		r = runes[0]
	case 0:
		t.errorf(tok.pos, "invalid character literal")
	default:
		switch val {
		case "newline":
			r = '\n'
		case "space":
			r = ' '
		case "tab":
			r = '\t'
		case "formfeed":
			r = '\f'
		case "backspace":
			r = 0x7f
		case "return":
			r = '\r'
		default:
			switch runes[0] {
			case 'o':
				n, err := strconv.ParseInt(val[1:], 8, 32)
				if len(val) != 4 || err != nil || n < 0 {
					t.errorf(tok.pos, "invalid octal literal")
				}
				r = rune(n)
			case 'u':
				n, err := strconv.ParseInt(val[1:], 16, 32)
				if len(val) != 5 || err != nil || n < 0 {
					t.errorf(tok.pos, "invalid unicode literal")
				}
				r = rune(n)
			default:
				t.errorf(tok.pos, "invalid character literal")
			}
		}
	}
	return &CharacterNode{tok.pos, r, tok.val}
}

func (t *Tree) parseList(start token) Node {
	var nodes []Node
	for {
		switch tok := t.next(); tok.typ {
		case tokRightParen:
			return &ListNode{start.pos, nodes}
		case tokEOF:
			t.unexpectedEOF(tok)
		}
		t.backup()
		node := t.parseNext()
		if t.includeNonSemantic || isSemantic(node) {
			nodes = append(nodes, node)
		}
	}
}

func (t *Tree) parseMap(start token) Node {
	var nodes []Node
	for {
		switch tok := t.next(); tok.typ {
		case tokRightBrace:
			return &MapNode{start.pos, nodes}
		case tokEOF:
			t.unexpectedEOF(tok)
		}
		t.backup()
		node := t.parseNext()
		if t.includeNonSemantic || isSemantic(node) {
			nodes = append(nodes, node)
		}
	}
}

func (t *Tree) parseVector(start token) Node {
	var nodes []Node
	for {
		switch tok := t.next(); tok.typ {
		case tokRightBracket:
			return &VectorNode{start.pos, nodes}
		case tokEOF:
			t.unexpectedEOF(tok)
		}
		t.backup()
		node := t.parseNext()
		if t.includeNonSemantic || isSemantic(node) {
			nodes = append(nodes, node)
		}
	}
}

func (t *Tree) parseDispatch(tok token) Node {
	switch tok.val {
	case "#(":
		return t.parseFnLiteral(tok)
	case "#_":
		return t.parseReaderDiscard(tok)
	case "#=":
		return t.parseReaderEval(tok)
	case `#"`:
		return t.parseRegex(tok)
	case "#{":
		return t.parseSet(tok)
	case "#'":
		return t.parseVarQuote(tok)
	case "#^":
		return t.parseMetadata(tok)
	case "#!":
	case "#<":
	default:
		t.unexpected(tok)
	}
	panic("unreached")
}

func (t *Tree) parseTag(start token) Node {
	tok := t.next()
	switch tok.typ {
	case tokSymbol:
		return &TagNode{start.pos, tok.val}
	case tokEOF:
		t.unexpectedEOF(tok)
	default:
		t.unexpected(tok)
	}
	panic("not reached")
}

func (t *Tree) parseFnLiteral(start token) Node {
	if t.inLambda {
		t.errorf(start.pos, "cannot nest fn literals")
	}
	tok := t.next()
	if tok.typ != tokLeftParen {
		panic("should not happen")
	}
	t.inLambda = true
	var nodes []Node
	for {
		switch tok = t.next(); tok.typ {
		case tokRightParen:
			t.inLambda = false
			return &FnLiteralNode{start.pos, nodes}
		case tokEOF:
			t.unexpectedEOF(tok)
		}
		t.backup()
		node := t.parseNext()
		if t.includeNonSemantic || isSemantic(node) {
			nodes = append(nodes, node)
		}
	}
}

func (t *Tree) parseMetadata(start token) Node {
	tok := t.next()
	if tok.typ == tokEOF {
		t.unexpectedEOF(tok)
	}
	t.backup()
	return &MetadataNode{start.pos, t.parseNext()}
}

func (t *Tree) parseReaderDiscard(start token) Node {
	tok := t.next()
	if tok.typ == tokEOF {
		t.unexpectedEOF(tok)
	}
	t.backup()
	return &ReaderDiscardNode{start.pos, t.parseNext()}
}

func (t *Tree) parseReaderEval(start token) Node {
	tok := t.next()
	if tok.typ == tokEOF {
		t.unexpectedEOF(tok)
	}
	t.backup()
	return &ReaderEvalNode{start.pos, t.parseNext()}
}

func (t *Tree) parseRegex(start token) Node {
	tok := t.next()
	if tok.typ != tokString {
		panic("should not happen")
	}
	return &RegexNode{start.pos, tok.val[1 : len(tok.val)-1]}
}

func (t *Tree) parseSet(start token) Node {
	tok := t.next()
	if tok.typ != tokLeftBrace {
		panic("should not happen")
	}
	var nodes []Node
	for {
		switch tok := t.next(); tok.typ {
		case tokRightBrace:
			return &SetNode{start.pos, nodes}
		case tokEOF:
			t.unexpectedEOF(tok)
		}
		t.backup()
		node := t.parseNext()
		if t.includeNonSemantic || isSemantic(node) {
			nodes = append(nodes, node)
		}
	}
}

func (t *Tree) parseVarQuote(start token) Node {
	switch tok := t.next(); tok.typ {
	case tokSymbol:
		return &VarQuoteNode{start.pos, tok.val}
	case tokEOF:
		t.unexpectedEOF(tok)
	default:
		t.unexpected(tok)
	}
	panic("unreached")
}
<|endoftext|>