<commit_before>// +build go1.9

package prometheus

import (
	"fmt"
	"log"
	"math"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/armon/go-metrics"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/push"
)

var (
	// DefaultPrometheusOpts is the default set of options used when creating a
	// PrometheusSink.
	DefaultPrometheusOpts = PrometheusOpts{
		Expiration: 60 * time.Second,
	}
)

// PrometheusOpts is used to configure the Prometheus Sink
type PrometheusOpts struct {
	// Expiration is the duration a metric is valid for, after which it will be
	// untracked. If the value is zero, a metric is never expired.
	Expiration time.Duration
	Registerer prometheus.Registerer

	// Gauges, Summaries, and Counters allow us to pre-declare metrics by giving their Name and ConstLabels to the
	// PrometheusSink when it is created. Metrics declared in this way will be initialized at zero and will not be
	// deleted when their expiry is reached.
	// - Gauges and Summaries will be set to NaN when they expire.
	// - Counters continue to Collect their last known value.
	// Ex:
	// PrometheusOpts{
	//     Expiration: 10 * time.Second,
	//     Gauges: []PrometheusGauge{
	//         {
	//	         Name: []string{ "application", "component", "measurement"},
	//           ConstLabels: []metrics.Label{ { Name: "datacenter", Value: "dc1" }, },
	//         },
	//     },
	// }
	Gauges     []PrometheusGauge
	Summaries  []PrometheusSummary
	Counters   []PrometheusCounter
}

type PrometheusSink struct {
	// If these will ever be copied, they should be converted to *sync.Map values and initialized appropriately
	gauges     sync.Map
	summaries  sync.Map
	counters   sync.Map
	expiration time.Duration
}

type PrometheusGauge struct {
	Name []string
	ConstLabels []metrics.Label
	Help string
	prometheus.Gauge
	updatedAt time.Time
	// canDelete is set if the metric is created during runtime so we know it's ephemeral and can delete it on expiry.
	canDelete bool
}

type PrometheusSummary struct {
	Name []string
	ConstLabels []metrics.Label
	Help string
	prometheus.Summary
	updatedAt time.Time
	canDelete bool
}

type PrometheusCounter struct {
	Name []string
	ConstLabels []metrics.Label
	Help string
	prometheus.Counter
	updatedAt time.Time
	canDelete bool
}

// NewPrometheusSink creates a new PrometheusSink using the default options.
func NewPrometheusSink() (*PrometheusSink, error) {
	return NewPrometheusSinkFrom(DefaultPrometheusOpts)
}

// NewPrometheusSinkFrom creates a new PrometheusSink using the passed options.
func NewPrometheusSinkFrom(opts PrometheusOpts) (*PrometheusSink, error) {
	sink := &PrometheusSink{
		gauges:     sync.Map{},
		summaries:  sync.Map{},
		counters:   sync.Map{},
		expiration: opts.Expiration,
	}

	initGauges(&sink.gauges, opts.Gauges)
	initSummaries(&sink.summaries, opts.Summaries)
	initCounters(&sink.counters, opts.Counters)

	reg := opts.Registerer
	if reg == nil {
		reg = prometheus.DefaultRegisterer
	}

	return sink, reg.Register(sink)
}

// Describe is needed to meet the Collector interface.
func (p *PrometheusSink) Describe(c chan<- *prometheus.Desc) {
	// We must emit some description otherwise an error is returned. This
	// description isn't shown to the user!
	prometheus.NewGauge(prometheus.GaugeOpts{Name: "Dummy", Help: "Dummy"}).Describe(c)
}

// Collect meets the collection interface and allows us to enforce our expiration
// logic to clean up ephemeral metrics if their value haven't been set for a
// duration exceeding our allowed expiration time.
func (p *PrometheusSink) Collect(c chan<- prometheus.Metric) {
	expire := p.expiration != 0
	now := time.Now()
	p.gauges.Range(func(k, v interface{}) bool {
		if v != nil {
			lastUpdate := v.(*PrometheusGauge).updatedAt
			if expire && lastUpdate.Add(p.expiration).Before(now) {
				if v.(*PrometheusGauge).canDelete {
					p.gauges.Delete(k)
					return true
				}
				// We have not observed the gauge this interval so we don't know its value.
				v.(*PrometheusGauge).Set(math.NaN())
			}
			v.(*PrometheusGauge).Collect(c)
		}
		return true
	})
	p.summaries.Range(func(k, v interface{}) bool {
		if v != nil {
			lastUpdate := v.(*PrometheusSummary).updatedAt
			if expire && lastUpdate.Add(p.expiration).Before(now) {
				if v.(*PrometheusSummary).canDelete {
					p.summaries.Delete(k)
					return true
				}
				// We have observed nothing in this interval.
				v.(*PrometheusSummary).Observe(math.NaN())
			}
			v.(*PrometheusSummary).Collect(c)
		}
		return true
	})
	p.counters.Range(func(k, v interface{}) bool {
		if v != nil {
			lastUpdate := v.(*PrometheusCounter).updatedAt
			if expire && lastUpdate.Add(p.expiration).Before(now) {
				if v.(*PrometheusCounter).canDelete {
					p.counters.Delete(k)
					return true
				}
				// Counters remain at their previous value when not observed, so we do not set it to NaN.
			}
			v.(*PrometheusCounter).Collect(c)
		}
		return true
	})
}

func initGauges(m *sync.Map, gauges []PrometheusGauge) {
	for _, gauge := range gauges {
		key, hash := flattenKey(gauge.Name, gauge.ConstLabels)
		g := prometheus.NewGauge(prometheus.GaugeOpts{
			Name:        key,
			Help:        gauge.Help,
			ConstLabels: prometheusLabels(gauge.ConstLabels),
		})
		g.Set(float64(0)) // Initialize at zero
		gauge.Gauge = g
		m.Store(hash, &gauge)
	}
	return
}

func initSummaries(m *sync.Map, summaries []PrometheusSummary) {
	for _, summary := range summaries {
		key, hash := flattenKey(summary.Name, summary.ConstLabels)
		s := prometheus.NewSummary(prometheus.SummaryOpts{
			Name:        key,
			Help:        summary.Help,
			ConstLabels: prometheusLabels(summary.ConstLabels),
		})
		s.Observe(float64(0)) // Initialize at zero
		summary.Summary = s
		m.Store(hash, &summary)
	}
	return
}

func initCounters(m *sync.Map, counters []PrometheusCounter) {
	for _, counter := range counters {
		key, hash := flattenKey(counter.Name, counter.ConstLabels)
		c := prometheus.NewCounter(prometheus.CounterOpts{
			Name:        key,
			Help:        counter.Help,
			ConstLabels: prometheusLabels(counter.ConstLabels),
		})
		c.Add(float64(0)) // Initialize at zero
		counter.Counter = c
		m.Store(hash, &counter)
	}
	return
}

var forbiddenChars = regexp.MustCompile("[ .=\\-/]")

func flattenKey(parts []string, labels []metrics.Label) (string, string) {
	key := strings.Join(parts, "_")
	key = forbiddenChars.ReplaceAllString(key, "_")

	hash := key
	for _, label := range labels {
		hash += fmt.Sprintf(";%s=%s", label.Name, label.Value)
	}

	return key, hash
}

func prometheusLabels(labels []metrics.Label) prometheus.Labels {
	l := make(prometheus.Labels)
	for _, label := range labels {
		l[label.Name] = label.Value
	}
	return l
}

func (p *PrometheusSink) SetGauge(parts []string, val float32) {
	p.SetGaugeWithLabels(parts, val, nil)
}

func (p *PrometheusSink) SetGaugeWithLabels(parts []string, val float32, labels []metrics.Label) {
	key, hash := flattenKey(parts, labels)
	pg, ok := p.gauges.Load(hash)

	// The sync.Map underlying gauges stores pointers to our structs. If we need to make updates,
	// rather than modifying the underlying value directly, which would be racy, we make a local
	// copy by dereferencing the pointer we get back, making the appropriate changes, and then
	// storing a pointer to our local copy. The underlying Prometheus types are threadsafe,
	// so there's no issues there. It's possible for racy updates to occur to the updatedAt
	// value, but since we're always setting it to time.Now(), it doesn't really matter.
	if ok {
		localGauge := *pg.(*PrometheusGauge)
		localGauge.Set(float64(val))
		localGauge.updatedAt = time.Now()
		p.gauges.Store(hash, &localGauge)

	// The gauge does not exist, create the gauge and allow it to be deleted
	} else {
		g := prometheus.NewGauge(prometheus.GaugeOpts{
			Name:        key,
			Help:        key,
			ConstLabels: prometheusLabels(labels),
		})
		g.Set(float64(val))
		pg = &PrometheusGauge{
			Gauge: g,
			updatedAt: time.Now(),
			canDelete: true,
		}
		p.gauges.Store(hash, pg)
	}
}

func (p *PrometheusSink) AddSample(parts []string, val float32) {
	p.AddSampleWithLabels(parts, val, nil)
}

func (p *PrometheusSink) AddSampleWithLabels(parts []string, val float32, labels []metrics.Label) {
	key, hash := flattenKey(parts, labels)
	ps, ok := p.summaries.Load(hash)

	// Does the summary already exist for this sample type?
	if ok {
		localSummary := *ps.(*PrometheusSummary)
		localSummary.Observe(float64(val))
		localSummary.updatedAt = time.Now()
		p.summaries.Store(hash, &localSummary)

	// The summary does not exist, create the Summary and allow it to be deleted
	} else {
		s := prometheus.NewSummary(prometheus.SummaryOpts{
			Name:        key,
			Help:        key,
			MaxAge:      10 * time.Second,
			ConstLabels: prometheusLabels(labels),
			Objectives:  map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
		})
		s.Observe(float64(val))
		ps = &PrometheusSummary{
			Summary: s,
			updatedAt: time.Now(),
			canDelete: true,
		}
		p.summaries.Store(hash, ps)
	}
}

// EmitKey is not implemented. Prometheus doesn’t offer a type for which an
// arbitrary number of values is retained, as Prometheus works with a pull
// model, rather than a push model.
func (p *PrometheusSink) EmitKey(key []string, val float32) {
}

func (p *PrometheusSink) IncrCounter(parts []string, val float32) {
	p.IncrCounterWithLabels(parts, val, nil)
}

func (p *PrometheusSink) IncrCounterWithLabels(parts []string, val float32, labels []metrics.Label) {
	key, hash := flattenKey(parts, labels)
	pc, ok := p.counters.Load(hash)

	// Does the counter exist?
	if ok {
		localCounter := *pc.(*PrometheusCounter)
		localCounter.Add(float64(val))
		localCounter.updatedAt = time.Now()
		p.counters.Store(hash, &localCounter)

	// The counter does not exist yet, create it and allow it to be deleted
	} else {
		c := prometheus.NewCounter(prometheus.CounterOpts{
			Name:        key,
			Help:        key,
			ConstLabels: prometheusLabels(labels),
		})
		c.Add(float64(val))
		pc = &PrometheusCounter{
			Counter: c,
			updatedAt: time.Now(),
			canDelete: true,
		}
		p.counters.Store(hash, pc)
	}
}

// PrometheusPushSink wraps a normal prometheus sink and provides an address and facilities to export it to an address
// on an interval.
type PrometheusPushSink struct {
	*PrometheusSink
	pusher       *push.Pusher
	address      string
	pushInterval time.Duration
	stopChan     chan struct{}
}

// NewPrometheusPushSink creates a PrometheusPushSink by taking an address, interval, and destination name.
func NewPrometheusPushSink(address string, pushInterval time.Duration, name string) (*PrometheusPushSink, error) {
	promSink := &PrometheusSink{
		gauges:     sync.Map{},
		summaries:  sync.Map{},
		counters:   sync.Map{},
		expiration: 60 * time.Second,
	}

	pusher := push.New(address, name).Collector(promSink)

	sink := &PrometheusPushSink{
		promSink,
		pusher,
		address,
		pushInterval,
		make(chan struct{}),
	}

	sink.flushMetrics()
	return sink, nil
}

func (s *PrometheusPushSink) flushMetrics() {
	ticker := time.NewTicker(s.pushInterval)

	go func() {
		for {
			select {
			case <-ticker.C:
				err := s.pusher.Push()
				if err != nil {
					log.Printf("[ERR] Error pushing to Prometheus! Err: %s", err)
				}
			case <-s.stopChan:
				ticker.Stop()
				return
			}
		}
	}()
}

func (s *PrometheusPushSink) Shutdown() {
	close(s.stopChan)
}
<commit_msg>add Help field to example<commit_after>// +build go1.9

package prometheus

import (
	"fmt"
	"log"
	"math"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/armon/go-metrics"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/push"
)

var (
	// DefaultPrometheusOpts is the default set of options used when creating a
	// PrometheusSink.
	DefaultPrometheusOpts = PrometheusOpts{
		Expiration: 60 * time.Second,
	}
)

// PrometheusOpts is used to configure the Prometheus Sink
type PrometheusOpts struct {
	// Expiration is the duration a metric is valid for, after which it will be
	// untracked. If the value is zero, a metric is never expired.
	Expiration time.Duration
	Registerer prometheus.Registerer

	// Gauges, Summaries, and Counters allow us to pre-declare metrics by giving their Name, Help, and ConstLabels to
	// the PrometheusSink when it is created. Metrics declared in this way will be initialized at zero and will not be
	// deleted when their expiry is reached.
	// - Gauges and Summaries will be set to NaN when they expire.
	// - Counters continue to Collect their last known value.
	// Ex:
	// PrometheusOpts{
	//     Expiration: 10 * time.Second,
	//     Gauges: []PrometheusGauge{
	//         {
	//	         Name: []string{ "application", "component", "measurement"},
	//           Help: "application_component_measurement provides an example of how to declare static metrics",
	//           ConstLabels: []metrics.Label{ { Name: "datacenter", Value: "dc1" }, },
	//         },
	//     },
	// }
	Gauges     []PrometheusGauge
	Summaries  []PrometheusSummary
	Counters   []PrometheusCounter
}

type PrometheusSink struct {
	// If these will ever be copied, they should be converted to *sync.Map values and initialized appropriately
	gauges     sync.Map
	summaries  sync.Map
	counters   sync.Map
	expiration time.Duration
}

type PrometheusGauge struct {
	Name []string
	ConstLabels []metrics.Label
	Help string
	prometheus.Gauge
	updatedAt time.Time
	// canDelete is set if the metric is created during runtime so we know it's ephemeral and can delete it on expiry.
	canDelete bool
}

type PrometheusSummary struct {
	Name []string
	ConstLabels []metrics.Label
	Help string
	prometheus.Summary
	updatedAt time.Time
	canDelete bool
}

type PrometheusCounter struct {
	Name []string
	ConstLabels []metrics.Label
	Help string
	prometheus.Counter
	updatedAt time.Time
	canDelete bool
}

// NewPrometheusSink creates a new PrometheusSink using the default options.
func NewPrometheusSink() (*PrometheusSink, error) {
	return NewPrometheusSinkFrom(DefaultPrometheusOpts)
}

// NewPrometheusSinkFrom creates a new PrometheusSink using the passed options.
func NewPrometheusSinkFrom(opts PrometheusOpts) (*PrometheusSink, error) {
	sink := &PrometheusSink{
		gauges:     sync.Map{},
		summaries:  sync.Map{},
		counters:   sync.Map{},
		expiration: opts.Expiration,
	}

	initGauges(&sink.gauges, opts.Gauges)
	initSummaries(&sink.summaries, opts.Summaries)
	initCounters(&sink.counters, opts.Counters)

	reg := opts.Registerer
	if reg == nil {
		reg = prometheus.DefaultRegisterer
	}

	return sink, reg.Register(sink)
}

// Describe is needed to meet the Collector interface.
func (p *PrometheusSink) Describe(c chan<- *prometheus.Desc) {
	// We must emit some description otherwise an error is returned. This
	// description isn't shown to the user!
	prometheus.NewGauge(prometheus.GaugeOpts{Name: "Dummy", Help: "Dummy"}).Describe(c)
}

// Collect meets the collection interface and allows us to enforce our expiration
// logic to clean up ephemeral metrics if their value haven't been set for a
// duration exceeding our allowed expiration time.
func (p *PrometheusSink) Collect(c chan<- prometheus.Metric) {
	expire := p.expiration != 0
	now := time.Now()
	p.gauges.Range(func(k, v interface{}) bool {
		if v != nil {
			lastUpdate := v.(*PrometheusGauge).updatedAt
			if expire && lastUpdate.Add(p.expiration).Before(now) {
				if v.(*PrometheusGauge).canDelete {
					p.gauges.Delete(k)
					return true
				}
				// We have not observed the gauge this interval so we don't know its value.
				v.(*PrometheusGauge).Set(math.NaN())
			}
			v.(*PrometheusGauge).Collect(c)
		}
		return true
	})
	p.summaries.Range(func(k, v interface{}) bool {
		if v != nil {
			lastUpdate := v.(*PrometheusSummary).updatedAt
			if expire && lastUpdate.Add(p.expiration).Before(now) {
				if v.(*PrometheusSummary).canDelete {
					p.summaries.Delete(k)
					return true
				}
				// We have observed nothing in this interval.
				v.(*PrometheusSummary).Observe(math.NaN())
			}
			v.(*PrometheusSummary).Collect(c)
		}
		return true
	})
	p.counters.Range(func(k, v interface{}) bool {
		if v != nil {
			lastUpdate := v.(*PrometheusCounter).updatedAt
			if expire && lastUpdate.Add(p.expiration).Before(now) {
				if v.(*PrometheusCounter).canDelete {
					p.counters.Delete(k)
					return true
				}
				// Counters remain at their previous value when not observed, so we do not set it to NaN.
			}
			v.(*PrometheusCounter).Collect(c)
		}
		return true
	})
}

func initGauges(m *sync.Map, gauges []PrometheusGauge) {
	for _, gauge := range gauges {
		key, hash := flattenKey(gauge.Name, gauge.ConstLabels)
		g := prometheus.NewGauge(prometheus.GaugeOpts{
			Name:        key,
			Help:        gauge.Help,
			ConstLabels: prometheusLabels(gauge.ConstLabels),
		})
		g.Set(float64(0)) // Initialize at zero
		gauge.Gauge = g
		m.Store(hash, &gauge)
	}
	return
}

func initSummaries(m *sync.Map, summaries []PrometheusSummary) {
	for _, summary := range summaries {
		key, hash := flattenKey(summary.Name, summary.ConstLabels)
		s := prometheus.NewSummary(prometheus.SummaryOpts{
			Name:        key,
			Help:        summary.Help,
			ConstLabels: prometheusLabels(summary.ConstLabels),
		})
		s.Observe(float64(0)) // Initialize at zero
		summary.Summary = s
		m.Store(hash, &summary)
	}
	return
}

func initCounters(m *sync.Map, counters []PrometheusCounter) {
	for _, counter := range counters {
		key, hash := flattenKey(counter.Name, counter.ConstLabels)
		c := prometheus.NewCounter(prometheus.CounterOpts{
			Name:        key,
			Help:        counter.Help,
			ConstLabels: prometheusLabels(counter.ConstLabels),
		})
		c.Add(float64(0)) // Initialize at zero
		counter.Counter = c
		m.Store(hash, &counter)
	}
	return
}

var forbiddenChars = regexp.MustCompile("[ .=\\-/]")

func flattenKey(parts []string, labels []metrics.Label) (string, string) {
	key := strings.Join(parts, "_")
	key = forbiddenChars.ReplaceAllString(key, "_")

	hash := key
	for _, label := range labels {
		hash += fmt.Sprintf(";%s=%s", label.Name, label.Value)
	}

	return key, hash
}

func prometheusLabels(labels []metrics.Label) prometheus.Labels {
	l := make(prometheus.Labels)
	for _, label := range labels {
		l[label.Name] = label.Value
	}
	return l
}

func (p *PrometheusSink) SetGauge(parts []string, val float32) {
	p.SetGaugeWithLabels(parts, val, nil)
}

func (p *PrometheusSink) SetGaugeWithLabels(parts []string, val float32, labels []metrics.Label) {
	key, hash := flattenKey(parts, labels)
	pg, ok := p.gauges.Load(hash)

	// The sync.Map underlying gauges stores pointers to our structs. If we need to make updates,
	// rather than modifying the underlying value directly, which would be racy, we make a local
	// copy by dereferencing the pointer we get back, making the appropriate changes, and then
	// storing a pointer to our local copy. The underlying Prometheus types are threadsafe,
	// so there's no issues there. It's possible for racy updates to occur to the updatedAt
	// value, but since we're always setting it to time.Now(), it doesn't really matter.
	if ok {
		localGauge := *pg.(*PrometheusGauge)
		localGauge.Set(float64(val))
		localGauge.updatedAt = time.Now()
		p.gauges.Store(hash, &localGauge)

	// The gauge does not exist, create the gauge and allow it to be deleted
	} else {
		g := prometheus.NewGauge(prometheus.GaugeOpts{
			Name:        key,
			Help:        key,
			ConstLabels: prometheusLabels(labels),
		})
		g.Set(float64(val))
		pg = &PrometheusGauge{
			Gauge: g,
			updatedAt: time.Now(),
			canDelete: true,
		}
		p.gauges.Store(hash, pg)
	}
}

func (p *PrometheusSink) AddSample(parts []string, val float32) {
	p.AddSampleWithLabels(parts, val, nil)
}

func (p *PrometheusSink) AddSampleWithLabels(parts []string, val float32, labels []metrics.Label) {
	key, hash := flattenKey(parts, labels)
	ps, ok := p.summaries.Load(hash)

	// Does the summary already exist for this sample type?
	if ok {
		localSummary := *ps.(*PrometheusSummary)
		localSummary.Observe(float64(val))
		localSummary.updatedAt = time.Now()
		p.summaries.Store(hash, &localSummary)

	// The summary does not exist, create the Summary and allow it to be deleted
	} else {
		s := prometheus.NewSummary(prometheus.SummaryOpts{
			Name:        key,
			Help:        key,
			MaxAge:      10 * time.Second,
			ConstLabels: prometheusLabels(labels),
			Objectives:  map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
		})
		s.Observe(float64(val))
		ps = &PrometheusSummary{
			Summary: s,
			updatedAt: time.Now(),
			canDelete: true,
		}
		p.summaries.Store(hash, ps)
	}
}

// EmitKey is not implemented. Prometheus doesn’t offer a type for which an
// arbitrary number of values is retained, as Prometheus works with a pull
// model, rather than a push model.
func (p *PrometheusSink) EmitKey(key []string, val float32) {
}

func (p *PrometheusSink) IncrCounter(parts []string, val float32) {
	p.IncrCounterWithLabels(parts, val, nil)
}

func (p *PrometheusSink) IncrCounterWithLabels(parts []string, val float32, labels []metrics.Label) {
	key, hash := flattenKey(parts, labels)
	pc, ok := p.counters.Load(hash)

	// Does the counter exist?
	if ok {
		localCounter := *pc.(*PrometheusCounter)
		localCounter.Add(float64(val))
		localCounter.updatedAt = time.Now()
		p.counters.Store(hash, &localCounter)

	// The counter does not exist yet, create it and allow it to be deleted
	} else {
		c := prometheus.NewCounter(prometheus.CounterOpts{
			Name:        key,
			Help:        key,
			ConstLabels: prometheusLabels(labels),
		})
		c.Add(float64(val))
		pc = &PrometheusCounter{
			Counter: c,
			updatedAt: time.Now(),
			canDelete: true,
		}
		p.counters.Store(hash, pc)
	}
}

// PrometheusPushSink wraps a normal prometheus sink and provides an address and facilities to export it to an address
// on an interval.
type PrometheusPushSink struct {
	*PrometheusSink
	pusher       *push.Pusher
	address      string
	pushInterval time.Duration
	stopChan     chan struct{}
}

// NewPrometheusPushSink creates a PrometheusPushSink by taking an address, interval, and destination name.
func NewPrometheusPushSink(address string, pushInterval time.Duration, name string) (*PrometheusPushSink, error) {
	promSink := &PrometheusSink{
		gauges:     sync.Map{},
		summaries:  sync.Map{},
		counters:   sync.Map{},
		expiration: 60 * time.Second,
	}

	pusher := push.New(address, name).Collector(promSink)

	sink := &PrometheusPushSink{
		promSink,
		pusher,
		address,
		pushInterval,
		make(chan struct{}),
	}

	sink.flushMetrics()
	return sink, nil
}

func (s *PrometheusPushSink) flushMetrics() {
	ticker := time.NewTicker(s.pushInterval)

	go func() {
		for {
			select {
			case <-ticker.C:
				err := s.pusher.Push()
				if err != nil {
					log.Printf("[ERR] Error pushing to Prometheus! Err: %s", err)
				}
			case <-s.stopChan:
				ticker.Stop()
				return
			}
		}
	}()
}

func (s *PrometheusPushSink) Shutdown() {
	close(s.stopChan)
}
<|endoftext|>


<commit_before>package serviceenv

import (
	"fmt"
	"math"
	"net"
	"os"
	"time"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"

	etcd "github.com/coreos/etcd/clientv3"
	loki "github.com/grafana/loki/pkg/logcli/client"
	log "github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
	kube "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

// ServiceEnv is a struct containing connections to other services in the
// cluster. In pachd, there is only one instance of this struct, but tests may
// create more, if they want to create multiple pachyderm "clusters" served in
// separate goroutines.
type ServiceEnv struct {
	*Configuration

	// pachAddress is the domain name or hostport where pachd can be reached
	pachAddress string
	// pachClient is the "template" client other clients returned by this library
	// are based on. It contains the original GRPC client connection and has no
	// ctx and therefore no auth credentials or cancellation
	pachClient *client.APIClient
	// pachEg coordinates the initialization of pachClient.  Note that ServiceEnv
	// uses a separate error group for each client, rather than one for all
	// three clients, so that pachd can initialize a ServiceEnv inside of its own
	// initialization (if GetEtcdClient() blocked on intialization of 'pachClient'
	// and pachd/main.go couldn't start the pachd server until GetEtcdClient() had
	// returned, then pachd would be unable to start)
	pachEg errgroup.Group

	// etcdAddress is the domain name or hostport where etcd can be reached
	etcdAddress string
	// etcdClient is an etcd client that's shared by all users of this environment
	etcdClient *etcd.Client
	// etcdEg coordinates the initialization of etcdClient (see pachdEg)
	etcdEg errgroup.Group

	// kubeClient is a kubernetes client that, if initialized, is shared by all
	// users of this environment
	kubeClient *kube.Clientset
	// kubeEg coordinates the initialization of kubeClient (see pachdEg)
	kubeEg errgroup.Group

	// lokiClient is a loki (log aggregator) client that is shared by all users
	// of this environment, it doesn't require an initialization funcion, so
	// there's no errgroup associated with it.
	lokiClient *loki.Client
}

// InitPachOnlyEnv initializes this service environment. This dials a GRPC
// connection to pachd only (in a background goroutine), and creates the
// template pachClient used by future calls to GetPachClient.
//
// This call returns immediately, but GetPachClient will block
// until the client is ready.
func InitPachOnlyEnv(config *Configuration) *ServiceEnv {
	env := &ServiceEnv{Configuration: config}
	env.pachAddress = net.JoinHostPort("127.0.0.1", fmt.Sprintf("%d", env.PeerPort))
	env.pachEg.Go(env.initPachClient)
	return env // env is not ready yet
}

// InitServiceEnv initializes this service environment. This dials a GRPC
// connection to pachd and etcd (in a background goroutine), and creates the
// template pachClient used by future calls to GetPachClient.
//
// This call returns immediately, but GetPachClient and GetEtcdClient block
// until their respective clients are ready.
func InitServiceEnv(config *Configuration) *ServiceEnv {
	env := InitPachOnlyEnv(config)
	env.etcdAddress = fmt.Sprintf("http://%s", net.JoinHostPort(env.EtcdHost, env.EtcdPort))
	env.etcdEg.Go(env.initEtcdClient)
	if env.LokiLogging && env.LokiHost != "" && env.LokiPort != "" {
		env.lokiClient = &loki.Client{
			Address: fmt.Sprintf("http://%s", net.JoinHostPort(env.LokiHost, env.LokiPort)),
		}
	}
	return env // env is not ready yet
}

// InitWithKube is like InitServiceEnv, but also assumes that it's run inside
// a kubernetes cluster and tries to connect to the kubernetes API server.
func InitWithKube(config *Configuration) *ServiceEnv {
	env := InitServiceEnv(config)
	env.kubeEg.Go(env.initKubeClient)
	return env // env is not ready yet
}

func (env *ServiceEnv) initPachClient() error {
	// validate argument
	if env.pachAddress == "" {
		return errors.New("cannot initialize pach client with empty pach address")
	}
	// Initialize pach client
	return backoff.Retry(func() error {
		var err error
		env.pachClient, err = client.NewFromAddress(env.pachAddress)
		if err != nil {
			return errors.Wrapf(err, "failed to initialize pach client")
		}
		return nil
	}, backoff.RetryEvery(time.Second).For(5*time.Minute))
}

func (env *ServiceEnv) initEtcdClient() error {
	// validate argument
	if env.etcdAddress == "" {
		return errors.New("cannot initialize pach client with empty pach address")
	}
	// Initialize etcd
	return backoff.Retry(func() error {
		var err error
		env.etcdClient, err = etcd.New(etcd.Config{
			Endpoints: []string{env.etcdAddress},
			// Use a long timeout with Etcd so that Pachyderm doesn't crash loop
			// while waiting for etcd to come up (makes startup net faster)
			DialTimeout:        3 * time.Minute,
			DialOptions:        client.DefaultDialOptions(), // SA1019 can't call grpc.Dial directly
			MaxCallSendMsgSize: math.MaxInt32,
			MaxCallRecvMsgSize: math.MaxInt32,
		})
		if err != nil {
			return errors.Wrapf(err, "failed to initialize etcd client")
		}
		return nil
	}, backoff.RetryEvery(time.Second).For(5*time.Minute))
}

func (env *ServiceEnv) initKubeClient() error {
	return backoff.Retry(func() error {
		// Get secure in-cluster config
		var kubeAddr string
		var ok bool
		cfg, err := rest.InClusterConfig()
		if err != nil {
			// InClusterConfig failed, fall back to insecure config
			log.Errorf("falling back to insecure kube client due to error from NewInCluster: %s", err)
			kubeAddr, ok = os.LookupEnv("KUBERNETES_PORT_443_TCP_ADDR")
			if !ok {
				return errors.Wrapf(err, "can't fall back to insecure kube client due to missing env var (failed to retrieve in-cluster config")
			}
			cfg = &rest.Config{
				Host: fmt.Sprintf("%s:443", kubeAddr),
				TLSClientConfig: rest.TLSClientConfig{
					Insecure: true,
				},
			}
		}
		env.kubeClient, err = kube.NewForConfig(cfg)
		if err != nil {
			return errors.Wrapf(err, "could not initialize kube client")
		}
		return nil
	}, backoff.RetryEvery(time.Second).For(5*time.Minute))
}

// GetPachClient returns a pachd client with the same authentication
// credentials and cancellation as 'ctx' (ensuring that auth credentials are
// propagated through downstream RPCs).
//
// Functions that receive RPCs should call this to convert their RPC context to
// a Pachyderm client, and internal Pachyderm calls should accept clients
// returned by this call.
//
// (Warning) Do not call this function during server setup unless it is in a goroutine.
// A Pachyderm client is not available until the server has been setup.
func (env *ServiceEnv) GetPachClient(ctx context.Context) *client.APIClient {
	if err := env.pachEg.Wait(); err != nil {
		panic(err) // If env can't connect, there's no sensible way to recover
	}
	return env.pachClient.WithCtx(ctx)
}

// GetEtcdClient returns the already connected etcd client without modification.
func (env *ServiceEnv) GetEtcdClient() *etcd.Client {
	if err := env.etcdEg.Wait(); err != nil {
		panic(err) // If env can't connect, there's no sensible way to recover
	}
	if env.etcdClient == nil {
		panic("service env never connected to etcd")
	}
	return env.etcdClient
}

// GetKubeClient returns the already connected Kubernetes API client without
// modification.
func (env *ServiceEnv) GetKubeClient() *kube.Clientset {
	if err := env.kubeEg.Wait(); err != nil {
		panic(err) // If env can't connect, there's no sensible way to recover
	}
	if env.kubeClient == nil {
		panic("service env never connected to kubernetes")
	}
	return env.kubeClient
}

// GetLokiClient returns the loki client, it doesn't require blocking on a
// connection because the client is just a dumb struct with no init function.
func (env *ServiceEnv) GetLokiClient() (*loki.Client, error) {
	if env.lokiClient == nil {
		return nil, errors.Errorf("loki not configured, is it running in the same namespace as pachd?")
	}
	return env.lokiClient, nil
}
<commit_msg>Construct loki client even if env-var is off.<commit_after>package serviceenv

import (
	"fmt"
	"math"
	"net"
	"os"
	"time"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"

	etcd "github.com/coreos/etcd/clientv3"
	loki "github.com/grafana/loki/pkg/logcli/client"
	log "github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
	kube "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

// ServiceEnv is a struct containing connections to other services in the
// cluster. In pachd, there is only one instance of this struct, but tests may
// create more, if they want to create multiple pachyderm "clusters" served in
// separate goroutines.
type ServiceEnv struct {
	*Configuration

	// pachAddress is the domain name or hostport where pachd can be reached
	pachAddress string
	// pachClient is the "template" client other clients returned by this library
	// are based on. It contains the original GRPC client connection and has no
	// ctx and therefore no auth credentials or cancellation
	pachClient *client.APIClient
	// pachEg coordinates the initialization of pachClient.  Note that ServiceEnv
	// uses a separate error group for each client, rather than one for all
	// three clients, so that pachd can initialize a ServiceEnv inside of its own
	// initialization (if GetEtcdClient() blocked on intialization of 'pachClient'
	// and pachd/main.go couldn't start the pachd server until GetEtcdClient() had
	// returned, then pachd would be unable to start)
	pachEg errgroup.Group

	// etcdAddress is the domain name or hostport where etcd can be reached
	etcdAddress string
	// etcdClient is an etcd client that's shared by all users of this environment
	etcdClient *etcd.Client
	// etcdEg coordinates the initialization of etcdClient (see pachdEg)
	etcdEg errgroup.Group

	// kubeClient is a kubernetes client that, if initialized, is shared by all
	// users of this environment
	kubeClient *kube.Clientset
	// kubeEg coordinates the initialization of kubeClient (see pachdEg)
	kubeEg errgroup.Group

	// lokiClient is a loki (log aggregator) client that is shared by all users
	// of this environment, it doesn't require an initialization funcion, so
	// there's no errgroup associated with it.
	lokiClient *loki.Client
}

// InitPachOnlyEnv initializes this service environment. This dials a GRPC
// connection to pachd only (in a background goroutine), and creates the
// template pachClient used by future calls to GetPachClient.
//
// This call returns immediately, but GetPachClient will block
// until the client is ready.
func InitPachOnlyEnv(config *Configuration) *ServiceEnv {
	env := &ServiceEnv{Configuration: config}
	env.pachAddress = net.JoinHostPort("127.0.0.1", fmt.Sprintf("%d", env.PeerPort))
	env.pachEg.Go(env.initPachClient)
	return env // env is not ready yet
}

// InitServiceEnv initializes this service environment. This dials a GRPC
// connection to pachd and etcd (in a background goroutine), and creates the
// template pachClient used by future calls to GetPachClient.
//
// This call returns immediately, but GetPachClient and GetEtcdClient block
// until their respective clients are ready.
func InitServiceEnv(config *Configuration) *ServiceEnv {
	env := InitPachOnlyEnv(config)
	env.etcdAddress = fmt.Sprintf("http://%s", net.JoinHostPort(env.EtcdHost, env.EtcdPort))
	env.etcdEg.Go(env.initEtcdClient)
	if env.LokiHost != "" && env.LokiPort != "" {
		env.lokiClient = &loki.Client{
			Address: fmt.Sprintf("http://%s", net.JoinHostPort(env.LokiHost, env.LokiPort)),
		}
	}
	return env // env is not ready yet
}

// InitWithKube is like InitServiceEnv, but also assumes that it's run inside
// a kubernetes cluster and tries to connect to the kubernetes API server.
func InitWithKube(config *Configuration) *ServiceEnv {
	env := InitServiceEnv(config)
	env.kubeEg.Go(env.initKubeClient)
	return env // env is not ready yet
}

func (env *ServiceEnv) initPachClient() error {
	// validate argument
	if env.pachAddress == "" {
		return errors.New("cannot initialize pach client with empty pach address")
	}
	// Initialize pach client
	return backoff.Retry(func() error {
		var err error
		env.pachClient, err = client.NewFromAddress(env.pachAddress)
		if err != nil {
			return errors.Wrapf(err, "failed to initialize pach client")
		}
		return nil
	}, backoff.RetryEvery(time.Second).For(5*time.Minute))
}

func (env *ServiceEnv) initEtcdClient() error {
	// validate argument
	if env.etcdAddress == "" {
		return errors.New("cannot initialize pach client with empty pach address")
	}
	// Initialize etcd
	return backoff.Retry(func() error {
		var err error
		env.etcdClient, err = etcd.New(etcd.Config{
			Endpoints: []string{env.etcdAddress},
			// Use a long timeout with Etcd so that Pachyderm doesn't crash loop
			// while waiting for etcd to come up (makes startup net faster)
			DialTimeout:        3 * time.Minute,
			DialOptions:        client.DefaultDialOptions(), // SA1019 can't call grpc.Dial directly
			MaxCallSendMsgSize: math.MaxInt32,
			MaxCallRecvMsgSize: math.MaxInt32,
		})
		if err != nil {
			return errors.Wrapf(err, "failed to initialize etcd client")
		}
		return nil
	}, backoff.RetryEvery(time.Second).For(5*time.Minute))
}

func (env *ServiceEnv) initKubeClient() error {
	return backoff.Retry(func() error {
		// Get secure in-cluster config
		var kubeAddr string
		var ok bool
		cfg, err := rest.InClusterConfig()
		if err != nil {
			// InClusterConfig failed, fall back to insecure config
			log.Errorf("falling back to insecure kube client due to error from NewInCluster: %s", err)
			kubeAddr, ok = os.LookupEnv("KUBERNETES_PORT_443_TCP_ADDR")
			if !ok {
				return errors.Wrapf(err, "can't fall back to insecure kube client due to missing env var (failed to retrieve in-cluster config")
			}
			cfg = &rest.Config{
				Host: fmt.Sprintf("%s:443", kubeAddr),
				TLSClientConfig: rest.TLSClientConfig{
					Insecure: true,
				},
			}
		}
		env.kubeClient, err = kube.NewForConfig(cfg)
		if err != nil {
			return errors.Wrapf(err, "could not initialize kube client")
		}
		return nil
	}, backoff.RetryEvery(time.Second).For(5*time.Minute))
}

// GetPachClient returns a pachd client with the same authentication
// credentials and cancellation as 'ctx' (ensuring that auth credentials are
// propagated through downstream RPCs).
//
// Functions that receive RPCs should call this to convert their RPC context to
// a Pachyderm client, and internal Pachyderm calls should accept clients
// returned by this call.
//
// (Warning) Do not call this function during server setup unless it is in a goroutine.
// A Pachyderm client is not available until the server has been setup.
func (env *ServiceEnv) GetPachClient(ctx context.Context) *client.APIClient {
	if err := env.pachEg.Wait(); err != nil {
		panic(err) // If env can't connect, there's no sensible way to recover
	}
	return env.pachClient.WithCtx(ctx)
}

// GetEtcdClient returns the already connected etcd client without modification.
func (env *ServiceEnv) GetEtcdClient() *etcd.Client {
	if err := env.etcdEg.Wait(); err != nil {
		panic(err) // If env can't connect, there's no sensible way to recover
	}
	if env.etcdClient == nil {
		panic("service env never connected to etcd")
	}
	return env.etcdClient
}

// GetKubeClient returns the already connected Kubernetes API client without
// modification.
func (env *ServiceEnv) GetKubeClient() *kube.Clientset {
	if err := env.kubeEg.Wait(); err != nil {
		panic(err) // If env can't connect, there's no sensible way to recover
	}
	if env.kubeClient == nil {
		panic("service env never connected to kubernetes")
	}
	return env.kubeClient
}

// GetLokiClient returns the loki client, it doesn't require blocking on a
// connection because the client is just a dumb struct with no init function.
func (env *ServiceEnv) GetLokiClient() (*loki.Client, error) {
	if env.lokiClient == nil {
		return nil, errors.Errorf("loki not configured, is it running in the same namespace as pachd?")
	}
	return env.lokiClient, nil
}
<|endoftext|>


<commit_before>package firego

import (
	"bytes"
	"io/ioutil"
	"net/http"
)

// Firebase represents a location in the cloud
type Firebase struct {
	url    string
	client *http.Client
}

// New creates a new Firebase reference
func New(url string) *Firebase {
	return &Firebase{
		url:    url,
		client: &http.Client{},
	}
}

// String returns the string representation of the
// Firebase reference
func (fb *Firebase) String() string {
	return fb.url
}

// Child creates a new Firebase reference for the requested
// child string
func (fb *Firebase) Child(child string) *Firebase {
	return &Firebase{
		url:    fb.url + "/" + child,
		client: fb.client,
	}
}

func (fb *Firebase) doRequest(method string, body []byte) ([]byte, error) {
	url := fb.url + "/.json"
	req, err := http.NewRequest(method, url, bytes.NewReader(body))
	if err != nil {
		return nil, err
	}

	resp, err := fb.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}
<commit_msg>returning response body if the response wasn't in the 200 range<commit_after>package firego

import (
	"bytes"
	"errors"
	"io/ioutil"
	"net/http"
)

// Firebase represents a location in the cloud
type Firebase struct {
	url    string
	client *http.Client
}

// New creates a new Firebase reference
func New(url string) *Firebase {
	return &Firebase{
		url:    url,
		client: &http.Client{},
	}
}

// String returns the string representation of the
// Firebase reference
func (fb *Firebase) String() string {
	return fb.url
}

// Child creates a new Firebase reference for the requested
// child string
func (fb *Firebase) Child(child string) *Firebase {
	return &Firebase{
		url:    fb.url + "/" + child,
		client: fb.client,
	}
}

func (fb *Firebase) doRequest(method string, body []byte) ([]byte, error) {
	url := fb.url + "/.json"
	req, err := http.NewRequest(method, url, bytes.NewReader(body))
	if err != nil {
		return nil, err
	}

	resp, err := fb.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode/200 != 1 {
		return nil, errors.New(string(respBody))
	}
	return respBody, nil
}
<|endoftext|>


<commit_before>package cli_test

import (
	"bytes"
	"fmt"
	"github.com/qadium/plumber/shell"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"strings"
	"testing"
	"time"
)

// this is a *functional test*
// we check that boostrap works by actually running the boostrap command
// and checking that the container is built and runs

func TestBootstrap(t *testing.T) {
	ctx, tempDir := NewTestContext(t)
	defer cleanTestDir(t, tempDir)

	// step 1. remove any image named plumber/test-manager from the current
	// set of docker images (ignore any errors)
	_ = shell.RunAndLog("docker", "rmi", ctx.Image)

	// step 2. invoke Bootstrap for building ctx.Image
	if err := ctx.Bootstrap(); err != nil {
		t.Errorf("Got an error during bootstrap: '%v'", err)
	}
	defer shell.RunAndLog("docker", "rmi", ctx.Image)

	// step 3. run the image (it *should* just echo in response)
	if err := shell.RunAndLog("docker", "run", "-d", "-p", "9800:9800", "--name", "plumber-test", ctx.Image); err != nil {
		t.Errorf("Got an error during docker run: '%v'", err)
	}
	defer shell.RunAndLog("docker", "rm", "-f", "plumber-test")
	// wait a bit for the container to come up
	time.Sleep(1 * time.Second)

	// step 4. send some JSON and check for echos
	// first, find the IP to connect to

	// get the DOCKER_HOST environment variable; if not defined, use
	// docker to find it
	hostIp := os.Getenv("DOCKER_HOST")
	if hostIp == "" {
		cmd := exec.Command("docker", "inspect", "--format='{{.NetworkSettings.Gateway}}'", "plumber-test")
		hostIpBytes, err := cmd.Output()
		if err != nil {
			t.Errorf("Got an error during docker inspect: '%v'", err)
		}
		hostIpBytes = bytes.Trim(hostIpBytes, "\r\n")
		hostIp = string(hostIpBytes)
	} else {
		hostUrl, err := url.Parse(hostIp)
		if err != nil {
			t.Errorf("Got an error during url parsing: '%v'", err)
		}
		// docker host is usually in the form of IP:PORT
		hostIp = strings.Split(hostUrl.Host, ":")[0]
	}

	// second, send over some JSON and verify result
	resp, err := http.Post(fmt.Sprintf("http://%s:9800", hostIp), "application/json", bytes.NewBufferString(`{"foo": 3}`))
	if err != nil {
		t.Error(err)
	}
	buf := new(bytes.Buffer)
	buf.ReadFrom(resp.Body)
	result := buf.String()
	if result != `{"foo": 3}` {
		t.Errorf("Got '%s'; did not get expected response", result)
	}
}
<commit_msg>bootstrap test silences output<commit_after>package cli_test

import (
	"bytes"
	"fmt"
	"github.com/qadium/plumber/shell"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"strings"
	"testing"
	"time"
	"io/ioutil"
	"log"
)

// this is a *functional test*
// we check that boostrap works by actually running the boostrap command
// and checking that the container is built and runs
func TestBootstrap(t *testing.T) {
	log.SetOutput(ioutil.Discard)

	ctx, tempDir := NewTestContext(t)
	defer cleanTestDir(t, tempDir)

	// step 1. remove any image named plumber/test-manager from the current
	// set of docker images (ignore any errors)
	_ = shell.RunAndLog("docker", "rmi", ctx.Image)

	// step 2. invoke Bootstrap for building ctx.Image
	if err := ctx.Bootstrap(); err != nil {
		t.Errorf("Got an error during bootstrap: '%v'", err)
	}
	defer shell.RunAndLog("docker", "rmi", ctx.Image)

	// step 3. run the image (it *should* just echo in response)
	if err := shell.RunAndLog("docker", "run", "-d", "-p", "9800:9800", "--name", "plumber-test", ctx.Image); err != nil {
		t.Errorf("Got an error during docker run: '%v'", err)
	}
	defer shell.RunAndLog("docker", "rm", "-f", "plumber-test")
	// wait a bit for the container to come up
	time.Sleep(1 * time.Second)

	// step 4. send some JSON and check for echos
	// first, find the IP to connect to

	// get the DOCKER_HOST environment variable; if not defined, use
	// docker to find it
	hostIp := os.Getenv("DOCKER_HOST")
	if hostIp == "" {
		cmd := exec.Command("docker", "inspect", "--format='{{.NetworkSettings.Gateway}}'", "plumber-test")
		hostIpBytes, err := cmd.Output()
		if err != nil {
			t.Errorf("Got an error during docker inspect: '%v'", err)
		}
		hostIpBytes = bytes.Trim(hostIpBytes, "\r\n")
		hostIp = string(hostIpBytes)
	} else {
		hostUrl, err := url.Parse(hostIp)
		if err != nil {
			t.Errorf("Got an error during url parsing: '%v'", err)
		}
		// docker host is usually in the form of IP:PORT
		hostIp = strings.Split(hostUrl.Host, ":")[0]
	}

	// second, send over some JSON and verify result
	resp, err := http.Post(fmt.Sprintf("http://%s:9800", hostIp), "application/json", bytes.NewBufferString(`{"foo": 3}`))
	if err != nil {
		t.Error(err)
	}
	buf := new(bytes.Buffer)
	buf.ReadFrom(resp.Body)
	result := buf.String()
	if result != `{"foo": 3}` {
		t.Errorf("Got '%s'; did not get expected response", result)
	}
}
<|endoftext|>


<commit_before>package k8s

import (
	"fmt"
	"net/http"

	"github.com/Sirupsen/logrus"
	"github.com/dgrijalva/jwt-go"
	"github.com/gorilla/mux"
)

type BackendAccess interface {
	AuthAndLookup(req *http.Request) (*jwt.Token, string, error)
	ServeRemoteHTTP(token *jwt.Token, hostKey string, rw http.ResponseWriter, req *http.Request) error
}

type handler struct {
	lookup              *Lookup
	accessKey           string
	secretKey           string
	frontendHTTPHandler BackendAccess
}

func Handler(frontendHTTPHandler BackendAccess, cattleAddr, accessKey, secretKey string) http.Handler {
	return &handler{
		lookup:              NewLookup(fmt.Sprintf("http://%s/v3/clusters", cattleAddr), accessKey, secretKey),
		accessKey:           accessKey,
		secretKey:           secretKey,
		frontendHTTPHandler: frontendHTTPHandler,
	}
}

func (h *handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	cluster, _, err := h.lookup.Lookup(req)
	if err != nil {
		logrus.Errorf("Failed to find cluster: %v", err)
		http.Error(rw, fmt.Sprintf("Failed to find cluster: %v", err), http.StatusInternalServerError)
		return
	}
	if cluster == nil {
		http.Error(rw, "Failed to find cluster", http.StatusNotFound)
		return
	}

	vars := mux.Vars(req)
	vars["service"] = fmt.Sprintf("k8s-api.%s", cluster.Id)

	oldAuth := req.Header.Get("Authorization")
	req.SetBasicAuth(h.accessKey, h.secretKey)

	token, hostKey, err := h.frontendHTTPHandler.AuthAndLookup(req)
	if err != nil {
		http.Error(rw, fmt.Sprintf("Failed to authorize cluster: %v", err), http.StatusInternalServerError)
		return
	}

	req.Header.Set("Authorization", oldAuth)

	if err := h.frontendHTTPHandler.ServeRemoteHTTP(token, hostKey, rw, req); err != nil {
		logrus.Errorf("Failed to forward request: %v", err)
		http.Error(rw, fmt.Sprintf("Failed to forward request: %v", err), http.StatusInternalServerError)
	}
}
<commit_msg>Always pass service creds right now<commit_after>package k8s

import (
	"fmt"
	"net/http"

	"github.com/Sirupsen/logrus"
	"github.com/dgrijalva/jwt-go"
	"github.com/gorilla/mux"
)

type BackendAccess interface {
	AuthAndLookup(req *http.Request) (*jwt.Token, string, error)
	ServeRemoteHTTP(token *jwt.Token, hostKey string, rw http.ResponseWriter, req *http.Request) error
}

type handler struct {
	lookup              *Lookup
	accessKey           string
	secretKey           string
	frontendHTTPHandler BackendAccess
}

func Handler(frontendHTTPHandler BackendAccess, cattleAddr, accessKey, secretKey string) http.Handler {
	return &handler{
		lookup:              NewLookup(fmt.Sprintf("http://%s/v3/clusters", cattleAddr), accessKey, secretKey),
		accessKey:           accessKey,
		secretKey:           secretKey,
		frontendHTTPHandler: frontendHTTPHandler,
	}
}

func (h *handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	cluster, _, err := h.lookup.Lookup(req)
	if err != nil {
		logrus.Errorf("Failed to find cluster: %v", err)
		http.Error(rw, fmt.Sprintf("Failed to find cluster: %v", err), http.StatusInternalServerError)
		return
	}
	if cluster == nil {
		http.Error(rw, "Failed to find cluster", http.StatusNotFound)
		return
	}

	vars := mux.Vars(req)
	vars["service"] = fmt.Sprintf("k8s-api.%s", cluster.Id)

	//oldAuth := req.Header.Get("Authorization")
	req.SetBasicAuth(h.accessKey, h.secretKey)

	token, hostKey, err := h.frontendHTTPHandler.AuthAndLookup(req)
	if err != nil {
		http.Error(rw, fmt.Sprintf("Failed to authorize cluster: %v", err), http.StatusInternalServerError)
		return
	}

	//req.Header.Set("Authorization", oldAuth)
	req.Header.Set("Authorization", "Bearer "+cluster.K8sClientConfig.BearerToken)
	req.Header.Set("X-API-Cluster-Id", cluster.Id)

	if err := h.frontendHTTPHandler.ServeRemoteHTTP(token, hostKey, rw, req); err != nil {
		logrus.Errorf("Failed to forward request: %v", err)
		http.Error(rw, fmt.Sprintf("Failed to forward request: %v", err), http.StatusInternalServerError)
	}
}
<|endoftext|>


<commit_before>package pgtype

import (
	"database/sql/driver"
	"encoding/binary"
	"fmt"
	"math"
	"strconv"

	"github.com/jackc/pgx/v5/internal/pgio"
)

type Float64Scanner interface {
	ScanFloat64(Float8) error
}

type Float64Valuer interface {
	Float64Value() (Float8, error)
}

type Float8 struct {
	Float64 float64
	Valid   bool
}

// ScanFloat64 implements the Float64Scanner interface.
func (f *Float8) ScanFloat64(n Float8) error {
	*f = n
	return nil
}

func (f Float8) Float64Value() (Float8, error) {
	return f, nil
}

func (f *Float8) ScanInt64(n Int8) error {
	*f = Float8{Float64: float64(n.Int64), Valid: n.Valid}
	return nil
}

func (f Float8) Int64Value() (Int8, error) {
	return Int8{Int64: int64(f.Float64), Valid: f.Valid}, nil
}

// Scan implements the database/sql Scanner interface.
func (f *Float8) Scan(src any) error {
	if src == nil {
		*f = Float8{}
		return nil
	}

	switch src := src.(type) {
	case float64:
		*f = Float8{Float64: src, Valid: true}
		return nil
	case string:
		n, err := strconv.ParseFloat(string(src), 64)
		if err != nil {
			return err
		}
		*f = Float8{Float64: n, Valid: true}
		return nil
	}

	return fmt.Errorf("cannot scan %T", src)
}

// Value implements the database/sql/driver Valuer interface.
func (f Float8) Value() (driver.Value, error) {
	if !f.Valid {
		return nil, nil
	}
	return f.Float64, nil
}

type Float8Codec struct{}

func (Float8Codec) FormatSupported(format int16) bool {
	return format == TextFormatCode || format == BinaryFormatCode
}

func (Float8Codec) PreferredFormat() int16 {
	return BinaryFormatCode
}

func (Float8Codec) PlanEncode(m *Map, oid uint32, format int16, value any) EncodePlan {
	switch format {
	case BinaryFormatCode:
		switch value.(type) {
		case float64:
			return encodePlanFloat8CodecBinaryFloat64{}
		case Float64Valuer:
			return encodePlanFloat8CodecBinaryFloat64Valuer{}
		case Int64Valuer:
			return encodePlanFloat8CodecBinaryInt64Valuer{}
		}
	case TextFormatCode:
		switch value.(type) {
		case float64:
			return encodePlanTextFloat64{}
		case Float64Valuer:
			return encodePlanTextFloat64Valuer{}
		case Int64Valuer:
			return encodePlanTextInt64Valuer{}
		}
	}

	return nil
}

type encodePlanFloat8CodecBinaryFloat64 struct{}

func (encodePlanFloat8CodecBinaryFloat64) Encode(value any, buf []byte) (newBuf []byte, err error) {
	n := value.(float64)
	return pgio.AppendUint64(buf, math.Float64bits(n)), nil
}

type encodePlanTextFloat64 struct{}

func (encodePlanTextFloat64) Encode(value any, buf []byte) (newBuf []byte, err error) {
	n := value.(float64)
	return append(buf, strconv.FormatFloat(n, 'f', -1, 64)...), nil
}

type encodePlanFloat8CodecBinaryFloat64Valuer struct{}

func (encodePlanFloat8CodecBinaryFloat64Valuer) Encode(value any, buf []byte) (newBuf []byte, err error) {
	n, err := value.(Float64Valuer).Float64Value()
	if err != nil {
		return nil, err
	}

	if !n.Valid {
		return nil, nil
	}

	return pgio.AppendUint64(buf, math.Float64bits(n.Float64)), nil
}

type encodePlanTextFloat64Valuer struct{}

func (encodePlanTextFloat64Valuer) Encode(value any, buf []byte) (newBuf []byte, err error) {
	n, err := value.(Float64Valuer).Float64Value()
	if err != nil {
		return nil, err
	}

	if !n.Valid {
		return nil, nil
	}

	return append(buf, strconv.FormatFloat(n.Float64, 'f', -1, 64)...), nil
}

type encodePlanFloat8CodecBinaryInt64Valuer struct{}

func (encodePlanFloat8CodecBinaryInt64Valuer) Encode(value any, buf []byte) (newBuf []byte, err error) {
	n, err := value.(Int64Valuer).Int64Value()
	if err != nil {
		return nil, err
	}

	if !n.Valid {
		return nil, nil
	}

	f := float64(n.Int64)
	return pgio.AppendUint64(buf, math.Float64bits(f)), nil
}

type encodePlanTextInt64Valuer struct{}

func (encodePlanTextInt64Valuer) Encode(value any, buf []byte) (newBuf []byte, err error) {
	n, err := value.(Int64Valuer).Int64Value()
	if err != nil {
		return nil, err
	}

	if !n.Valid {
		return nil, nil
	}

	return append(buf, strconv.FormatInt(n.Int64, 10)...), nil
}

func (Float8Codec) PlanScan(m *Map, oid uint32, format int16, target any) ScanPlan {

	switch format {
	case BinaryFormatCode:
		switch target.(type) {
		case *float64:
			return scanPlanBinaryFloat8ToFloat64{}
		case Float64Scanner:
			return scanPlanBinaryFloat8ToFloat64Scanner{}
		case Int64Scanner:
			return scanPlanBinaryFloat8ToInt64Scanner{}
		case TextScanner:
			return scanPlanBinaryFloat8ToTextScanner{}
		}
	case TextFormatCode:
		switch target.(type) {
		case *float64:
			return scanPlanTextAnyToFloat64{}
		case Float64Scanner:
			return scanPlanTextAnyToFloat64Scanner{}
		case Int64Scanner:
			return scanPlanTextAnyToInt64Scanner{}
		}
	}

	return nil
}

type scanPlanBinaryFloat8ToFloat64 struct{}

func (scanPlanBinaryFloat8ToFloat64) Scan(src []byte, dst any) error {
	if src == nil {
		return fmt.Errorf("cannot scan NULL into %T", dst)
	}

	if len(src) != 8 {
		return fmt.Errorf("invalid length for float8: %v", len(src))
	}

	n := int64(binary.BigEndian.Uint64(src))
	f := (dst).(*float64)
	*f = math.Float64frombits(uint64(n))

	return nil
}

type scanPlanBinaryFloat8ToFloat64Scanner struct{}

func (scanPlanBinaryFloat8ToFloat64Scanner) Scan(src []byte, dst any) error {
	s := (dst).(Float64Scanner)

	if src == nil {
		return s.ScanFloat64(Float8{})
	}

	if len(src) != 8 {
		return fmt.Errorf("invalid length for float8: %v", len(src))
	}

	n := int64(binary.BigEndian.Uint64(src))
	return s.ScanFloat64(Float8{Float64: math.Float64frombits(uint64(n)), Valid: true})
}

type scanPlanBinaryFloat8ToInt64Scanner struct{}

func (scanPlanBinaryFloat8ToInt64Scanner) Scan(src []byte, dst any) error {
	s := (dst).(Int64Scanner)

	if src == nil {
		return s.ScanInt64(Int8{})
	}

	if len(src) != 8 {
		return fmt.Errorf("invalid length for float8: %v", len(src))
	}

	ui64 := int64(binary.BigEndian.Uint64(src))
	f64 := math.Float64frombits(uint64(ui64))
	i64 := int64(f64)
	if f64 != float64(i64) {
		return fmt.Errorf("cannot losslessly convert %v to int64", f64)
	}

	return s.ScanInt64(Int8{Int64: i64, Valid: true})
}

type scanPlanBinaryFloat8ToTextScanner struct{}

func (scanPlanBinaryFloat8ToTextScanner) Scan(src []byte, dst any) error {
	s := (dst).(TextScanner)

	if src == nil {
		return s.ScanText(Text{})
	}

	if len(src) != 8 {
		return fmt.Errorf("invalid length for float8: %v", len(src))
	}

	ui64 := int64(binary.BigEndian.Uint64(src))
	f64 := math.Float64frombits(uint64(ui64))

	return s.ScanText(Text{String: strconv.FormatFloat(f64, 'f', -1, 64), Valid: true})
}

type scanPlanTextAnyToFloat64 struct{}

func (scanPlanTextAnyToFloat64) Scan(src []byte, dst any) error {
	if src == nil {
		return fmt.Errorf("cannot scan NULL into %T", dst)
	}

	n, err := strconv.ParseFloat(string(src), 64)
	if err != nil {
		return err
	}

	f := (dst).(*float64)
	*f = n

	return nil
}

type scanPlanTextAnyToFloat64Scanner struct{}

func (scanPlanTextAnyToFloat64Scanner) Scan(src []byte, dst any) error {
	s := (dst).(Float64Scanner)

	if src == nil {
		return s.ScanFloat64(Float8{})
	}

	n, err := strconv.ParseFloat(string(src), 64)
	if err != nil {
		return err
	}

	return s.ScanFloat64(Float8{Float64: n, Valid: true})
}

func (c Float8Codec) DecodeDatabaseSQLValue(m *Map, oid uint32, format int16, src []byte) (driver.Value, error) {
	return c.DecodeValue(m, oid, format, src)
}

func (c Float8Codec) DecodeValue(m *Map, oid uint32, format int16, src []byte) (any, error) {
	if src == nil {
		return nil, nil
	}

	var n float64
	err := codecScan(c, m, oid, format, src, &n)
	if err != nil {
		return nil, err
	}
	return n, nil
}
<commit_msg>feat: add marshalJSON for float8 type<commit_after>package pgtype

import (
	"database/sql/driver"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"math"
	"strconv"

	"github.com/jackc/pgx/v5/internal/pgio"
)

type Float64Scanner interface {
	ScanFloat64(Float8) error
}

type Float64Valuer interface {
	Float64Value() (Float8, error)
}

type Float8 struct {
	Float64 float64
	Valid   bool
}

// ScanFloat64 implements the Float64Scanner interface.
func (f *Float8) ScanFloat64(n Float8) error {
	*f = n
	return nil
}

func (f Float8) Float64Value() (Float8, error) {
	return f, nil
}

func (f *Float8) ScanInt64(n Int8) error {
	*f = Float8{Float64: float64(n.Int64), Valid: n.Valid}
	return nil
}

func (f Float8) Int64Value() (Int8, error) {
	return Int8{Int64: int64(f.Float64), Valid: f.Valid}, nil
}

// Scan implements the database/sql Scanner interface.
func (f *Float8) Scan(src any) error {
	if src == nil {
		*f = Float8{}
		return nil
	}

	switch src := src.(type) {
	case float64:
		*f = Float8{Float64: src, Valid: true}
		return nil
	case string:
		n, err := strconv.ParseFloat(string(src), 64)
		if err != nil {
			return err
		}
		*f = Float8{Float64: n, Valid: true}
		return nil
	}

	return fmt.Errorf("cannot scan %T", src)
}

// Value implements the database/sql/driver Valuer interface.
func (f Float8) Value() (driver.Value, error) {
	if !f.Valid {
		return nil, nil
	}
	return f.Float64, nil
}

type Float8Codec struct{}

func (Float8Codec) FormatSupported(format int16) bool {
	return format == TextFormatCode || format == BinaryFormatCode
}

func (Float8Codec) PreferredFormat() int16 {
	return BinaryFormatCode
}

func (Float8Codec) PlanEncode(m *Map, oid uint32, format int16, value any) EncodePlan {
	switch format {
	case BinaryFormatCode:
		switch value.(type) {
		case float64:
			return encodePlanFloat8CodecBinaryFloat64{}
		case Float64Valuer:
			return encodePlanFloat8CodecBinaryFloat64Valuer{}
		case Int64Valuer:
			return encodePlanFloat8CodecBinaryInt64Valuer{}
		}
	case TextFormatCode:
		switch value.(type) {
		case float64:
			return encodePlanTextFloat64{}
		case Float64Valuer:
			return encodePlanTextFloat64Valuer{}
		case Int64Valuer:
			return encodePlanTextInt64Valuer{}
		}
	}

	return nil
}

func (f *Float8) MarshalJSON() ([]byte, error) {
	if !f.Valid {
		return []byte("null"), nil
	}
	return json.Marshal(f.Float64)
}

type encodePlanFloat8CodecBinaryFloat64 struct{}

func (encodePlanFloat8CodecBinaryFloat64) Encode(value any, buf []byte) (newBuf []byte, err error) {
	n := value.(float64)
	return pgio.AppendUint64(buf, math.Float64bits(n)), nil
}

type encodePlanTextFloat64 struct{}

func (encodePlanTextFloat64) Encode(value any, buf []byte) (newBuf []byte, err error) {
	n := value.(float64)
	return append(buf, strconv.FormatFloat(n, 'f', -1, 64)...), nil
}

type encodePlanFloat8CodecBinaryFloat64Valuer struct{}

func (encodePlanFloat8CodecBinaryFloat64Valuer) Encode(value any, buf []byte) (newBuf []byte, err error) {
	n, err := value.(Float64Valuer).Float64Value()
	if err != nil {
		return nil, err
	}

	if !n.Valid {
		return nil, nil
	}

	return pgio.AppendUint64(buf, math.Float64bits(n.Float64)), nil
}

type encodePlanTextFloat64Valuer struct{}

func (encodePlanTextFloat64Valuer) Encode(value any, buf []byte) (newBuf []byte, err error) {
	n, err := value.(Float64Valuer).Float64Value()
	if err != nil {
		return nil, err
	}

	if !n.Valid {
		return nil, nil
	}

	return append(buf, strconv.FormatFloat(n.Float64, 'f', -1, 64)...), nil
}

type encodePlanFloat8CodecBinaryInt64Valuer struct{}

func (encodePlanFloat8CodecBinaryInt64Valuer) Encode(value any, buf []byte) (newBuf []byte, err error) {
	n, err := value.(Int64Valuer).Int64Value()
	if err != nil {
		return nil, err
	}

	if !n.Valid {
		return nil, nil
	}

	f := float64(n.Int64)
	return pgio.AppendUint64(buf, math.Float64bits(f)), nil
}

type encodePlanTextInt64Valuer struct{}

func (encodePlanTextInt64Valuer) Encode(value any, buf []byte) (newBuf []byte, err error) {
	n, err := value.(Int64Valuer).Int64Value()
	if err != nil {
		return nil, err
	}

	if !n.Valid {
		return nil, nil
	}

	return append(buf, strconv.FormatInt(n.Int64, 10)...), nil
}

func (Float8Codec) PlanScan(m *Map, oid uint32, format int16, target any) ScanPlan {

	switch format {
	case BinaryFormatCode:
		switch target.(type) {
		case *float64:
			return scanPlanBinaryFloat8ToFloat64{}
		case Float64Scanner:
			return scanPlanBinaryFloat8ToFloat64Scanner{}
		case Int64Scanner:
			return scanPlanBinaryFloat8ToInt64Scanner{}
		case TextScanner:
			return scanPlanBinaryFloat8ToTextScanner{}
		}
	case TextFormatCode:
		switch target.(type) {
		case *float64:
			return scanPlanTextAnyToFloat64{}
		case Float64Scanner:
			return scanPlanTextAnyToFloat64Scanner{}
		case Int64Scanner:
			return scanPlanTextAnyToInt64Scanner{}
		}
	}

	return nil
}

type scanPlanBinaryFloat8ToFloat64 struct{}

func (scanPlanBinaryFloat8ToFloat64) Scan(src []byte, dst any) error {
	if src == nil {
		return fmt.Errorf("cannot scan NULL into %T", dst)
	}

	if len(src) != 8 {
		return fmt.Errorf("invalid length for float8: %v", len(src))
	}

	n := int64(binary.BigEndian.Uint64(src))
	f := (dst).(*float64)
	*f = math.Float64frombits(uint64(n))

	return nil
}

type scanPlanBinaryFloat8ToFloat64Scanner struct{}

func (scanPlanBinaryFloat8ToFloat64Scanner) Scan(src []byte, dst any) error {
	s := (dst).(Float64Scanner)

	if src == nil {
		return s.ScanFloat64(Float8{})
	}

	if len(src) != 8 {
		return fmt.Errorf("invalid length for float8: %v", len(src))
	}

	n := int64(binary.BigEndian.Uint64(src))
	return s.ScanFloat64(Float8{Float64: math.Float64frombits(uint64(n)), Valid: true})
}

type scanPlanBinaryFloat8ToInt64Scanner struct{}

func (scanPlanBinaryFloat8ToInt64Scanner) Scan(src []byte, dst any) error {
	s := (dst).(Int64Scanner)

	if src == nil {
		return s.ScanInt64(Int8{})
	}

	if len(src) != 8 {
		return fmt.Errorf("invalid length for float8: %v", len(src))
	}

	ui64 := int64(binary.BigEndian.Uint64(src))
	f64 := math.Float64frombits(uint64(ui64))
	i64 := int64(f64)
	if f64 != float64(i64) {
		return fmt.Errorf("cannot losslessly convert %v to int64", f64)
	}

	return s.ScanInt64(Int8{Int64: i64, Valid: true})
}

type scanPlanBinaryFloat8ToTextScanner struct{}

func (scanPlanBinaryFloat8ToTextScanner) Scan(src []byte, dst any) error {
	s := (dst).(TextScanner)

	if src == nil {
		return s.ScanText(Text{})
	}

	if len(src) != 8 {
		return fmt.Errorf("invalid length for float8: %v", len(src))
	}

	ui64 := int64(binary.BigEndian.Uint64(src))
	f64 := math.Float64frombits(uint64(ui64))

	return s.ScanText(Text{String: strconv.FormatFloat(f64, 'f', -1, 64), Valid: true})
}

type scanPlanTextAnyToFloat64 struct{}

func (scanPlanTextAnyToFloat64) Scan(src []byte, dst any) error {
	if src == nil {
		return fmt.Errorf("cannot scan NULL into %T", dst)
	}

	n, err := strconv.ParseFloat(string(src), 64)
	if err != nil {
		return err
	}

	f := (dst).(*float64)
	*f = n

	return nil
}

type scanPlanTextAnyToFloat64Scanner struct{}

func (scanPlanTextAnyToFloat64Scanner) Scan(src []byte, dst any) error {
	s := (dst).(Float64Scanner)

	if src == nil {
		return s.ScanFloat64(Float8{})
	}

	n, err := strconv.ParseFloat(string(src), 64)
	if err != nil {
		return err
	}

	return s.ScanFloat64(Float8{Float64: n, Valid: true})
}

func (c Float8Codec) DecodeDatabaseSQLValue(m *Map, oid uint32, format int16, src []byte) (driver.Value, error) {
	return c.DecodeValue(m, oid, format, src)
}

func (c Float8Codec) DecodeValue(m *Map, oid uint32, format int16, src []byte) (any, error) {
	if src == nil {
		return nil, nil
	}

	var n float64
	err := codecScan(c, m, oid, format, src, &n)
	if err != nil {
		return nil, err
	}
	return n, nil
}
<|endoftext|>


<commit_before>package provider

import "fmt"

const (
	minionImage = "quay.io/netsys/di-minion:latest"
)

func cloudConfigUbuntu(keys []string, user string, ubuntuVersion string) string {
	cloudConfig := `#!/bin/bash

initialize_ovs() {
	cat <<- EOF > /etc/systemd/system/ovs.service
	[Unit]
	Description=OVS

	[Service]
	ExecStart=/sbin/modprobe openvswitch
	ExecStartPost=/sbin/modprobe vport_geneve

	[Install]
	WantedBy=multi-user.target
	EOF
}

initialize_docker() {
	PRIVATE_IPv4="$(curl http://instance-data/latest/meta-data/local-ipv4)"
	mkdir -p /etc/systemd/system/docker.service.d

	cat <<- EOF > /etc/systemd/system/docker.service.d/override.conf
	[Unit]
	Description=docker

	[Service]
	ExecStart=
	ExecStart=/usr/bin/docker daemon --bridge=none \
	-H "${PRIVATE_IPv4}:2375" -H unix:///var/run/docker.sock \

	[Install]
	WantedBy=multi-user.target
	EOF
}

initialize_minion() {
	cat <<- EOF > /etc/systemd/system/minion.service
	[Unit]
	Description=DI Minion
	After=docker.service
	Requires=docker.service

	[Service]
	TimeoutSec=1000
	ExecStartPre=-/usr/bin/mkdir -p /var/run/netns
	ExecStartPre=-/usr/bin/docker kill minion
	ExecStartPre=-/usr/bin/docker rm minion
	ExecStartPre=/usr/bin/docker pull %[1]s
	ExecStart=/usr/bin/docker run --net=host --name=minion --privileged \
	-v /var/run/docker.sock:/var/run/docker.sock \
	-v /proc:/hostproc:ro -v /var/run/netns:/var/run/netns:rw %[1]s

	[Install]
	WantedBy=multi-user.target
	EOF
}

install_docker() {
	# Disable default sources list since we don't use them anyways
	mv /etc/apt/sources.list /etc/apt/sources.list.bak

	echo "deb https://apt.dockerproject.org/repo ubuntu-%[3]s main" > /etc/apt/sources.list.d/docker.list
	apt-get update
	apt-get install docker-engine=1.9.1-0~%[3]s -y --force-yes
	systemctl stop docker.service
}

echo -n "Start Boot Script: " >> /var/log/bootscript.log
date >> /var/log/bootscript.log

USER_DIR=/home/%[2]s
export DEBIAN_FRONTEND=noninteractive

install_docker
initialize_ovs
initialize_docker
initialize_minion

# Reload because we replaced the docker.service provided by the package
systemctl daemon-reload

# Enable our services to run on boot
systemctl enable {ovs,docker,minion}.service

# Start our services
systemctl restart {ovs,docker,minion}.service

# Create dirs and files with correct users and permissions
install -d -o %[2]s -m 700 $USER_DIR/.ssh
install -o %[2]s -m 600 /dev/null $USER_DIR/.ssh/authorized_keys

# allow the user to use docker without sudo
usermod -aG docker %[2]s

echo -n "Completed Boot Script: " >> /var/log/bootscript.log
date >> /var/log/bootscript.log
    `
	cloudConfig = fmt.Sprintf(cloudConfig, minionImage, user, ubuntuVersion)

	if len(keys) > 0 {
		for _, key := range keys {
			cloudConfig += fmt.Sprintf("echo %s >> $USER_DIR/.ssh/authorized_keys \n", key)
		}
	}

	return cloudConfig
}

func cloudConfigCoreOS(keys []string) string {
	cloudConfig := `#cloud-config

coreos:
    units:
        - name: ovs.service
          command: start
          content: |
            [Unit]
            Description=OVS
            [Service]
            ExecStart=/sbin/modprobe openvswitch
            ExecStartPost=/sbin/modprobe vport_geneve

        - name: docker.service
          command: start
          content: |
            [Unit]
            Description=docker
            [Service]
            ExecStart=/usr/bin/docker daemon --bridge=none \
            -H $private_ipv4:2375 -H unix:///var/run/docker.sock \
            --cluster-store=etcd://127.0.0.1:2379 --cluster-advertise=$private_ipv4:0

        - name: minion.service
          command: start
          content: |
            [Unit]
            Description=DI Minion
            After=docker.service
            Requires=docker.service

            [Service]
            TimeoutSec=1000
            ExecStartPre=-/usr/bin/mkdir -p /var/run/netns
            ExecStartPre=-/usr/bin/docker kill minion
            ExecStartPre=-/usr/bin/docker rm minion
            ExecStartPre=/usr/bin/docker pull %s
            ExecStart=/usr/bin/docker run --net=host --name=minion --privileged \
            -v /var/run/docker.sock:/var/run/docker.sock \
            -v /proc:/hostproc:ro -v /var/run/netns:/var/run/netns:rw

`
	cloudConfig = fmt.Sprintf(cloudConfig, minionImage, minionImage)

	if len(keys) > 0 {
		cloudConfig += "ssh_authorized_keys:\n"
		for _, key := range keys {
			cloudConfig += fmt.Sprintf("    - \"%s\"\n", key)
		}
	}

	return cloudConfig
}
<commit_msg>vagrant: Properly get the internal IP for docker<commit_after>package provider

import "fmt"

const (
	minionImage = "quay.io/netsys/di-minion:latest"
)

func cloudConfigUbuntu(keys []string, user string, ubuntuVersion string) string {
	cloudConfig := `#!/bin/bash

initialize_ovs() {
	cat <<- EOF > /etc/systemd/system/ovs.service
	[Unit]
	Description=OVS

	[Service]
	ExecStart=/sbin/modprobe openvswitch
	ExecStartPost=/sbin/modprobe vport_geneve

	[Install]
	WantedBy=multi-user.target
	EOF
}

initialize_docker() {
	# If getting the AWS internal IP works, then use that; otherwise, manually
	# parse it ourselves.
	PRIVATE_IPv4="$(curl -s --connect-timeout 5 http://instance-data/latest/meta-data/local-ipv4)"
	if [ $? -ne 0 ] ; then
		PRIVATE_IPv4="$(ip address show eth1 | grep 'inet ' | sed -e 's/^.*inet //' -e 's/\/.*$//' | tr -d '\n')"
	fi

	mkdir -p /etc/systemd/system/docker.service.d

	cat <<- EOF > /etc/systemd/system/docker.service.d/override.conf
	[Unit]
	Description=docker

	[Service]
	ExecStart=
	ExecStart=/usr/bin/docker daemon --bridge=none \
	-H "${PRIVATE_IPv4}:2375" -H unix:///var/run/docker.sock \

	[Install]
	WantedBy=multi-user.target
	EOF
}

initialize_minion() {
	cat <<- EOF > /etc/systemd/system/minion.service
	[Unit]
	Description=DI Minion
	After=docker.service
	Requires=docker.service

	[Service]
	TimeoutSec=1000
	ExecStartPre=-/usr/bin/mkdir -p /var/run/netns
	ExecStartPre=-/usr/bin/docker kill minion
	ExecStartPre=-/usr/bin/docker rm minion
	ExecStartPre=/usr/bin/docker pull %[1]s
	ExecStart=/usr/bin/docker run --net=host --name=minion --privileged \
	-v /var/run/docker.sock:/var/run/docker.sock \
	-v /proc:/hostproc:ro -v /var/run/netns:/var/run/netns:rw %[1]s

	[Install]
	WantedBy=multi-user.target
	EOF
}

install_docker() {
	# Disable default sources list since we don't use them anyways
	mv /etc/apt/sources.list /etc/apt/sources.list.bak

	echo "deb https://apt.dockerproject.org/repo ubuntu-%[3]s main" > /etc/apt/sources.list.d/docker.list
	apt-get update
	apt-get install docker-engine=1.9.1-0~%[3]s -y --force-yes
	systemctl stop docker.service
}

echo -n "Start Boot Script: " >> /var/log/bootscript.log
date >> /var/log/bootscript.log

USER_DIR=/home/%[2]s
export DEBIAN_FRONTEND=noninteractive

install_docker
initialize_ovs
initialize_docker
initialize_minion

# Reload because we replaced the docker.service provided by the package
systemctl daemon-reload

# Enable our services to run on boot
systemctl enable {ovs,docker,minion}.service

# Start our services
systemctl restart {ovs,docker,minion}.service

# Create dirs and files with correct users and permissions
install -d -o %[2]s -m 700 $USER_DIR/.ssh
install -o %[2]s -m 600 /dev/null $USER_DIR/.ssh/authorized_keys

# allow the user to use docker without sudo
usermod -aG docker %[2]s

echo -n "Completed Boot Script: " >> /var/log/bootscript.log
date >> /var/log/bootscript.log
    `
	cloudConfig = fmt.Sprintf(cloudConfig, minionImage, user, ubuntuVersion)

	if len(keys) > 0 {
		for _, key := range keys {
			cloudConfig += fmt.Sprintf("echo %s >> $USER_DIR/.ssh/authorized_keys \n", key)
		}
	}

	return cloudConfig
}

func cloudConfigCoreOS(keys []string) string {
	cloudConfig := `#cloud-config

coreos:
    units:
        - name: ovs.service
          command: start
          content: |
            [Unit]
            Description=OVS
            [Service]
            ExecStart=/sbin/modprobe openvswitch
            ExecStartPost=/sbin/modprobe vport_geneve

        - name: docker.service
          command: start
          content: |
            [Unit]
            Description=docker
            [Service]
            ExecStart=/usr/bin/docker daemon --bridge=none \
            -H $private_ipv4:2375 -H unix:///var/run/docker.sock \
            --cluster-store=etcd://127.0.0.1:2379 --cluster-advertise=$private_ipv4:0

        - name: minion.service
          command: start
          content: |
            [Unit]
            Description=DI Minion
            After=docker.service
            Requires=docker.service

            [Service]
            TimeoutSec=1000
            ExecStartPre=-/usr/bin/mkdir -p /var/run/netns
            ExecStartPre=-/usr/bin/docker kill minion
            ExecStartPre=-/usr/bin/docker rm minion
            ExecStartPre=/usr/bin/docker pull %s
            ExecStart=/usr/bin/docker run --net=host --name=minion --privileged \
            -v /var/run/docker.sock:/var/run/docker.sock \
            -v /proc:/hostproc:ro -v /var/run/netns:/var/run/netns:rw

`
	cloudConfig = fmt.Sprintf(cloudConfig, minionImage, minionImage)

	if len(keys) > 0 {
		cloudConfig += "ssh_authorized_keys:\n"
		for _, key := range keys {
			cloudConfig += fmt.Sprintf("    - \"%s\"\n", key)
		}
	}

	return cloudConfig
}
<|endoftext|>


<commit_before>//   Copyright 2016 DigitalOcean
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

// Command ceph_exporter provides a Prometheus exporter for a Ceph cluster.
package main

import (
	"flag"
	"log"
	"net/http"
	"sync"

	"github.com/digitalocean/ceph_exporter/collectors"

	"github.com/ceph/go-ceph/rados"
	"github.com/prometheus/client_golang/prometheus"
)

// CephExporter wraps all the ceph collectors and provides a single global
// exporter to extracts metrics out of. It also ensures that the collection
// is done in a thread-safe manner, the necessary requirement stated by
// prometheus. It also implements a prometheus.Collector interface in order
// to register it correctly.
type CephExporter struct {
	mu         sync.Mutex
	collectors []prometheus.Collector
}

// Verify that the exporter implements the interface correctly.
var _ prometheus.Collector = &CephExporter{}

// NewCephExporter creates an instance to CephExporter and returns a reference
// to it. We can choose to enable a collector to extract stats out of by adding
// it to the list of collectors.
func NewCephExporter(conn *rados.Conn) *CephExporter {
	return &CephExporter{
		collectors: []prometheus.Collector{
			collectors.NewClusterUsageCollector(conn),
			collectors.NewPoolUsageCollector(conn),
			collectors.NewClusterHealthCollector(conn),
			collectors.NewMonitorCollector(conn),
			collectors.NewDfCollector(conn),
		},
	}
}

// Describe sends all the descriptors of the collectors included to
// the provided channel.
func (c *CephExporter) Describe(ch chan<- *prometheus.Desc) {
	for _, cc := range c.collectors {
		cc.Describe(ch)
	}
}

// Collect sends the collected metrics from each of the collectors to
// prometheus. Collect could be called several times concurrently
// and thus its run is protected by a single mutex.
func (c *CephExporter) Collect(ch chan<- prometheus.Metric) {
	c.mu.Lock()
	defer c.mu.Unlock()

	for _, cc := range c.collectors {
		cc.Collect(ch)
	}
}

func main() {
	var (
		addr        = flag.String("telemetry.addr", ":9128", "host:port for ceph exporter")
		metricsPath = flag.String("telemetry.path", "/metrics", "URL path for surfacing collected metrics")

		cephConfig = flag.String("ceph.config", "", "path to ceph config file")
	)
	flag.Parse()

	conn, err := rados.NewConn()
	if err != nil {
		log.Fatalf("cannot create new ceph connection: %s", err)
	}

	if *cephConfig != "" {
		err = conn.ReadConfigFile(*cephConfig)
	} else {
		err = conn.ReadDefaultConfigFile()
	}
	if err != nil {
		log.Fatalf("cannot read ceph config file: %s", err)
	}

	if err := conn.Connect(); err != nil {
		log.Fatalf("cannot connect to ceph cluster: %s", err)
	}
	defer conn.Shutdown()

	prometheus.MustRegister(NewCephExporter(conn))

	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, *metricsPath, http.StatusMovedPermanently)
	})

	log.Printf("Starting ceph exporter on %q", *addr)
	if err := http.ListenAndServe(*addr, nil); err != nil {
		log.Fatalf("cannot start ceph exporter: %s", err)
	}
}
<commit_msg>remove this failing line<commit_after>//   Copyright 2016 DigitalOcean
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

// Command ceph_exporter provides a Prometheus exporter for a Ceph cluster.
package main

import (
	"flag"
	"log"
	"net/http"
	"sync"

	"github.com/digitalocean/ceph_exporter/collectors"

	"github.com/ceph/go-ceph/rados"
	"github.com/prometheus/client_golang/prometheus"
)

// CephExporter wraps all the ceph collectors and provides a single global
// exporter to extracts metrics out of. It also ensures that the collection
// is done in a thread-safe manner, the necessary requirement stated by
// prometheus. It also implements a prometheus.Collector interface in order
// to register it correctly.
type CephExporter struct {
	mu         sync.Mutex
	collectors []prometheus.Collector
}

// Verify that the exporter implements the interface correctly.
var _ prometheus.Collector = &CephExporter{}

// NewCephExporter creates an instance to CephExporter and returns a reference
// to it. We can choose to enable a collector to extract stats out of by adding
// it to the list of collectors.
func NewCephExporter(conn *rados.Conn) *CephExporter {
	return &CephExporter{
		collectors: []prometheus.Collector{
			collectors.NewClusterUsageCollector(conn),
			collectors.NewPoolUsageCollector(conn),
			collectors.NewClusterHealthCollector(conn),
			collectors.NewMonitorCollector(conn),
		},
	}
}

// Describe sends all the descriptors of the collectors included to
// the provided channel.
func (c *CephExporter) Describe(ch chan<- *prometheus.Desc) {
	for _, cc := range c.collectors {
		cc.Describe(ch)
	}
}

// Collect sends the collected metrics from each of the collectors to
// prometheus. Collect could be called several times concurrently
// and thus its run is protected by a single mutex.
func (c *CephExporter) Collect(ch chan<- prometheus.Metric) {
	c.mu.Lock()
	defer c.mu.Unlock()

	for _, cc := range c.collectors {
		cc.Collect(ch)
	}
}

func main() {
	var (
		addr        = flag.String("telemetry.addr", ":9128", "host:port for ceph exporter")
		metricsPath = flag.String("telemetry.path", "/metrics", "URL path for surfacing collected metrics")

		cephConfig = flag.String("ceph.config", "", "path to ceph config file")
	)
	flag.Parse()

	conn, err := rados.NewConn()
	if err != nil {
		log.Fatalf("cannot create new ceph connection: %s", err)
	}

	if *cephConfig != "" {
		err = conn.ReadConfigFile(*cephConfig)
	} else {
		err = conn.ReadDefaultConfigFile()
	}
	if err != nil {
		log.Fatalf("cannot read ceph config file: %s", err)
	}

	if err := conn.Connect(); err != nil {
		log.Fatalf("cannot connect to ceph cluster: %s", err)
	}
	defer conn.Shutdown()

	prometheus.MustRegister(NewCephExporter(conn))

	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, *metricsPath, http.StatusMovedPermanently)
	})

	log.Printf("Starting ceph exporter on %q", *addr)
	if err := http.ListenAndServe(*addr, nil); err != nil {
		log.Fatalf("cannot start ceph exporter: %s", err)
	}
}
<|endoftext|>


<commit_before>// GraphicsMagick processor
package graphicsmagick

import (
	"container/list"
	"fmt"
	"github.com/pierrre/imageserver"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"time"
)

const (
	globalParameterName = "graphicsmagick"
	tempDirPrefix       = "imageserver_"
)

// Processes an image with GraphicsMagick command line (mogrify command)
//
// All parameters are extracted from the "graphicsmagick" node parameter and are optionals.
//
// See GraphicsMagick documentation for more information about arguments.
//
// Parameters
//
// - width / height: sizes for "-resize" argument (both optionals)
//
// - fill: "^" for "-resize" argument
//
// - ignore_ratio: "!" for "-resize" argument
//
// - only_shrink_larger: ">" for "-resize" argument
//
// - only_enlarge_smaller: "<" for "-resize" argument
//
// - background: color for "-background" argument, 3/4/6/8 lower case hexadecimal characters
//
// - extent: "-extent" parameter, uses width/height parameters and add "-gravity center" argument
//
// - format: "-format" parameter
//
// - quality: "-quality" parameter
type GraphicsMagickProcessor struct {
	Executable string // path to "gm" executable, usually "/usr/bin/gm"

	Timeout          time.Duration     // timeout for process, optional
	TempDir          string            // temp directory for image files, optional
	AllowedFormats   []string          // allowed format list, optional
	DefaultQualities map[string]string // default qualities by format, optional
}

func (processor *GraphicsMagickProcessor) Process(sourceImage *imageserver.Image, parameters imageserver.Parameters) (*imageserver.Image, error) {
	parameters, err := processor.getParameters(parameters)
	if err != nil {
		return nil, err
	}
	if parameters == nil || parameters.Empty() {
		return sourceImage, nil
	}

	arguments := list.New()

	width, height, err := processor.buildArgumentsResize(arguments, parameters)
	if err != nil {
		return nil, err
	}

	err = processor.buildArgumentsBackground(arguments, parameters)
	if err != nil {
		return nil, err
	}

	err = processor.buildArgumentsExtent(arguments, parameters, width, height)
	if err != nil {
		return nil, err
	}

	format, formatSpecified, err := processor.buildArgumentsFormat(arguments, parameters, sourceImage)
	if err != nil {
		return nil, err
	}

	err = processor.buildArgumentsQuality(arguments, parameters, format)
	if err != nil {
		return nil, err
	}

	if arguments.Len() == 0 {
		return sourceImage, nil
	}

	arguments.PushFront("mogrify")

	tempDir, err := ioutil.TempDir(processor.TempDir, tempDirPrefix)
	if err != nil {
		return nil, err
	}
	defer os.RemoveAll(tempDir)

	file := filepath.Join(tempDir, "image")
	arguments.PushBack(file)
	err = ioutil.WriteFile(file, sourceImage.Data, os.FileMode(0600))
	if err != nil {
		return nil, err
	}

	argumentSlice := processor.convertArgumentsToSlice(arguments)

	cmd := exec.Command(processor.Executable, argumentSlice...)

	err = processor.runCommandTimeout(cmd)
	if err != nil {
		return nil, err
	}

	if formatSpecified {
		file = fmt.Sprintf("%s.%s", file, format)
	}
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}

	image := &imageserver.Image{
		Type: format,
		Data: data,
	}

	return image, nil
}

func (processor *GraphicsMagickProcessor) getParameters(parameters imageserver.Parameters) (imageserver.Parameters, error) {
	if !parameters.Has(globalParameterName) {
		return nil, nil
	}

	return parameters.GetParameters(globalParameterName)
}

func (processor *GraphicsMagickProcessor) buildArgumentsResize(arguments *list.List, parameters imageserver.Parameters) (width int, height int, err error) {
	width, _ = parameters.GetInt("width")
	if width < 0 {
		return 0, 0, imageserver.NewError("Invalid width parameter")
	}

	height, _ = parameters.GetInt("height")
	if height < 0 {
		return 0, 0, imageserver.NewError("Invalid height parameter")
	}

	if width != 0 || height != 0 {
		widthString := ""
		if width != 0 {
			widthString = strconv.Itoa(width)
		}
		heightString := ""
		if height != 0 {
			heightString = strconv.Itoa(height)
		}
		resize := fmt.Sprintf("%sx%s", widthString, heightString)

		if fill, _ := parameters.GetBool("fill"); fill {
			resize = resize + "^"
		}

		if ignoreRatio, _ := parameters.GetBool("ignore_ratio"); ignoreRatio {
			resize = resize + "!"
		}

		if onlyShrinkLarger, _ := parameters.GetBool("only_shrink_larger"); onlyShrinkLarger {
			resize = resize + ">"
		}

		if onlyEnlargeSmaller, _ := parameters.GetBool("only_enlarge_smaller"); onlyEnlargeSmaller {
			resize = resize + "<"
		}

		arguments.PushBack("-resize")
		arguments.PushBack(resize)
	}

	return width, height, nil
}

func (processor *GraphicsMagickProcessor) buildArgumentsBackground(arguments *list.List, parameters imageserver.Parameters) error {
	background, _ := parameters.GetString("background")

	if backgroundLength := len(background); backgroundLength > 0 {
		if backgroundLength != 6 && backgroundLength != 8 && backgroundLength != 3 && backgroundLength != 4 {
			return imageserver.NewError("Invalid background parameter")
		}

		for _, r := range background {
			if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
				return imageserver.NewError("Invalid background parameter")
			}
		}

		arguments.PushBack("-background")
		arguments.PushBack(fmt.Sprintf("#%s", background))
	}

	return nil
}

func (processor *GraphicsMagickProcessor) buildArgumentsExtent(arguments *list.List, parameters imageserver.Parameters, width int, height int) error {
	if width != 0 && height != 0 {
		if extent, _ := parameters.GetBool("extent"); extent {
			arguments.PushBack("-gravity")
			arguments.PushBack("center")

			arguments.PushBack("-extent")
			arguments.PushBack(fmt.Sprintf("%dx%d", width, height))
		}
	}

	return nil
}

func (processor *GraphicsMagickProcessor) buildArgumentsFormat(arguments *list.List, parameters imageserver.Parameters, sourceImage *imageserver.Image) (format string, formatSpecified bool, err error) {
	format, _ = parameters.GetString("format")

	formatSpecified = true
	if len(format) == 0 {
		format = sourceImage.Type
		formatSpecified = false
	}

	if processor.AllowedFormats != nil {
		ok := false
		for _, f := range processor.AllowedFormats {
			if f == format {
				ok = true
				break
			}
		}
		if !ok {
			return "", false, imageserver.NewError("Invalid format parameter")
		}
	}

	if formatSpecified {
		arguments.PushBack("-format")
		arguments.PushBack(format)
	}

	return format, formatSpecified, nil
}

func (processor *GraphicsMagickProcessor) buildArgumentsQuality(arguments *list.List, parameters imageserver.Parameters, format string) error {
	quality, _ := parameters.GetString("quality")

	if len(quality) == 0 && arguments.Len() == 0 {
		return nil
	}

	if len(quality) == 0 && processor.DefaultQualities != nil {
		if q, ok := processor.DefaultQualities[format]; ok {
			quality = q
		}
	}

	if len(quality) > 0 {
		qualityInt, err := strconv.Atoi(quality)
		if err != nil {
			return imageserver.NewError("Invalid quality parameter (parse int error)")
		}

		if qualityInt < 0 {
			return imageserver.NewError("Invalid quality parameter (less than 0)")
		}

		if format == "jpeg" {
			if qualityInt < 0 || qualityInt > 100 {
				return imageserver.NewError("Invalid quality parameter (must be between 0 and 100)")
			}
		}

		arguments.PushBack("-quality")
		arguments.PushBack(quality)
	}

	return nil
}

func (processor *GraphicsMagickProcessor) convertArgumentsToSlice(arguments *list.List) []string {
	argumentSlice := make([]string, 0, arguments.Len())
	for e := arguments.Front(); e != nil; e = e.Next() {
		argumentSlice = append(argumentSlice, e.Value.(string))
	}
	return argumentSlice
}

func (processor *GraphicsMagickProcessor) runCommandTimeout(cmd *exec.Cmd) error {
	err := cmd.Start()
	if err != nil {
		return err
	}

	cmdChan := make(chan error)
	go func() {
		cmdChan <- cmd.Wait()
	}()

	var timeoutChan <-chan time.Time
	if processor.Timeout != 0 {
		timeoutChan = time.After(processor.Timeout)
	}

	select {
	case err = <-cmdChan:
		if err != nil {
			return err
		}

		return nil
	case <-timeoutChan:
		err = cmd.Process.Kill()
		if err != nil {
			return err
		}

		<-cmdChan

		return fmt.Errorf("GraphicsMagickProcessor command timeout after %s: %+v", processor.Timeout, cmd)
	}
}
<commit_msg>rename function<commit_after>// GraphicsMagick processor
package graphicsmagick

import (
	"container/list"
	"fmt"
	"github.com/pierrre/imageserver"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"time"
)

const (
	globalParameterName = "graphicsmagick"
	tempDirPrefix       = "imageserver_"
)

// Processes an image with GraphicsMagick command line (mogrify command)
//
// All parameters are extracted from the "graphicsmagick" node parameter and are optionals.
//
// See GraphicsMagick documentation for more information about arguments.
//
// Parameters
//
// - width / height: sizes for "-resize" argument (both optionals)
//
// - fill: "^" for "-resize" argument
//
// - ignore_ratio: "!" for "-resize" argument
//
// - only_shrink_larger: ">" for "-resize" argument
//
// - only_enlarge_smaller: "<" for "-resize" argument
//
// - background: color for "-background" argument, 3/4/6/8 lower case hexadecimal characters
//
// - extent: "-extent" parameter, uses width/height parameters and add "-gravity center" argument
//
// - format: "-format" parameter
//
// - quality: "-quality" parameter
type GraphicsMagickProcessor struct {
	Executable string // path to "gm" executable, usually "/usr/bin/gm"

	Timeout          time.Duration     // timeout for process, optional
	TempDir          string            // temp directory for image files, optional
	AllowedFormats   []string          // allowed format list, optional
	DefaultQualities map[string]string // default qualities by format, optional
}

func (processor *GraphicsMagickProcessor) Process(sourceImage *imageserver.Image, parameters imageserver.Parameters) (*imageserver.Image, error) {
	parameters, err := processor.getParameters(parameters)
	if err != nil {
		return nil, err
	}
	if parameters == nil || parameters.Empty() {
		return sourceImage, nil
	}

	arguments := list.New()

	width, height, err := processor.buildArgumentsResize(arguments, parameters)
	if err != nil {
		return nil, err
	}

	err = processor.buildArgumentsBackground(arguments, parameters)
	if err != nil {
		return nil, err
	}

	err = processor.buildArgumentsExtent(arguments, parameters, width, height)
	if err != nil {
		return nil, err
	}

	format, formatSpecified, err := processor.buildArgumentsFormat(arguments, parameters, sourceImage)
	if err != nil {
		return nil, err
	}

	err = processor.buildArgumentsQuality(arguments, parameters, format)
	if err != nil {
		return nil, err
	}

	if arguments.Len() == 0 {
		return sourceImage, nil
	}

	arguments.PushFront("mogrify")

	tempDir, err := ioutil.TempDir(processor.TempDir, tempDirPrefix)
	if err != nil {
		return nil, err
	}
	defer os.RemoveAll(tempDir)

	file := filepath.Join(tempDir, "image")
	arguments.PushBack(file)
	err = ioutil.WriteFile(file, sourceImage.Data, os.FileMode(0600))
	if err != nil {
		return nil, err
	}

	argumentSlice := processor.convertArgumentsToSlice(arguments)

	cmd := exec.Command(processor.Executable, argumentSlice...)

	err = processor.runCommand(cmd)
	if err != nil {
		return nil, err
	}

	if formatSpecified {
		file = fmt.Sprintf("%s.%s", file, format)
	}
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}

	image := &imageserver.Image{
		Type: format,
		Data: data,
	}

	return image, nil
}

func (processor *GraphicsMagickProcessor) getParameters(parameters imageserver.Parameters) (imageserver.Parameters, error) {
	if !parameters.Has(globalParameterName) {
		return nil, nil
	}

	return parameters.GetParameters(globalParameterName)
}

func (processor *GraphicsMagickProcessor) buildArgumentsResize(arguments *list.List, parameters imageserver.Parameters) (width int, height int, err error) {
	width, _ = parameters.GetInt("width")
	if width < 0 {
		return 0, 0, imageserver.NewError("Invalid width parameter")
	}

	height, _ = parameters.GetInt("height")
	if height < 0 {
		return 0, 0, imageserver.NewError("Invalid height parameter")
	}

	if width != 0 || height != 0 {
		widthString := ""
		if width != 0 {
			widthString = strconv.Itoa(width)
		}
		heightString := ""
		if height != 0 {
			heightString = strconv.Itoa(height)
		}
		resize := fmt.Sprintf("%sx%s", widthString, heightString)

		if fill, _ := parameters.GetBool("fill"); fill {
			resize = resize + "^"
		}

		if ignoreRatio, _ := parameters.GetBool("ignore_ratio"); ignoreRatio {
			resize = resize + "!"
		}

		if onlyShrinkLarger, _ := parameters.GetBool("only_shrink_larger"); onlyShrinkLarger {
			resize = resize + ">"
		}

		if onlyEnlargeSmaller, _ := parameters.GetBool("only_enlarge_smaller"); onlyEnlargeSmaller {
			resize = resize + "<"
		}

		arguments.PushBack("-resize")
		arguments.PushBack(resize)
	}

	return width, height, nil
}

func (processor *GraphicsMagickProcessor) buildArgumentsBackground(arguments *list.List, parameters imageserver.Parameters) error {
	background, _ := parameters.GetString("background")

	if backgroundLength := len(background); backgroundLength > 0 {
		if backgroundLength != 6 && backgroundLength != 8 && backgroundLength != 3 && backgroundLength != 4 {
			return imageserver.NewError("Invalid background parameter")
		}

		for _, r := range background {
			if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
				return imageserver.NewError("Invalid background parameter")
			}
		}

		arguments.PushBack("-background")
		arguments.PushBack(fmt.Sprintf("#%s", background))
	}

	return nil
}

func (processor *GraphicsMagickProcessor) buildArgumentsExtent(arguments *list.List, parameters imageserver.Parameters, width int, height int) error {
	if width != 0 && height != 0 {
		if extent, _ := parameters.GetBool("extent"); extent {
			arguments.PushBack("-gravity")
			arguments.PushBack("center")

			arguments.PushBack("-extent")
			arguments.PushBack(fmt.Sprintf("%dx%d", width, height))
		}
	}

	return nil
}

func (processor *GraphicsMagickProcessor) buildArgumentsFormat(arguments *list.List, parameters imageserver.Parameters, sourceImage *imageserver.Image) (format string, formatSpecified bool, err error) {
	format, _ = parameters.GetString("format")

	formatSpecified = true
	if len(format) == 0 {
		format = sourceImage.Type
		formatSpecified = false
	}

	if processor.AllowedFormats != nil {
		ok := false
		for _, f := range processor.AllowedFormats {
			if f == format {
				ok = true
				break
			}
		}
		if !ok {
			return "", false, imageserver.NewError("Invalid format parameter")
		}
	}

	if formatSpecified {
		arguments.PushBack("-format")
		arguments.PushBack(format)
	}

	return format, formatSpecified, nil
}

func (processor *GraphicsMagickProcessor) buildArgumentsQuality(arguments *list.List, parameters imageserver.Parameters, format string) error {
	quality, _ := parameters.GetString("quality")

	if len(quality) == 0 && arguments.Len() == 0 {
		return nil
	}

	if len(quality) == 0 && processor.DefaultQualities != nil {
		if q, ok := processor.DefaultQualities[format]; ok {
			quality = q
		}
	}

	if len(quality) > 0 {
		qualityInt, err := strconv.Atoi(quality)
		if err != nil {
			return imageserver.NewError("Invalid quality parameter (parse int error)")
		}

		if qualityInt < 0 {
			return imageserver.NewError("Invalid quality parameter (less than 0)")
		}

		if format == "jpeg" {
			if qualityInt < 0 || qualityInt > 100 {
				return imageserver.NewError("Invalid quality parameter (must be between 0 and 100)")
			}
		}

		arguments.PushBack("-quality")
		arguments.PushBack(quality)
	}

	return nil
}

func (processor *GraphicsMagickProcessor) convertArgumentsToSlice(arguments *list.List) []string {
	argumentSlice := make([]string, 0, arguments.Len())
	for e := arguments.Front(); e != nil; e = e.Next() {
		argumentSlice = append(argumentSlice, e.Value.(string))
	}
	return argumentSlice
}

func (processor *GraphicsMagickProcessor) runCommand(cmd *exec.Cmd) error {
	err := cmd.Start()
	if err != nil {
		return err
	}

	cmdChan := make(chan error)
	go func() {
		cmdChan <- cmd.Wait()
	}()

	var timeoutChan <-chan time.Time
	if processor.Timeout != 0 {
		timeoutChan = time.After(processor.Timeout)
	}

	select {
	case err = <-cmdChan:
		if err != nil {
			return err
		}

		return nil
	case <-timeoutChan:
		err = cmd.Process.Kill()
		if err != nil {
			return err
		}

		<-cmdChan

		return fmt.Errorf("GraphicsMagickProcessor command timeout after %s: %+v", processor.Timeout, cmd)
	}
}
<|endoftext|>