<commit_before>package main

import (
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/jeffail/gabs"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
)

const namespace = "chronos"

type Exporter struct {
	scraper      Scraper
	mapper       *Mapper
	duration     prometheus.Gauge
	scrapeError  prometheus.Gauge
	totalErrors  prometheus.Counter
	totalScrapes prometheus.Counter
	Counters     *CounterContainer
	Gauges       *GaugeContainer
}

// Describe implements prometheus.Collector.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	log.Debugln("Describing metrics")
	metricCh := make(chan prometheus.Metric)
	doneCh := make(chan struct{})

	go func() {
		for m := range metricCh {
			ch <- m.Desc()
		}
		close(doneCh)
	}()

	e.Collect(metricCh)
	close(metricCh)
	<-doneCh
}

// Collect implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	log.Debugln("Collecting metrics")
	e.scrape(ch)

	ch <- e.duration
	ch <- e.totalScrapes
	ch <- e.totalErrors
	ch <- e.scrapeError
}

func (e *Exporter) scrape(ch chan<- prometheus.Metric) {
	e.totalScrapes.Inc()

	var err error
	defer func(begin time.Time) {
		e.duration.Set(time.Since(begin).Seconds())
		if err == nil {
			e.scrapeError.Set(0)
		} else {
			e.totalErrors.Inc()
			e.scrapeError.Set(1)
		}
	}(time.Now())

	content, err := e.scraper.Scrape()
	if err != nil {
		log.Debugf("Problem scraping metrics endpoint: %v\n", err)
		return
	}

	json, err := gabs.ParseJSON(content)
	if err != nil {
		log.Debugf("Problem parsing metrics response: %v\n", err)
		return
	}

	e.scrapeMetrics(json, ch)
}

func (e *Exporter) scrapeMetrics(json *gabs.Container, ch chan<- prometheus.Metric) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		switch key {
		case "message":
			log.Errorf("Problem collecting metrics: %s\n", element.Data().(string))
			return
		case "version":
			data := element.Data()
			version, ok := data.(string)
			if !ok {
				log.Errorf(fmt.Sprintf("Bad conversion! Unexpected value \"%v\" for version\n", data))
			} else {
				gauge, _ := e.Gauges.Fetch("metrics_version", "Chronos metrics version", "version")
				gauge.WithLabelValues(version).Set(1)
				gauge.Collect(ch)
			}

		case "counters":
			e.scrapeCounters(element)
		case "gauges":
			e.scrapeGauges(element)
		case "histograms":
			e.scrapeHistograms(element)
		case "meters":
			e.scrapeMeters(element)
		case "timers":
			e.scrapeTimers(element)
		}
	}

	for _, counter := range e.Counters.counters {
		counter.Collect(ch)
	}
	for _, gauge := range e.Gauges.gauges {
		gauge.Collect(ch)
	}
}

func (e *Exporter) scrapeCounters(json *gabs.Container) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		new, err := e.scrapeCounter(key, element)
		if err != nil {
			log.Debug(err)
		} else if new {
			log.Infof("Added counter %q\n", key)
		}
	}
}

func (e *Exporter) scrapeCounter(metric string, json *gabs.Container) (bool, error) {
	data := json.Path("count").Data()
	count, ok := data.(float64)
	if !ok {
		return false, errors.New(fmt.Sprintf("Bad conversion! Unexpected value \"%v\" for counter %s\n", data, metric))
	}

	counter, new := e.mapper.counter(metric)
	counter.With(e.mapper.labels(metric)).Set(count)
	return new, nil
}

func (e *Exporter) scrapeGauges(json *gabs.Container) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		new, err := e.scrapeGauge(key, element)
		if err != nil {
			log.Debug(err)
		} else if new {
			log.Infof("Added gauge %q\n", key)
		}
	}
}

func (e *Exporter) scrapeGauge(metric string, json *gabs.Container) (bool, error) {
	data := json.Path("value").Data()
	value, ok := data.(float64)
	if !ok {
		return false, errors.New(fmt.Sprintf("Bad conversion! Unexpected value \"%v\" for gauge %s\n", data, metric))
	}

	gauge, new := e.mapper.counter(metric)
	gauge.With(e.mapper.labels(metric)).Set(value)
	return new, nil
}

func (e *Exporter) scrapeMeters(json *gabs.Container) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		new, err := e.scrapeMeter(key, element)
		if err != nil {
			log.Debug(err)
		} else if new {
			log.Infof("Added meter %q\n", key)
		}
	}
}

func (e *Exporter) scrapeMeter(metric string, json *gabs.Container) (bool, error) {
	count, ok := json.Path("count").Data().(float64)
	if !ok {
		return false, errors.New(fmt.Sprintf("Bad meter! %s has no count\n", metric))
	}
	units, ok := json.Path("units").Data().(string)
	if !ok {
		return false, errors.New(fmt.Sprintf("Bad meter! %s has no units\n", metric))
	}

	counter, rates, new := e.mapper.meter(metric, units)
	counter.WithLabelValues().Set(count)

	properties, _ := json.ChildrenMap()
	for key, property := range properties {
		if strings.Contains(key, "rate") {
			if value, ok := property.Data().(float64); ok {
				rates.WithLabelValues(renameRate(key)).Set(value)
			}
		}
	}

	return new, nil
}

func (e *Exporter) scrapeHistograms(json *gabs.Container) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		new, err := e.scrapeHistogram(key, element)
		if err != nil {
			log.Debug(err)
		} else if new {
			log.Infof("Added histogram %q\n", key)
		}
	}
}

func (e *Exporter) scrapeHistogram(metric string, json *gabs.Container) (bool, error) {
	count, ok := json.Path("count").Data().(float64)
	if !ok {
		return false, errors.New(fmt.Sprintf("Bad historgram! %s has no count\n", metric))
	}

	counter, percentiles, min, max, mean, stddev, new := e.mapper.histogram(metric)
	counter.With(e.mapper.labels(metric)).Set(count)

	properties, _ := json.ChildrenMap()
	for key, property := range properties {
		switch key {
		case "p50", "p75", "p95", "p98", "p99", "p999":
			if value, ok := property.Data().(float64); ok {
				percentiles.WithLabelValues(
					e.mapper.labelValues(metric, "0."+key[1:])...).Set(value)
			}
		case "min":
			if value, ok := property.Data().(float64); ok {
				min.With(e.mapper.labels(metric)).Set(value)
			}
		case "max":
			if value, ok := property.Data().(float64); ok {
				max.With(e.mapper.labels(metric)).Set(value)
			}
		case "mean":
			if value, ok := property.Data().(float64); ok {
				mean.With(e.mapper.labels(metric)).Set(value)
			}
		case "stddev":
			if value, ok := property.Data().(float64); ok {
				stddev.With(e.mapper.labels(metric)).Set(value)
			}
		}
	}

	return new, nil
}

func (e *Exporter) scrapeTimers(json *gabs.Container) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		new, err := e.scrapeTimer(key, element)
		if err != nil {
			log.Debug(err)
		} else if new {
			log.Infof("Added timer %q\n", key)
		}
	}
}

func (e *Exporter) scrapeTimer(metric string, json *gabs.Container) (bool, error) {
	count, ok := json.Path("count").Data().(float64)
	if !ok {
		return false, errors.New(fmt.Sprintf("Bad timer! %s has no count\n", metric))
	}
	units, ok := json.Path("rate_units").Data().(string)
	if !ok {
		return false, errors.New(fmt.Sprintf("Bad timer! %s has no units\n", metric))
	}

	counter, rates, percentiles, min, max, mean, stddev, new := e.mapper.timer(metric, units)
	counter.WithLabelValues().Set(count)

	properties, _ := json.ChildrenMap()
	for key, property := range properties {
		switch key {
		case "mean_rate", "m1_rate", "m5_rate", "m15_rate":
			if value, ok := property.Data().(float64); ok {
				rates.WithLabelValues(renameRate(key)).Set(value)
			}

		case "p50", "p75", "p95", "p98", "p99", "p999":
			if value, ok := property.Data().(float64); ok {
				percentiles.WithLabelValues("0." + key[1:]).Set(value)
			}
		case "min":
			if value, ok := property.Data().(float64); ok {
				min.WithLabelValues().Set(value)
			}
		case "max":
			if value, ok := property.Data().(float64); ok {
				max.WithLabelValues().Set(value)
			}
		case "mean":
			if value, ok := property.Data().(float64); ok {
				mean.WithLabelValues().Set(value)
			}
		case "stddev":
			if value, ok := property.Data().(float64); ok {
				stddev.WithLabelValues().Set(value)
			}
		}
	}

	return new, nil
}

func NewExporter(s Scraper) *Exporter {
	counters := NewCounterContainer()
	gauges := NewGaugeContainer()
	mapper := &Mapper{counters, gauges}
	return &Exporter{
		scraper:  s,
		mapper:   mapper,
		Counters: counters,
		Gauges:   gauges,
		duration: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "last_scrape_duration_seconds",
			Help:      "Duration of the last scrape of metrics from Chronos.",
		}),
		scrapeError: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "last_scrape_error",
			Help:      "Whether the last scrape of metrics from Chronos resulted in an error (1 for error, 0 for success).",
		}),
		totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "scrapes_total",
			Help:      "Total number of times Chronos was scraped for metrics.",
		}),
		totalErrors: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "errors_total",
			Help:      "Total number of times the exporter experienced errors collecting Chronos metrics.",
		}),
	}
}
<commit_msg>fix for issue #2<commit_after>package main

import (
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/jeffail/gabs"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
)

const namespace = "chronos"

type Exporter struct {
	scraper      Scraper
	mapper       *Mapper
	duration     prometheus.Gauge
	scrapeError  prometheus.Gauge
	totalErrors  prometheus.Counter
	totalScrapes prometheus.Counter
	Counters     *CounterContainer
	Gauges       *GaugeContainer
}

// Describe implements prometheus.Collector.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	log.Debugln("Describing metrics")
	metricCh := make(chan prometheus.Metric)
	doneCh := make(chan struct{})

	go func() {
		for m := range metricCh {
			ch <- m.Desc()
		}
		close(doneCh)
	}()

	e.Collect(metricCh)
	close(metricCh)
	<-doneCh
}

// Collect implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	log.Debugln("Collecting metrics")
	e.scrape(ch)

	ch <- e.duration
	ch <- e.totalScrapes
	ch <- e.totalErrors
	ch <- e.scrapeError
}

func (e *Exporter) scrape(ch chan<- prometheus.Metric) {
	e.totalScrapes.Inc()

	var err error
	defer func(begin time.Time) {
		e.duration.Set(time.Since(begin).Seconds())
		if err == nil {
			e.scrapeError.Set(0)
		} else {
			e.totalErrors.Inc()
			e.scrapeError.Set(1)
		}
	}(time.Now())

	content, err := e.scraper.Scrape()
	if err != nil {
		log.Debugf("Problem scraping metrics endpoint: %v\n", err)
		return
	}

	json, err := gabs.ParseJSON(content)
	if err != nil {
		log.Debugf("Problem parsing metrics response: %v\n", err)
		return
	}

	e.scrapeMetrics(json, ch)
}

func (e *Exporter) scrapeMetrics(json *gabs.Container, ch chan<- prometheus.Metric) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		switch key {
		case "message":
			log.Errorf("Problem collecting metrics: %s\n", element.Data().(string))
			return
		case "version":
			data := element.Data()
			version, ok := data.(string)
			if !ok {
				log.Errorf(fmt.Sprintf("Bad conversion! Unexpected value \"%v\" for version\n", data))
			} else {
				gauge, _ := e.Gauges.Fetch("metrics_version", "Chronos metrics version", "version")
				gauge.WithLabelValues(version).Set(1)
			}

		case "counters":
			e.scrapeCounters(element)
		case "gauges":
			e.scrapeGauges(element)
		case "histograms":
			e.scrapeHistograms(element)
		case "meters":
			e.scrapeMeters(element)
		case "timers":
			e.scrapeTimers(element)
		}
	}

	for _, counter := range e.Counters.counters {
		counter.Collect(ch)
	}
	for _, gauge := range e.Gauges.gauges {
		gauge.Collect(ch)
	}
}

func (e *Exporter) scrapeCounters(json *gabs.Container) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		new, err := e.scrapeCounter(key, element)
		if err != nil {
			log.Debug(err)
		} else if new {
			log.Infof("Added counter %q\n", key)
		}
	}
}

func (e *Exporter) scrapeCounter(metric string, json *gabs.Container) (bool, error) {
	data := json.Path("count").Data()
	count, ok := data.(float64)
	if !ok {
		return false, errors.New(fmt.Sprintf("Bad conversion! Unexpected value \"%v\" for counter %s\n", data, metric))
	}

	counter, new := e.mapper.counter(metric)
	counter.With(e.mapper.labels(metric)).Set(count)
	return new, nil
}

func (e *Exporter) scrapeGauges(json *gabs.Container) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		new, err := e.scrapeGauge(key, element)
		if err != nil {
			log.Debug(err)
		} else if new {
			log.Infof("Added gauge %q\n", key)
		}
	}
}

func (e *Exporter) scrapeGauge(metric string, json *gabs.Container) (bool, error) {
	data := json.Path("value").Data()
	value, ok := data.(float64)
	if !ok {
		return false, errors.New(fmt.Sprintf("Bad conversion! Unexpected value \"%v\" for gauge %s\n", data, metric))
	}

	gauge, new := e.mapper.counter(metric)
	gauge.With(e.mapper.labels(metric)).Set(value)
	return new, nil
}

func (e *Exporter) scrapeMeters(json *gabs.Container) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		new, err := e.scrapeMeter(key, element)
		if err != nil {
			log.Debug(err)
		} else if new {
			log.Infof("Added meter %q\n", key)
		}
	}
}

func (e *Exporter) scrapeMeter(metric string, json *gabs.Container) (bool, error) {
	count, ok := json.Path("count").Data().(float64)
	if !ok {
		return false, errors.New(fmt.Sprintf("Bad meter! %s has no count\n", metric))
	}
	units, ok := json.Path("units").Data().(string)
	if !ok {
		return false, errors.New(fmt.Sprintf("Bad meter! %s has no units\n", metric))
	}

	counter, rates, new := e.mapper.meter(metric, units)
	counter.WithLabelValues().Set(count)

	properties, _ := json.ChildrenMap()
	for key, property := range properties {
		if strings.Contains(key, "rate") {
			if value, ok := property.Data().(float64); ok {
				rates.WithLabelValues(renameRate(key)).Set(value)
			}
		}
	}

	return new, nil
}

func (e *Exporter) scrapeHistograms(json *gabs.Container) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		new, err := e.scrapeHistogram(key, element)
		if err != nil {
			log.Debug(err)
		} else if new {
			log.Infof("Added histogram %q\n", key)
		}
	}
}

func (e *Exporter) scrapeHistogram(metric string, json *gabs.Container) (bool, error) {
	count, ok := json.Path("count").Data().(float64)
	if !ok {
		return false, errors.New(fmt.Sprintf("Bad historgram! %s has no count\n", metric))
	}

	counter, percentiles, min, max, mean, stddev, new := e.mapper.histogram(metric)
	counter.With(e.mapper.labels(metric)).Set(count)

	properties, _ := json.ChildrenMap()
	for key, property := range properties {
		switch key {
		case "p50", "p75", "p95", "p98", "p99", "p999":
			if value, ok := property.Data().(float64); ok {
				percentiles.WithLabelValues(
					e.mapper.labelValues(metric, "0."+key[1:])...).Set(value)
			}
		case "min":
			if value, ok := property.Data().(float64); ok {
				min.With(e.mapper.labels(metric)).Set(value)
			}
		case "max":
			if value, ok := property.Data().(float64); ok {
				max.With(e.mapper.labels(metric)).Set(value)
			}
		case "mean":
			if value, ok := property.Data().(float64); ok {
				mean.With(e.mapper.labels(metric)).Set(value)
			}
		case "stddev":
			if value, ok := property.Data().(float64); ok {
				stddev.With(e.mapper.labels(metric)).Set(value)
			}
		}
	}

	return new, nil
}

func (e *Exporter) scrapeTimers(json *gabs.Container) {
	elements, _ := json.ChildrenMap()
	for key, element := range elements {
		new, err := e.scrapeTimer(key, element)
		if err != nil {
			log.Debug(err)
		} else if new {
			log.Infof("Added timer %q\n", key)
		}
	}
}

func (e *Exporter) scrapeTimer(metric string, json *gabs.Container) (bool, error) {
	count, ok := json.Path("count").Data().(float64)
	if !ok {
		return false, errors.New(fmt.Sprintf("Bad timer! %s has no count\n", metric))
	}
	units, ok := json.Path("rate_units").Data().(string)
	if !ok {
		return false, errors.New(fmt.Sprintf("Bad timer! %s has no units\n", metric))
	}

	counter, rates, percentiles, min, max, mean, stddev, new := e.mapper.timer(metric, units)
	counter.WithLabelValues().Set(count)

	properties, _ := json.ChildrenMap()
	for key, property := range properties {
		switch key {
		case "mean_rate", "m1_rate", "m5_rate", "m15_rate":
			if value, ok := property.Data().(float64); ok {
				rates.WithLabelValues(renameRate(key)).Set(value)
			}

		case "p50", "p75", "p95", "p98", "p99", "p999":
			if value, ok := property.Data().(float64); ok {
				percentiles.WithLabelValues("0." + key[1:]).Set(value)
			}
		case "min":
			if value, ok := property.Data().(float64); ok {
				min.WithLabelValues().Set(value)
			}
		case "max":
			if value, ok := property.Data().(float64); ok {
				max.WithLabelValues().Set(value)
			}
		case "mean":
			if value, ok := property.Data().(float64); ok {
				mean.WithLabelValues().Set(value)
			}
		case "stddev":
			if value, ok := property.Data().(float64); ok {
				stddev.WithLabelValues().Set(value)
			}
		}
	}

	return new, nil
}

func NewExporter(s Scraper) *Exporter {
	counters := NewCounterContainer()
	gauges := NewGaugeContainer()
	mapper := &Mapper{counters, gauges}
	return &Exporter{
		scraper:  s,
		mapper:   mapper,
		Counters: counters,
		Gauges:   gauges,
		duration: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "last_scrape_duration_seconds",
			Help:      "Duration of the last scrape of metrics from Chronos.",
		}),
		scrapeError: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "last_scrape_error",
			Help:      "Whether the last scrape of metrics from Chronos resulted in an error (1 for error, 0 for success).",
		}),
		totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "scrapes_total",
			Help:      "Total number of times Chronos was scraped for metrics.",
		}),
		totalErrors: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: "exporter",
			Name:      "errors_total",
			Help:      "Total number of times the exporter experienced errors collecting Chronos metrics.",
		}),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/hashicorp/raft"
	"github.com/hashicorp/raft-mdb"
)

var (
	raftDir = flag.String("raftdir", "/tmp/r", "")
	peers   = flag.String("peers", "", "comma-separated host:port tuples")
	listen  = flag.String("listen", ":6000", "")
	listenhttp  = flag.String("listenhttp", ":8000", "")
	node    *raft.Raft
)

// Snapshot holds the data needed to serialize storage
type Snapshot struct {
	uuids   [][]byte
	entries []string
}

func uint32ToBytes(u uint32) []byte {
	buf := make([]byte, 4)
	binary.BigEndian.PutUint32(buf, u)
	return buf
}

// Converts bytes to an integer
func bytesToUint32(b []byte) uint32 {
	return binary.BigEndian.Uint32(b)
}

// Persist writes a snapshot to a file. We just serialize all active entries.
func (s *Snapshot) Persist(sink raft.SnapshotSink) error {
	_, err := sink.Write([]byte{0x0})
	if err != nil {
		sink.Cancel()
		return err
	}

	for i, e := range s.entries {
		_, err = sink.Write(s.uuids[i])
		if err != nil {
			sink.Cancel()
			return err
		}

		b := []byte(e)
		_, err = sink.Write(uint32ToBytes(uint32(len(b))))
		if err != nil {
			sink.Cancel()
			return err
		}

		_, err = sink.Write(b)
		if err != nil {
			sink.Cancel()
			return err
		}
	}

	return sink.Close()
}

// Release cleans up a snapshot. We don't need to do anything.
func (s *Snapshot) Release() {
}

type FSM struct {
	store *Storage
}

func (fsm *FSM) Apply(l *raft.Log) interface{} {
	log.Printf("TODO: apply %v\n", l)
	return nil
}

func (fsm *FSM) Snapshot() (raft.FSMSnapshot, error) {
	uuids, entries, err := fsm.store.GetAll()
	snapshot := &Snapshot{uuids, entries}
	return snapshot, err
}

func (fsm *FSM) Restore(snap io.ReadCloser) error {
	defer snap.Close()

	s, err := NewStorage()
	if err != nil {
		return err
	}

	// swap in the restored storage, with time emission channels.
	s.c = fsm.store.c
	s.C = fsm.store.C
	fsm.store.Close()
	fsm.store = s

	b := make([]byte, 1)
	_, err = snap.Read(b)
	if b[0] != byte(0x00) {
		msg := "Unknown snapshot schema version"
		log.Printf(msg)
		return fmt.Errorf(msg)
	}

	uuid := make([]byte, 16)
	size := make([]byte, 4)
	count := 0
	for {
		_, err = snap.Read(uuid)
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		_, err = snap.Read(size)
		if err != nil {
			return err
		}

		eb := make([]byte, bytesToUint32(size))
		_, err = snap.Read(eb)
		if err != nil {
			return err
		}
		e := string(eb)
		err = s.Add(uuid, e)
		if err != nil {
			return err
		}

		count++
	}

	log.Printf("Restored snapshot. entries=%d", count)
	return nil
}

func handleRequest(res http.ResponseWriter, req *http.Request) {
	log.Printf("reading body…\n")
	data, err := ioutil.ReadAll(req.Body)
	if err != nil {
		log.Println("error reading request:", err)
		return
	}
	log.Printf("proposing()\n")
	node.Apply(data, 10*time.Second)
	log.Println("proposed")
}

func main() {
	flag.Parse()
	log.Printf("hey\n")

	a, err := net.ResolveTCPAddr("tcp", *listen)
	if err != nil {
		log.Fatal(err)
	}

	transport, err := raft.NewTCPTransport(*listen, a, 3, 10*time.Second, nil)
	if err != nil {
		log.Fatal(err)
	}

	peerStore := raft.NewJSONPeers(*raftDir, transport)

	var p []net.Addr

	config := raft.DefaultConfig()
	if *peers == "" {
		config.EnableSingleNode = true
	} else {
		p, err = peerStore.Peers()
		if err != nil {
			log.Fatal(err)
		}
		for _, addr := range strings.Split(*peers, ",") {
			peer, err := net.ResolveTCPAddr("tcp", addr)
			if err != nil {
				log.Fatal(err)
			}
			log.Printf("adding %v\n", peer)
			p = raft.AddUniquePeer(p, peer)
			peerStore.SetPeers(p)
		}
	}

	fss, err := raft.NewFileSnapshotStore(*raftDir, 1, nil)
	if err != nil {
		log.Fatal(err)
	}

	mdb, err := raftmdb.NewMDBStore(*raftDir)
	if err != nil {
		log.Fatal(err)
	}

	storage, err := NewStorage()
	if err != nil {
		log.Fatal(err)
	}

	fsm := &FSM{storage}

	// NewRaft(*Config, FSM, LogStore, StableStore, SnapshotStore, PeerStore, Transport)
	node, err = raft.NewRaft(config, fsm, mdb, mdb, fss, peerStore, transport)
	if err != nil {
		log.Fatal(err)
	}

	// TODO: observeleaderchanges?
	// TODO: observenexttime?

	//http.HandleFunc("/msg", handleMessages)
	http.HandleFunc("/put", handleRequest)
	go func() {
		err := http.ListenAndServe(*listenhttp, nil)
		if err != nil {
			log.Fatal(err)
		}
	}()

	node.SetPeers(p)

	for {

		time.Sleep(1000 * time.Millisecond)

		// err = s.raft.Add(b, raftMaxTime)
		// if err != nil {
		// 	eWrapper.Done(false)
		// }

		// eWrapper.Done(true)

	}
}
<commit_msg>gofmt<commit_after>package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/hashicorp/raft"
	"github.com/hashicorp/raft-mdb"
)

var (
	raftDir    = flag.String("raftdir", "/tmp/r", "")
	peers      = flag.String("peers", "", "comma-separated host:port tuples")
	listen     = flag.String("listen", ":6000", "")
	listenhttp = flag.String("listenhttp", ":8000", "")
	node       *raft.Raft
)

// Snapshot holds the data needed to serialize storage
type Snapshot struct {
	uuids   [][]byte
	entries []string
}

func uint32ToBytes(u uint32) []byte {
	buf := make([]byte, 4)
	binary.BigEndian.PutUint32(buf, u)
	return buf
}

// Converts bytes to an integer
func bytesToUint32(b []byte) uint32 {
	return binary.BigEndian.Uint32(b)
}

// Persist writes a snapshot to a file. We just serialize all active entries.
func (s *Snapshot) Persist(sink raft.SnapshotSink) error {
	_, err := sink.Write([]byte{0x0})
	if err != nil {
		sink.Cancel()
		return err
	}

	for i, e := range s.entries {
		_, err = sink.Write(s.uuids[i])
		if err != nil {
			sink.Cancel()
			return err
		}

		b := []byte(e)
		_, err = sink.Write(uint32ToBytes(uint32(len(b))))
		if err != nil {
			sink.Cancel()
			return err
		}

		_, err = sink.Write(b)
		if err != nil {
			sink.Cancel()
			return err
		}
	}

	return sink.Close()
}

// Release cleans up a snapshot. We don't need to do anything.
func (s *Snapshot) Release() {
}

type FSM struct {
	store *Storage
}

func (fsm *FSM) Apply(l *raft.Log) interface{} {
	log.Printf("TODO: apply %v\n", l)
	return nil
}

func (fsm *FSM) Snapshot() (raft.FSMSnapshot, error) {
	uuids, entries, err := fsm.store.GetAll()
	snapshot := &Snapshot{uuids, entries}
	return snapshot, err
}

func (fsm *FSM) Restore(snap io.ReadCloser) error {
	defer snap.Close()

	s, err := NewStorage()
	if err != nil {
		return err
	}

	// swap in the restored storage, with time emission channels.
	s.c = fsm.store.c
	s.C = fsm.store.C
	fsm.store.Close()
	fsm.store = s

	b := make([]byte, 1)
	_, err = snap.Read(b)
	if b[0] != byte(0x00) {
		msg := "Unknown snapshot schema version"
		log.Printf(msg)
		return fmt.Errorf(msg)
	}

	uuid := make([]byte, 16)
	size := make([]byte, 4)
	count := 0
	for {
		_, err = snap.Read(uuid)
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		_, err = snap.Read(size)
		if err != nil {
			return err
		}

		eb := make([]byte, bytesToUint32(size))
		_, err = snap.Read(eb)
		if err != nil {
			return err
		}
		e := string(eb)
		err = s.Add(uuid, e)
		if err != nil {
			return err
		}

		count++
	}

	log.Printf("Restored snapshot. entries=%d", count)
	return nil
}

func handleRequest(res http.ResponseWriter, req *http.Request) {
	log.Printf("reading body…\n")
	data, err := ioutil.ReadAll(req.Body)
	if err != nil {
		log.Println("error reading request:", err)
		return
	}
	log.Printf("proposing()\n")
	node.Apply(data, 10*time.Second)
	log.Println("proposed")
}

func main() {
	flag.Parse()
	log.Printf("hey\n")

	a, err := net.ResolveTCPAddr("tcp", *listen)
	if err != nil {
		log.Fatal(err)
	}

	transport, err := raft.NewTCPTransport(*listen, a, 3, 10*time.Second, nil)
	if err != nil {
		log.Fatal(err)
	}

	peerStore := raft.NewJSONPeers(*raftDir, transport)

	var p []net.Addr

	config := raft.DefaultConfig()
	if *peers == "" {
		config.EnableSingleNode = true
	} else {
		p, err = peerStore.Peers()
		if err != nil {
			log.Fatal(err)
		}
		for _, addr := range strings.Split(*peers, ",") {
			peer, err := net.ResolveTCPAddr("tcp", addr)
			if err != nil {
				log.Fatal(err)
			}
			log.Printf("adding %v\n", peer)
			p = raft.AddUniquePeer(p, peer)
			peerStore.SetPeers(p)
		}
	}

	fss, err := raft.NewFileSnapshotStore(*raftDir, 1, nil)
	if err != nil {
		log.Fatal(err)
	}

	mdb, err := raftmdb.NewMDBStore(*raftDir)
	if err != nil {
		log.Fatal(err)
	}

	storage, err := NewStorage()
	if err != nil {
		log.Fatal(err)
	}

	fsm := &FSM{storage}

	// NewRaft(*Config, FSM, LogStore, StableStore, SnapshotStore, PeerStore, Transport)
	node, err = raft.NewRaft(config, fsm, mdb, mdb, fss, peerStore, transport)
	if err != nil {
		log.Fatal(err)
	}

	// TODO: observeleaderchanges?
	// TODO: observenexttime?

	//http.HandleFunc("/msg", handleMessages)
	http.HandleFunc("/put", handleRequest)
	go func() {
		err := http.ListenAndServe(*listenhttp, nil)
		if err != nil {
			log.Fatal(err)
		}
	}()

	node.SetPeers(p)

	for {

		time.Sleep(1000 * time.Millisecond)

		// err = s.raft.Add(b, raftMaxTime)
		// if err != nil {
		// 	eWrapper.Done(false)
		// }

		// eWrapper.Done(true)

	}
}
<|endoftext|>


<commit_before>package command

import (
	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"

	// Previously core-bundled components, split into their own plugins but
	// still vendored with Packer for now. Importing as library instead of
	// forcing use of packer init, until packer v1.8.0
	exoscaleimportpostprocessor "github.com/exoscale/packer-plugin-exoscale/post-processor/exoscale-import"
	amazonchrootbuilder "github.com/hashicorp/packer-plugin-amazon/builder/chroot"
	amazonebsbuilder "github.com/hashicorp/packer-plugin-amazon/builder/ebs"
	amazonebssurrogatebuilder "github.com/hashicorp/packer-plugin-amazon/builder/ebssurrogate"
	amazonebsvolumebuilder "github.com/hashicorp/packer-plugin-amazon/builder/ebsvolume"
	amazoninstancebuilder "github.com/hashicorp/packer-plugin-amazon/builder/instance"
	amazonamidatasource "github.com/hashicorp/packer-plugin-amazon/datasource/ami"
	amazonsecretsmanagerdatasource "github.com/hashicorp/packer-plugin-amazon/datasource/secretsmanager"
	anazibimportpostprocessor "github.com/hashicorp/packer-plugin-amazon/post-processor/import"
	ansibleprovisioner "github.com/hashicorp/packer-plugin-ansible/provisioner/ansible"
	ansiblelocalprovisioner "github.com/hashicorp/packer-plugin-ansible/provisioner/ansible-local"
	dockerbuilder "github.com/hashicorp/packer-plugin-docker/builder/docker"
	dockerimportpostprocessor "github.com/hashicorp/packer-plugin-docker/post-processor/docker-import"
	dockerpushpostprocessor "github.com/hashicorp/packer-plugin-docker/post-processor/docker-push"
	dockersavepostprocessor "github.com/hashicorp/packer-plugin-docker/post-processor/docker-save"
	dockertagpostprocessor "github.com/hashicorp/packer-plugin-docker/post-processor/docker-tag"
	googlecomputebuilder "github.com/hashicorp/packer-plugin-googlecompute/builder/googlecompute"
	googlecomputeexportpostprocessor "github.com/hashicorp/packer-plugin-googlecompute/post-processor/googlecompute-export"
	googlecomputeimportpostprocessor "github.com/hashicorp/packer-plugin-googlecompute/post-processor/googlecompute-import"
	virtualboxisobuilder "github.com/hashicorp/packer-plugin-virtualbox/builder/virtualbox/iso"
	virtualboxovfbuilder "github.com/hashicorp/packer-plugin-virtualbox/builder/virtualbox/ovf"
	virtualboxvmbuilder "github.com/hashicorp/packer-plugin-virtualbox/builder/virtualbox/vm"
	vmwareisobuilder "github.com/hashicorp/packer-plugin-vmware/builder/vmware/iso"
	vmwarevmxbuilder "github.com/hashicorp/packer-plugin-vmware/builder/vmware/vmx"
	vsphereclonebuilder "github.com/hashicorp/packer-plugin-vsphere/builder/vsphere/clone"
	vsphereisobuilder "github.com/hashicorp/packer-plugin-vsphere/builder/vsphere/iso"
	vspherepostprocessor "github.com/hashicorp/packer-plugin-vsphere/post-processor/vsphere"
	vspheretemplatepostprocessor "github.com/hashicorp/packer-plugin-vsphere/post-processor/vsphere-template"
)

// VendoredDatasources are datasource components that were once bundled with the
// Packer core, but are now being imported from their counterpart plugin repos
var VendoredDatasources = map[string]packersdk.Datasource{
	"amazon-ami":            new(amazonamidatasource.Datasource),
	"amazon-secretsmanager": new(amazonsecretsmanagerdatasource.Datasource),
}

// VendoredBuilders are builder components that were once bundled with the
// Packer core, but are now being imported from their counterpart plugin repos
var VendoredBuilders = map[string]packersdk.Builder{
	"docker":              new(dockerbuilder.Builder),
	"amazon-ebs":          new(amazonebsbuilder.Builder),
	"amazon-chroot":       new(amazonchrootbuilder.Builder),
	"amazon-ebssurrogate": new(amazonebssurrogatebuilder.Builder),
	"amazon-ebsvolume":    new(amazonebsvolumebuilder.Builder),
	"amazon-instance":     new(amazoninstancebuilder.Builder),
	"googlecompute":       new(googlecomputebuilder.Builder),
	"vsphere-clone":       new(vsphereclonebuilder.Builder),
	"vsphere-iso":         new(vsphereisobuilder.Builder),
	"virtualbox-iso":      new(virtualboxisobuilder.Builder),
	"virtualbox-ovf":      new(virtualboxovfbuilder.Builder),
	"virtualbox-vm":       new(virtualboxvmbuilder.Builder),
	"vmware-iso":          new(vmwareisobuilder.Builder),
	"vmware-vmx":          new(vmwarevmxbuilder.Builder),
}

// VendoredProvisioners are provisioner components that were once bundled with the
// Packer core, but are now being imported from their counterpart plugin repos
var VendoredProvisioners = map[string]packersdk.Provisioner{
	"ansible":       new(ansibleprovisioner.Provisioner),
	"ansible-local": new(ansiblelocalprovisioner.Provisioner),
}

// VendoredPostProcessors are post-processor components that were once bundled with the
// Packer core, but are now being imported from their counterpart plugin repos
var VendoredPostProcessors = map[string]packersdk.PostProcessor{
	"amazon-import":        new(anazibimportpostprocessor.PostProcessor),
	"docker-import":        new(dockerimportpostprocessor.PostProcessor),
	"docker-push":          new(dockerpushpostprocessor.PostProcessor),
	"docker-save":          new(dockersavepostprocessor.PostProcessor),
	"docker-tag":           new(dockertagpostprocessor.PostProcessor),
	"exoscale-import":      new(exoscaleimportpostprocessor.PostProcessor),
	"googlecompute-export": new(googlecomputeexportpostprocessor.PostProcessor),
	"googlecompute-import": new(googlecomputeimportpostprocessor.PostProcessor),
	"vsphere-template":     new(vspheretemplatepostprocessor.PostProcessor),
	"vsphere":              new(vspherepostprocessor.PostProcessor),
}

// Upon init lets load up any plugins that were vendored manually into the default
// set of plugins.
func init() {
	for k, v := range VendoredDatasources {
		if _, ok := Datasources[k]; ok {
			continue
		}
		Datasources[k] = v
	}

	for k, v := range VendoredBuilders {
		if _, ok := Builders[k]; ok {
			continue
		}
		Builders[k] = v
	}

	for k, v := range VendoredProvisioners {
		if _, ok := Provisioners[k]; ok {
			continue
		}
		Provisioners[k] = v
	}

	for k, v := range VendoredPostProcessors {
		if _, ok := PostProcessors[k]; ok {
			continue
		}
		PostProcessors[k] = v
	}
}
<commit_msg>Update vendored_plugins.go<commit_after>package command

import (
	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"

	// Previously core-bundled components, split into their own plugins but
	// still vendored with Packer for now. Importing as library instead of
	// forcing use of packer init, until packer v1.8.0
	exoscaleimportpostprocessor "github.com/exoscale/packer-plugin-exoscale/post-processor/exoscale-import"
	amazonchrootbuilder "github.com/hashicorp/packer-plugin-amazon/builder/chroot"
	amazonebsbuilder "github.com/hashicorp/packer-plugin-amazon/builder/ebs"
	amazonebssurrogatebuilder "github.com/hashicorp/packer-plugin-amazon/builder/ebssurrogate"
	amazonebsvolumebuilder "github.com/hashicorp/packer-plugin-amazon/builder/ebsvolume"
	amazoninstancebuilder "github.com/hashicorp/packer-plugin-amazon/builder/instance"
	amazonamidatasource "github.com/hashicorp/packer-plugin-amazon/datasource/ami"
	amazonsecretsmanagerdatasource "github.com/hashicorp/packer-plugin-amazon/datasource/secretsmanager"
	anazibimportpostprocessor "github.com/hashicorp/packer-plugin-amazon/post-processor/import"
	ansibleprovisioner "github.com/hashicorp/packer-plugin-ansible/provisioner/ansible"
	ansiblelocalprovisioner "github.com/hashicorp/packer-plugin-ansible/provisioner/ansible-local"
	dockerbuilder "github.com/hashicorp/packer-plugin-docker/builder/docker"
	dockerimportpostprocessor "github.com/hashicorp/packer-plugin-docker/post-processor/docker-import"
	dockerpushpostprocessor "github.com/hashicorp/packer-plugin-docker/post-processor/docker-push"
	dockersavepostprocessor "github.com/hashicorp/packer-plugin-docker/post-processor/docker-save"
	dockertagpostprocessor "github.com/hashicorp/packer-plugin-docker/post-processor/docker-tag"
	googlecomputebuilder "github.com/hashicorp/packer-plugin-googlecompute/builder/googlecompute"
	googlecomputeexportpostprocessor "github.com/hashicorp/packer-plugin-googlecompute/post-processor/googlecompute-export"
	googlecomputeimportpostprocessor "github.com/hashicorp/packer-plugin-googlecompute/post-processor/googlecompute-import"
	virtualboxisobuilder "github.com/hashicorp/packer-plugin-virtualbox/builder/virtualbox/iso"
	virtualboxovfbuilder "github.com/hashicorp/packer-plugin-virtualbox/builder/virtualbox/ovf"
	virtualboxvmbuilder "github.com/hashicorp/packer-plugin-virtualbox/builder/virtualbox/vm"
	vmwareisobuilder "github.com/hashicorp/packer-plugin-vmware/builder/vmware/iso"
	vmwarevmxbuilder "github.com/hashicorp/packer-plugin-vmware/builder/vmware/vmx"
	vsphereclonebuilder "github.com/hashicorp/packer-plugin-vsphere/builder/vsphere/clone"
	vsphereisobuilder "github.com/hashicorp/packer-plugin-vsphere/builder/vsphere/iso"
	vspherepostprocessor "github.com/hashicorp/packer-plugin-vsphere/post-processor/vsphere"
	vspheretemplatepostprocessor "github.com/hashicorp/packer-plugin-vsphere/post-processor/vsphere-template"
)

// VendoredDatasources are datasource components that were once bundled with the
// Packer core, but are now being imported from their counterpart plugin repos
var VendoredDatasources = map[string]packersdk.Datasource{
	"amazon-ami":            new(amazonamidatasource.Datasource),
	"amazon-secretsmanager": new(amazonsecretsmanagerdatasource.Datasource),
}

// VendoredBuilders are builder components that were once bundled with the
// Packer core, but are now being imported from their counterpart plugin repos
var VendoredBuilders = map[string]packersdk.Builder{
	"amazon-ebs":          new(amazonebsbuilder.Builder),
	"amazon-chroot":       new(amazonchrootbuilder.Builder),
	"amazon-ebssurrogate": new(amazonebssurrogatebuilder.Builder),
	"amazon-ebsvolume":    new(amazonebsvolumebuilder.Builder),
	"amazon-instance":     new(amazoninstancebuilder.Builder),
	"docker":              new(dockerbuilder.Builder),
	"googlecompute":       new(googlecomputebuilder.Builder),
	"vsphere-clone":       new(vsphereclonebuilder.Builder),
	"vsphere-iso":         new(vsphereisobuilder.Builder),
	"virtualbox-iso":      new(virtualboxisobuilder.Builder),
	"virtualbox-ovf":      new(virtualboxovfbuilder.Builder),
	"virtualbox-vm":       new(virtualboxvmbuilder.Builder),
	"vmware-iso":          new(vmwareisobuilder.Builder),
	"vmware-vmx":          new(vmwarevmxbuilder.Builder),
}

// VendoredProvisioners are provisioner components that were once bundled with the
// Packer core, but are now being imported from their counterpart plugin repos
var VendoredProvisioners = map[string]packersdk.Provisioner{
	"ansible":       new(ansibleprovisioner.Provisioner),
	"ansible-local": new(ansiblelocalprovisioner.Provisioner),
}

// VendoredPostProcessors are post-processor components that were once bundled with the
// Packer core, but are now being imported from their counterpart plugin repos
var VendoredPostProcessors = map[string]packersdk.PostProcessor{
	"amazon-import":        new(anazibimportpostprocessor.PostProcessor),
	"docker-import":        new(dockerimportpostprocessor.PostProcessor),
	"docker-push":          new(dockerpushpostprocessor.PostProcessor),
	"docker-save":          new(dockersavepostprocessor.PostProcessor),
	"docker-tag":           new(dockertagpostprocessor.PostProcessor),
	"exoscale-import":      new(exoscaleimportpostprocessor.PostProcessor),
	"googlecompute-export": new(googlecomputeexportpostprocessor.PostProcessor),
	"googlecompute-import": new(googlecomputeimportpostprocessor.PostProcessor),
	"vsphere-template":     new(vspheretemplatepostprocessor.PostProcessor),
	"vsphere":              new(vspherepostprocessor.PostProcessor),
}

// Upon init lets load up any plugins that were vendored manually into the default
// set of plugins.
func init() {
	for k, v := range VendoredDatasources {
		if _, ok := Datasources[k]; ok {
			continue
		}
		Datasources[k] = v
	}

	for k, v := range VendoredBuilders {
		if _, ok := Builders[k]; ok {
			continue
		}
		Builders[k] = v
	}

	for k, v := range VendoredProvisioners {
		if _, ok := Provisioners[k]; ok {
			continue
		}
		Provisioners[k] = v
	}

	for k, v := range VendoredPostProcessors {
		if _, ok := PostProcessors[k]; ok {
			continue
		}
		PostProcessors[k] = v
	}
}
<|endoftext|>


<commit_before>package common

import (
	"encoding/binary"
	"net"
)

const (
	// MTU - The max size packet to recieve from the TUN device
	MTU = 65475

	// HeaderSize - The size of the perpended data
	HeaderSize = 16

	// FooterSize - The size of the appended data
	FooterSize = 16

	// MaxPacketLength - The maximum packet size to send via the UDP device
	MaxPacketLength = HeaderSize + MTU + FooterSize
)

const (
	// IPStart - The ip start position
	IPStart = 0

	// IPEnd - The ip end position
	IPEnd = 4

	// NonceStart - The nonce start position
	NonceStart = 4

	// NonceEnd - The nonce end postion
	NonceEnd = 16

	// PacketStart - The packet start position
	PacketStart = 16
)

// IPtoInt takes a string ip in the form '0.0.0.0' and returns a uint32 that represents that ipaddress
func IPtoInt(IP string) uint32 {
	buf := net.ParseIP(IP).To4()
	return binary.LittleEndian.Uint32(buf)
}
<commit_msg>Added an InttoIP helper method in common<commit_after>package common

import (
	"encoding/binary"
	"net"
)

const (
	// MTU - The max size packet to recieve from the TUN device
	MTU = 65475

	// HeaderSize - The size of the perpended data
	HeaderSize = 16

	// FooterSize - The size of the appended data
	FooterSize = 16

	// MaxPacketLength - The maximum packet size to send via the UDP device
	MaxPacketLength = HeaderSize + MTU + FooterSize
)

const (
	// IPStart - The ip start position
	IPStart = 0

	// IPEnd - The ip end position
	IPEnd = 4

	// NonceStart - The nonce start position
	NonceStart = 4

	// NonceEnd - The nonce end postion
	NonceEnd = 16

	// PacketStart - The packet start position
	PacketStart = 16
)

// IPtoInt takes a string ip in the form '0.0.0.0' and returns a uint32 that represents that ipaddress
func IPtoInt(IP string) uint32 {
	buf := net.ParseIP(IP).To4()
	return binary.LittleEndian.Uint32(buf)
}

func InttoIP(IP uint32) string {
	buf := make([]byte, 4)
	binary.LittleEndian.PutUint32(buf, IP)
	return string(buf)
}
<|endoftext|>


<commit_before>package commands_helpers

import (
	"archive/zip"
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
)

type ArchiveCommand struct {
	Paths     []string `long:"path" description:"Add paths to archive"`
	Untracked bool     `long:"untracked" description:"Add git untracked files"`
	File      string   `long:"file" description:"The path to file"`
	Verbose   bool     `long:"verbose" description:"Detailed information"`
	List      bool     `long:"list" description:"List files to archive"`

	wd    string
	files map[string]os.FileInfo
}

func isTarArchive(fileName string) bool {
	if strings.HasSuffix(fileName, ".tgz") || strings.HasSuffix(fileName, ".tar.gz") {
		return true
	}
	return false
}

func isZipArchive(fileName string) bool {
	if strings.HasSuffix(fileName, ".zip") {
		return true
	}
	return false
}

func (c *ArchiveCommand) isChanged(modTime time.Time) bool {
	for _, info := range c.files {
		if modTime.Before(info.ModTime()) {
			return true
		}
	}
	return false
}

func (c *ArchiveCommand) zipArchiveChanged() bool {
	archive, err := zip.OpenReader(c.File)
	if err != nil {
		logrus.Warningf("%s: %v", c.File, err)
		return true
	}
	defer archive.Close()
	for _, file := range archive.File {
		_, err := os.Lstat(file.Name)
		if os.IsNotExist(err) {
			return true
		}
	}
	return false
}

func (c *ArchiveCommand) isFileListChanged() bool {
	if isZipArchive(c.File) {
		return c.zipArchiveChanged()
	} else {
		logrus.Warningln("The archive can't be verified if file list changed: operation not supported")
		// TODO: this is not supported
		return false
	}
}

func (c *ArchiveCommand) sortedFiles() []string {
	files := make([]string, len(c.files))

	i := 0
	for file := range c.files {
		files[i] = file
		i++
	}

	sort.Strings(files)
	return files
}

func (c *ArchiveCommand) add(path string, info os.FileInfo) (err error) {
	if info == nil {
		info, err = os.Lstat(path)
	}
	if err == nil {
		c.files[path] = info
	} else if os.IsNotExist(err) {
		logrus.Warningln("File", path, "doesn't exist")
		err = nil
	}
	return
}

func (c *ArchiveCommand) process(match string) error {
	absolute, err := filepath.Abs(match)
	if err != nil {
		return err
	}

	relative, err := filepath.Rel(c.wd, absolute)
	if err != nil {
		return err
	}

	// store relative path if points to current working directory
	if strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
		return c.add(absolute, nil)
	} else {
		return c.add(relative, nil)
	}
}

func (c *ArchiveCommand) processPaths() {
	for _, path := range c.Paths {
		matches, err := filepath.Glob(path)
		if err != nil {
			logrus.Warningf("%s: %v", path, err)
			continue
		}

		found := 0

		for _, match := range matches {
			err := filepath.Walk(match, func(path string, info os.FileInfo, err error) error {
				found++
				return c.process(path)
			})
			if err != nil {
				logrus.Warningln("Walking", match, err)
			}
		}

		if found == 0 {
			logrus.Warningf("%s: no matching files", path)
		} else {
			logrus.Infof("%s: found %d matching files", path, found)
		}
	}
}

func (c *ArchiveCommand) processUntracked() {
	if !c.Untracked {
		return
	}

	found := 0

	var output bytes.Buffer
	cmd := exec.Command("git", "ls-files", "-o")
	cmd.Env = os.Environ()
	cmd.Stdout = &output
	cmd.Stderr = os.Stderr
	logrus.Debugln("Executing command:", strings.Join(cmd.Args, " "))
	err := cmd.Run()
	if err == nil {
		reader := bufio.NewReader(&output)
		for {
			line, _, err := reader.ReadLine()
			if err == io.EOF {
				break
			} else if err != nil {
				logrus.Warningln(err)
				break
			}
			c.process(string(line))
		}

		if found == 0 {
			logrus.Warningf("untracked: no files")
		} else {
			logrus.Infof("untracked: found %d files", found)
		}
	} else {
		logrus.Warningf("untracked: %v", err)
	}
}

func (c *ArchiveCommand) listFiles() {
	if len(c.files) == 0 {
		logrus.Infoln("No files to archive.")
		return
	}

	for _, file := range c.sortedFiles() {
		println(string(file))
	}
}

func (c *ArchiveCommand) createZipArchive(w io.Writer, fileNames []string) error {
	archive := zip.NewWriter(w)
	defer archive.Close()

	for _, fileName := range fileNames {
		fi, err := os.Lstat(fileName)
		if err != nil {
			logrus.Warningln("File ignored: %q: %v", fileName, err)
			continue
		}

		fh, err := zip.FileInfoHeader(fi)
		fh.Name = fileName
		fh.Extra = createZipExtra(fi)

		switch fi.Mode() & os.ModeType {
		case os.ModeDir:
			fh.Name += "/"

			_, err := archive.CreateHeader(fh)
			if err != nil {
				return err
			}

		case os.ModeSymlink:
			fw, err := archive.CreateHeader(fh)
			if err != nil {
				return err
			}

			link, err := os.Readlink(fileName)
			if err != nil {
				return err
			}

			io.WriteString(fw, link)

		case os.ModeNamedPipe, os.ModeSocket, os.ModeDevice:
			// Ignore the files that of these types
			logrus.Warningln("File ignored: %q", fileName)

		default:
			fh.Method = zip.Deflate
			fw, err := archive.CreateHeader(fh)
			if err != nil {
				return err
			}

			file, err := os.Open(fileName)
			if err != nil {
				return err
			}

			_, err = io.Copy(fw, file)
			file.Close()
			if err != nil {
				return err
			}
			break
		}

		if c.Verbose {
			fmt.Printf("%v\t%d\t%s\n", fh.Mode(), fh.UncompressedSize64, fh.Name)
		}
	}

	return nil
}

func (c *ArchiveCommand) createTarArchive(w io.Writer, files []string) error {
	var list bytes.Buffer
	for _, file := range c.sortedFiles() {
		list.WriteString(string(file) + "\n")
	}

	flags := "-zcP"
	if c.Verbose {
		flags += "v"
	}

	cmd := exec.Command("tar", flags, "-T", "-", "--no-recursion")
	cmd.Env = os.Environ()
	cmd.Stdin = &list
	cmd.Stdout = w
	cmd.Stderr = os.Stderr
	logrus.Debugln("Executing command:", strings.Join(cmd.Args, " "))
	return cmd.Run()
}

func (c *ArchiveCommand) createArchive(w io.Writer, files []string) error {
	if isTarArchive(c.File) {
		return c.createTarArchive(w, files)
	} else if isZipArchive(c.File) {
		return c.createZipArchive(w, files)
	} else {
		return fmt.Errorf("Unsupported archive format: %q", c.File)
	}
}

func (c *ArchiveCommand) archive() {
	if len(c.files) == 0 {
		logrus.Infoln("No files to archive.")
		return
	}

	logrus.Infoln("Creating archive", filepath.Base(c.File), "...")

	// create directories to store archive
	os.MkdirAll(filepath.Dir(c.File), 0700)

	tempFile, err := ioutil.TempFile(filepath.Dir(c.File), "archive_")
	if err != nil {
		logrus.Fatalln("Failed to create temporary archive", err)
	}
	defer tempFile.Close()
	defer os.Remove(tempFile.Name())

	logrus.Debugln("Temporary file:", tempFile.Name())
	err = c.createArchive(tempFile, c.sortedFiles())
	if err != nil {
		logrus.Fatalln("Failed to create archive:", err)
	}
	tempFile.Close()

	err = os.Rename(tempFile.Name(), c.File)
	if err != nil {
		logrus.Warningln("Failed to rename archive:", err)
	}

	logrus.Infoln("Done!")
}

func (c *ArchiveCommand) Execute(context *cli.Context) {
	logrus.SetFormatter(
		&logrus.TextFormatter{
			ForceColors:      true,
			DisableTimestamp: false,
		},
	)

	wd, err := os.Getwd()
	if err != nil {
		logrus.Fatalln("Failed to get current working directory:", err)
	}
	if c.File == "" && !c.List {
		logrus.Fatalln("Missing archive file name!")
	}

	c.wd = wd
	c.files = make(map[string]os.FileInfo)

	c.processPaths()
	c.processUntracked()

	ai, err := os.Stat(c.File)
	if err != nil && !os.IsNotExist(err) {
		logrus.Fatalln("Failed to verify archive:", c.File, err)
	}
	if ai != nil {
		if !c.isChanged(ai.ModTime()) && !c.zipArchiveChanged() {
			logrus.Infoln("Archive is up to date!")
			return
		}
	}

	if c.List {
		c.listFiles()
	} else {
		c.archive()
	}
}

func init() {
	common.RegisterCommand2("archive", "find and archive files (internal)", &ArchiveCommand{})
}
<commit_msg>Make possible to cache absolute paths<commit_after>package commands_helpers

import (
	"archive/zip"
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
)

type ArchiveCommand struct {
	Paths     []string `long:"path" description:"Add paths to archive"`
	Untracked bool     `long:"untracked" description:"Add git untracked files"`
	File      string   `long:"file" description:"The path to file"`
	Verbose   bool     `long:"verbose" description:"Detailed information"`
	List      bool     `long:"list" description:"List files to archive"`

	wd    string
	files map[string]os.FileInfo
}

func isTarArchive(fileName string) bool {
	if strings.HasSuffix(fileName, ".tgz") || strings.HasSuffix(fileName, ".tar.gz") {
		return true
	}
	return false
}

func isZipArchive(fileName string) bool {
	if strings.HasSuffix(fileName, ".zip") {
		return true
	}
	return false
}

func (c *ArchiveCommand) isChanged(modTime time.Time) bool {
	for _, info := range c.files {
		if modTime.Before(info.ModTime()) {
			return true
		}
	}
	return false
}

func (c *ArchiveCommand) zipArchiveChanged() bool {
	archive, err := zip.OpenReader(c.File)
	if err != nil {
		logrus.Warningf("%s: %v", c.File, err)
		return true
	}
	defer archive.Close()
	for _, file := range archive.File {
		_, err := os.Lstat(file.Name)
		if os.IsNotExist(err) {
			return true
		}
	}
	return false
}

func (c *ArchiveCommand) isFileListChanged() bool {
	if isZipArchive(c.File) {
		return c.zipArchiveChanged()
	} else {
		logrus.Warningln("The archive can't be verified if file list changed: operation not supported")
		// TODO: this is not supported
		return false
	}
}

func (c *ArchiveCommand) sortedFiles() []string {
	files := make([]string, len(c.files))

	i := 0
	for file := range c.files {
		files[i] = file
		i++
	}

	sort.Strings(files)
	return files
}

func (c *ArchiveCommand) add(path string, info os.FileInfo) (err error) {
	if info == nil {
		info, err = os.Lstat(path)
	}
	if err == nil {
		c.files[path] = info
	} else if os.IsNotExist(err) {
		logrus.Warningln("File", path, "doesn't exist")
		err = nil
	}
	return
}

func (c *ArchiveCommand) process(match string) error {
	absolute, err := filepath.Abs(match)
	if err != nil {
		return err
	}

	// If we can't make the path relative, always save an absolute
	relative, err := filepath.Rel(c.wd, absolute)
	if err != nil {
		return c.add(absolute, nil)
	}

	// store relative path if points to current working directory
	if strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
		return c.add(absolute, nil)
	} else {
		return c.add(relative, nil)
	}
}

func (c *ArchiveCommand) processPaths() {
	for _, path := range c.Paths {
		matches, err := filepath.Glob(path)
		if err != nil {
			logrus.Warningf("%s: %v", path, err)
			continue
		}

		found := 0

		for _, match := range matches {
			err := filepath.Walk(match, func(path string, info os.FileInfo, err error) error {
				found++
				return c.process(path)
			})
			if err != nil {
				logrus.Warningln("Walking", match, err)
			}
		}

		if found == 0 {
			logrus.Warningf("%s: no matching files", path)
		} else {
			logrus.Infof("%s: found %d matching files", path, found)
		}
	}
}

func (c *ArchiveCommand) processUntracked() {
	if !c.Untracked {
		return
	}

	found := 0

	var output bytes.Buffer
	cmd := exec.Command("git", "ls-files", "-o")
	cmd.Env = os.Environ()
	cmd.Stdout = &output
	cmd.Stderr = os.Stderr
	logrus.Debugln("Executing command:", strings.Join(cmd.Args, " "))
	err := cmd.Run()
	if err == nil {
		reader := bufio.NewReader(&output)
		for {
			line, _, err := reader.ReadLine()
			if err == io.EOF {
				break
			} else if err != nil {
				logrus.Warningln(err)
				break
			}
			c.process(string(line))
		}

		if found == 0 {
			logrus.Warningf("untracked: no files")
		} else {
			logrus.Infof("untracked: found %d files", found)
		}
	} else {
		logrus.Warningf("untracked: %v", err)
	}
}

func (c *ArchiveCommand) listFiles() {
	if len(c.files) == 0 {
		logrus.Infoln("No files to archive.")
		return
	}

	for _, file := range c.sortedFiles() {
		println(string(file))
	}
}

func (c *ArchiveCommand) createZipArchive(w io.Writer, fileNames []string) error {
	archive := zip.NewWriter(w)
	defer archive.Close()

	for _, fileName := range fileNames {
		fi, err := os.Lstat(fileName)
		if err != nil {
			logrus.Warningln("File ignored: %q: %v", fileName, err)
			continue
		}

		fh, err := zip.FileInfoHeader(fi)
		fh.Name = fileName
		fh.Extra = createZipExtra(fi)

		switch fi.Mode() & os.ModeType {
		case os.ModeDir:
			fh.Name += "/"

			_, err := archive.CreateHeader(fh)
			if err != nil {
				return err
			}

		case os.ModeSymlink:
			fw, err := archive.CreateHeader(fh)
			if err != nil {
				return err
			}

			link, err := os.Readlink(fileName)
			if err != nil {
				return err
			}

			io.WriteString(fw, link)

		case os.ModeNamedPipe, os.ModeSocket, os.ModeDevice:
			// Ignore the files that of these types
			logrus.Warningln("File ignored: %q", fileName)

		default:
			fh.Method = zip.Deflate
			fw, err := archive.CreateHeader(fh)
			if err != nil {
				return err
			}

			file, err := os.Open(fileName)
			if err != nil {
				return err
			}

			_, err = io.Copy(fw, file)
			file.Close()
			if err != nil {
				return err
			}
			break
		}

		if c.Verbose {
			fmt.Printf("%v\t%d\t%s\n", fh.Mode(), fh.UncompressedSize64, fh.Name)
		}
	}

	return nil
}

func (c *ArchiveCommand) createTarArchive(w io.Writer, files []string) error {
	var list bytes.Buffer
	for _, file := range c.sortedFiles() {
		list.WriteString(string(file) + "\n")
	}

	flags := "-zcP"
	if c.Verbose {
		flags += "v"
	}

	cmd := exec.Command("tar", flags, "-T", "-", "--no-recursion")
	cmd.Env = os.Environ()
	cmd.Stdin = &list
	cmd.Stdout = w
	cmd.Stderr = os.Stderr
	logrus.Debugln("Executing command:", strings.Join(cmd.Args, " "))
	return cmd.Run()
}

func (c *ArchiveCommand) createArchive(w io.Writer, files []string) error {
	if isTarArchive(c.File) {
		return c.createTarArchive(w, files)
	} else if isZipArchive(c.File) {
		return c.createZipArchive(w, files)
	} else {
		return fmt.Errorf("Unsupported archive format: %q", c.File)
	}
}

func (c *ArchiveCommand) archive() {
	if len(c.files) == 0 {
		logrus.Infoln("No files to archive.")
		return
	}

	logrus.Infoln("Creating archive", filepath.Base(c.File), "...")

	// create directories to store archive
	os.MkdirAll(filepath.Dir(c.File), 0700)

	tempFile, err := ioutil.TempFile(filepath.Dir(c.File), "archive_")
	if err != nil {
		logrus.Fatalln("Failed to create temporary archive", err)
	}
	defer tempFile.Close()
	defer os.Remove(tempFile.Name())

	logrus.Debugln("Temporary file:", tempFile.Name())
	err = c.createArchive(tempFile, c.sortedFiles())
	if err != nil {
		logrus.Fatalln("Failed to create archive:", err)
	}
	tempFile.Close()

	err = os.Rename(tempFile.Name(), c.File)
	if err != nil {
		logrus.Warningln("Failed to rename archive:", err)
	}

	logrus.Infoln("Done!")
}

func (c *ArchiveCommand) Execute(context *cli.Context) {
	logrus.SetFormatter(
		&logrus.TextFormatter{
			ForceColors:      true,
			DisableTimestamp: false,
		},
	)

	wd, err := os.Getwd()
	if err != nil {
		logrus.Fatalln("Failed to get current working directory:", err)
	}
	if c.File == "" && !c.List {
		logrus.Fatalln("Missing archive file name!")
	}

	c.wd = wd
	c.files = make(map[string]os.FileInfo)

	c.processPaths()
	c.processUntracked()

	ai, err := os.Stat(c.File)
	if err != nil && !os.IsNotExist(err) {
		logrus.Fatalln("Failed to verify archive:", c.File, err)
	}
	if ai != nil {
		if !c.isChanged(ai.ModTime()) && !c.zipArchiveChanged() {
			logrus.Infoln("Archive is up to date!")
			return
		}
	}

	if c.List {
		c.listFiles()
	} else {
		c.archive()
	}
}

func init() {
	common.RegisterCommand2("archive", "find and archive files (internal)", &ArchiveCommand{})
}
<|endoftext|>


<commit_before>/**
 *  Copyright 2014 Paul Querna
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

package common

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"io/ioutil"
	"os"
	"time"
)

var bufWriteSize = 64 * 1024

type Result struct {
	Type     string
	Id       string
	Error    bool
	Start    time.Time
	Duration time.Duration
	// TODO: should we just do a map[string]interface{}?
	Meta    map[string]string
	Metrics map[string]float64
}

// Calling Done() multiple times is OK, some Tasks will call it before
// they return, because they spend significant time extracting metadata.
func (r *Result) Done() {
	if r.Duration == 0 {
		r.Duration = time.Since(r.Start)
	}
}

func (r *Result) MarshalJSON() ([]byte, error) {

	errb := `false`
	if r.Error {
		errb = `true`
	}

	start, err := json.Marshal(r.Start)
	if err != nil {
		return nil, err
	}

	duration, err := json.Marshal(r.Duration)
	if err != nil {
		return nil, err
	}

	meta, err := json.Marshal(r.Meta)
	if err != nil {
		return nil, err
	}

	metrics, err := json.Marshal(r.Metrics)
	if err != nil {
		return nil, err
	}

	// TODO: benchmark
	return bytes.Join(
		[][]byte{
			[]byte(`{`),
			[]byte(`"Type":"` + r.Type + `"`),
			[]byte(`,"Id":"` + r.Id + `"`),
			[]byte(`,"Error":"` + errb + `"`),
			[]byte(`,"Start":`),
			start,
			[]byte(`,"Duration":`),
			duration,
			[]byte(`,"Meta":`),
			meta,
			[]byte(`,"Metrics":`),
			metrics,
			[]byte(`}`),
		},
		[]byte{}), nil
}

func NewResult(taskType string, id string) *Result {
	r := &Result{Id: id, Type: taskType}
	r.Meta = make(map[string]string)
	r.Metrics = make(map[string]float64)
	return r
}

type ResultArchiveWriter struct {
	Path    string
	fwriter *os.File
	gwriter *gzip.Writer
	writer  *bufio.Writer
}

func (raw *ResultArchiveWriter) Write(rv *Result) error {
	b, err := json.Marshal(rv)
	if err != nil {
		return err
	}
	raw.writer.Write(b)
	raw.writer.WriteString("\n")
	return nil
}

func (raw *ResultArchiveWriter) Close() error {
	raw.writer.Flush()
	raw.gwriter.Close()
	return raw.fwriter.Close()
}

func (raw *ResultArchiveWriter) Remove() error {
	return os.Remove(raw.Path)
}

func NewResultArchiveWriter() *ResultArchiveWriter {
	tfile, err := ioutil.TempFile("", "hurlgz")
	if err != nil {
		panic(err)
	}

	gwriter, err := gzip.NewWriterLevel(tfile, gzip.BestSpeed)

	if err != nil {
		panic(err)
	}

	return &ResultArchiveWriter{
		Path:    tfile.Name(),
		fwriter: tfile,
		gwriter: gwriter,
		writer:  bufio.NewWriterSize(gwriter, bufWriteSize),
	}
}

type ResultArchiveReader struct {
	Path    string
	rrfile  *os.File
	gfile   *gzip.Reader
	scanner *bufio.Scanner
}

func NewResultArchiveReader(path string) *ResultArchiveReader {
	rar := &ResultArchiveReader{Path: path}
	err := rar.open(path)
	if err != nil {
		panic(err)
	}

	return rar
}

func (rar *ResultArchiveReader) Close() error {
	rar.scanner = nil
	if rar.gfile != nil {
		rar.gfile.Close()
		rar.gfile = nil
	}
	if rar.rrfile != nil {
		rar.rrfile.Close()
		rar.rrfile = nil
	}
	return nil
}

func (rr *ResultArchiveReader) Reset() {
	rr.Close()
	rr.open(rr.Path)
}

func (rr *ResultArchiveReader) open(path string) error {
	var err error
	rr.rrfile, err = os.Open(path)
	if err != nil {
		return err
	}

	rr.gfile, err = gzip.NewReader(rr.rrfile)
	if err != nil {
		return err
	}

	rr.scanner = bufio.NewScanner(rr.gfile)
	rr.scanner.Split(bufio.ScanLines)
	return nil
}

func (rr *ResultArchiveReader) Entry() *Result {
	b := rr.scanner.Bytes()
	rv := &Result{}
	json.Unmarshal(b, rv)
	return rv
}

func (rr *ResultArchiveReader) Scan() bool {
	return rr.scanner.Scan()
}
<commit_msg>more spec<commit_after>/**
 *  Copyright 2014 Paul Querna
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

package common

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"io/ioutil"
	"os"
	"strconv"
	"time"
)

var bufWriteSize = 64 * 1024

type Result struct {
	Type     string
	Id       string
	Error    bool
	Start    time.Time
	Duration time.Duration
	// TODO: should we just do a map[string]interface{}?
	Meta    map[string]string
	Metrics map[string]float64
}

// Calling Done() multiple times is OK, some Tasks will call it before
// they return, because they spend significant time extracting metadata.
func (r *Result) Done() {
	if r.Duration == 0 {
		r.Duration = time.Since(r.Start)
	}
}

func (r *Result) MarshalJSON() ([]byte, error) {

	errb := `false`
	if r.Error {
		errb = `true`
	}

	start, err := r.Start.MarshalJSON()
	if err != nil {
		return nil, err
	}

	duration := strconv.AppendInt([]byte{}, int64(r.Duration), 10)

	meta, err := json.Marshal(r.Meta)
	if err != nil {
		return nil, err
	}

	metrics, err := json.Marshal(r.Metrics)
	if err != nil {
		return nil, err
	}

	// TODO: benchmark
	return bytes.Join(
		[][]byte{
			[]byte(`{`),
			[]byte(`"Type":"` + r.Type + `"`),
			[]byte(`,"Id":"` + r.Id + `"`),
			[]byte(`,"Error":"` + errb + `"`),
			[]byte(`,"Start":`),
			start,
			[]byte(`,"Duration":`),
			duration,
			[]byte(`,"Meta":`),
			meta,
			[]byte(`,"Metrics":`),
			metrics,
			[]byte(`}`),
		},
		[]byte{}), nil
}

func NewResult(taskType string, id string) *Result {
	r := &Result{Id: id, Type: taskType}
	r.Meta = make(map[string]string)
	r.Metrics = make(map[string]float64)
	return r
}

type ResultArchiveWriter struct {
	Path    string
	fwriter *os.File
	gwriter *gzip.Writer
	writer  *bufio.Writer
}

func (raw *ResultArchiveWriter) Write(rv *Result) error {
	b, err := json.Marshal(rv)
	if err != nil {
		return err
	}
	raw.writer.Write(b)
	raw.writer.WriteString("\n")
	return nil
}

func (raw *ResultArchiveWriter) Close() error {
	raw.writer.Flush()
	raw.gwriter.Close()
	return raw.fwriter.Close()
}

func (raw *ResultArchiveWriter) Remove() error {
	return os.Remove(raw.Path)
}

func NewResultArchiveWriter() *ResultArchiveWriter {
	tfile, err := ioutil.TempFile("", "hurlgz")
	if err != nil {
		panic(err)
	}

	gwriter, err := gzip.NewWriterLevel(tfile, gzip.BestSpeed)

	if err != nil {
		panic(err)
	}

	return &ResultArchiveWriter{
		Path:    tfile.Name(),
		fwriter: tfile,
		gwriter: gwriter,
		writer:  bufio.NewWriterSize(gwriter, bufWriteSize),
	}
}

type ResultArchiveReader struct {
	Path    string
	rrfile  *os.File
	gfile   *gzip.Reader
	scanner *bufio.Scanner
}

func NewResultArchiveReader(path string) *ResultArchiveReader {
	rar := &ResultArchiveReader{Path: path}
	err := rar.open(path)
	if err != nil {
		panic(err)
	}

	return rar
}

func (rar *ResultArchiveReader) Close() error {
	rar.scanner = nil
	if rar.gfile != nil {
		rar.gfile.Close()
		rar.gfile = nil
	}
	if rar.rrfile != nil {
		rar.rrfile.Close()
		rar.rrfile = nil
	}
	return nil
}

func (rr *ResultArchiveReader) Reset() {
	rr.Close()
	rr.open(rr.Path)
}

func (rr *ResultArchiveReader) open(path string) error {
	var err error
	rr.rrfile, err = os.Open(path)
	if err != nil {
		return err
	}

	rr.gfile, err = gzip.NewReader(rr.rrfile)
	if err != nil {
		return err
	}

	rr.scanner = bufio.NewScanner(rr.gfile)
	rr.scanner.Split(bufio.ScanLines)
	return nil
}

func (rr *ResultArchiveReader) Entry() *Result {
	b := rr.scanner.Bytes()
	rv := &Result{}
	json.Unmarshal(b, rv)
	return rv
}

func (rr *ResultArchiveReader) Scan() bool {
	return rr.scanner.Scan()
}
<|endoftext|>


<commit_before>package driver

import (
	"context"
	"fmt"
	"log"
	"regexp"
	"sync"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/hashicorp/nomad/nomad/structs"
)

var (
	// createCoordinator allows us to only create a single coordinator
	createCoordinator sync.Once

	// globalCoordinator is the shared coordinator and should only be retreived
	// using the GetDockerCoordinator() method.
	globalCoordinator *dockerCoordinator

	// imageNotFoundMatcher is a regex expression that matches the image not
	// found error Docker returns.
	imageNotFoundMatcher = regexp.MustCompile(`Error: image .+ not found`)
)

// pullFuture is a sharable future for retrieving a pulled images ID and any
// error that may have occurred during the pull.
type pullFuture struct {
	waitCh chan struct{}

	err     error
	imageID string
}

// newPullFuture returns a new pull future
func newPullFuture() *pullFuture {
	return &pullFuture{
		waitCh: make(chan struct{}),
	}
}

// wait waits till the future has a result
func (p *pullFuture) wait() *pullFuture {
	<-p.waitCh
	return p
}

// result returns the results of the future and should only ever be called after
// wait returns.
func (p *pullFuture) result() (imageID string, err error) {
	return p.imageID, p.err
}

// set is used to set the results and unblock any waiter. This may only be
// called once.
func (p *pullFuture) set(imageID string, err error) {
	p.imageID = imageID
	p.err = err
	close(p.waitCh)
}

// DockerImageClient provides the methods required to do CRUD operations on the
// Docker images
type DockerImageClient interface {
	PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) error
	InspectImage(id string) (*docker.Image, error)
	RemoveImage(id string) error
}

// dockerCoordinatorConfig is used to configure the Docker coordinator.
type dockerCoordinatorConfig struct {
	// logger is the logger the coordinator should use
	logger *log.Logger

	// cleanup marks whether images should be deleting when the reference count
	// is zero
	cleanup bool

	// client is the Docker client to use for communicating with Docker
	client DockerImageClient

	// removeDelay is the delay between an image's reference count going to
	// zero and the image actually being deleted.
	removeDelay time.Duration
}

// dockerCoordinator is used to coordinate actions against images to prevent
// racy deletions. It can be thought of as a reference counter on images.
type dockerCoordinator struct {
	*dockerCoordinatorConfig

	// imageLock is used to lock access to all images
	imageLock sync.Mutex

	// pullFutures is used to allow multiple callers to pull the same image but
	// only have one request be sent to Docker
	pullFutures map[string]*pullFuture

	// imageRefCount is the reference count of image IDs
	imageRefCount map[string]map[string]struct{}

	// deleteFuture is indexed by image ID and has a cancable delete future
	deleteFuture map[string]context.CancelFunc
}

// NewDockerCoordinator returns a new Docker coordinator
func NewDockerCoordinator(config *dockerCoordinatorConfig) *dockerCoordinator {
	if config.client == nil {
		return nil
	}

	return &dockerCoordinator{
		dockerCoordinatorConfig: config,
		pullFutures:             make(map[string]*pullFuture),
		imageRefCount:           make(map[string]map[string]struct{}),
		deleteFuture:            make(map[string]context.CancelFunc),
	}
}

// GetDockerCoordinator returns the shared dockerCoordinator instance
func GetDockerCoordinator(config *dockerCoordinatorConfig) *dockerCoordinator {
	createCoordinator.Do(func() {
		globalCoordinator = NewDockerCoordinator(config)
	})

	return globalCoordinator
}

// PullImage is used to pull an image. It returns the pulled imaged ID or an
// error that occurred during the pull
func (d *dockerCoordinator) PullImage(image string, authOptions *docker.AuthConfiguration, callerID string) (imageID string, err error) {
	// Get the future
	d.imageLock.Lock()
	future, ok := d.pullFutures[image]
	if !ok {
		// Make the future
		future = newPullFuture()
		d.pullFutures[image] = future
		go d.pullImageImpl(image, authOptions, future)
	}
	d.imageLock.Unlock()

	// We unlock while we wait since this can take a while
	id, err := future.wait().result()

	d.imageLock.Lock()
	defer d.imageLock.Unlock()

	// Delete the future since we don't need it and we don't want to cache an
	// image being there if it has possibly been manually deleted (outside of
	// Nomad).
	if _, ok := d.pullFutures[image]; ok {
		delete(d.pullFutures, image)
	}

	// If we are cleaning up, we increment the reference count on the image
	if err == nil && d.cleanup {
		d.incrementImageReferenceImpl(id, image, callerID)
	}

	return id, err
}

// pullImageImpl is the implementation of pulling an image. The results are
// returned via the passed future
func (d *dockerCoordinator) pullImageImpl(image string, authOptions *docker.AuthConfiguration, future *pullFuture) {
	// Parse the repo and tag
	repo, tag := docker.ParseRepositoryTag(image)
	if tag == "" {
		tag = "latest"
	}
	pullOptions := docker.PullImageOptions{
		Repository: repo,
		Tag:        tag,
	}

	// Attempt to pull the image
	var auth docker.AuthConfiguration
	if authOptions != nil {
		auth = *authOptions
	}
	err := d.client.PullImage(pullOptions, auth)
	if err != nil {
		d.logger.Printf("[ERR] driver.docker: failed pulling container %s:%s: %s", repo, tag, err)
		future.set("", recoverablePullError(err, image))
		return
	}

	d.logger.Printf("[DEBUG] driver.docker: docker pull %s:%s succeeded", repo, tag)

	dockerImage, err := d.client.InspectImage(image)
	if err != nil {
		d.logger.Printf("[ERR] driver.docker: failed getting image id for %q: %v", image, err)
		future.set("", recoverableErrTimeouts(err))
		return
	}

	future.set(dockerImage.ID, nil)
	return
}

// IncrementImageReference is used to increment an image reference count
func (d *dockerCoordinator) IncrementImageReference(imageID, imageName, callerID string) {
	d.imageLock.Lock()
	defer d.imageLock.Unlock()
	if d.cleanup {
		d.incrementImageReferenceImpl(imageID, imageName, callerID)
	}
}

// incrementImageReferenceImpl assumes the lock is held
func (d *dockerCoordinator) incrementImageReferenceImpl(imageID, imageName, callerID string) {
	// Cancel any pending delete
	if cancel, ok := d.deleteFuture[imageID]; ok {
		d.logger.Printf("[DEBUG] driver.docker: cancelling removal of image %q", imageName)
		cancel()
		delete(d.deleteFuture, imageID)
	}

	// Increment the reference
	references, ok := d.imageRefCount[imageID]
	if !ok {
		references = make(map[string]struct{})
		d.imageRefCount[imageID] = references
	}

	if _, ok := references[callerID]; !ok {
		references[callerID] = struct{}{}
		d.logger.Printf("[DEBUG] driver.docker: image %q (%v) reference count incremented: %d", imageName, imageID, len(references))
	}
}

// RemoveImage removes the given image. If there are any errors removing the
// image, the remove is retried internally.
func (d *dockerCoordinator) RemoveImage(imageID, callerID string) {
	d.imageLock.Lock()
	defer d.imageLock.Unlock()

	if !d.cleanup {
		return
	}

	references, ok := d.imageRefCount[imageID]
	if !ok {
		d.logger.Printf("[WARN] driver.docker: RemoveImage on non-referenced counted image id %q", imageID)
		return
	}

	// Decrement the reference count
	delete(references, callerID)
	count := len(references)
	d.logger.Printf("[DEBUG] driver.docker: image id %q reference count decremented: %d", imageID, count)

	// Nothing to do
	if count != 0 {
		return
	}

	// This should never be the case but we safefty guard so we don't leak a
	// cancel.
	if cancel, ok := d.deleteFuture[imageID]; ok {
		d.logger.Printf("[ERR] driver.docker: image id %q has lingering delete future", imageID)
		cancel()
	}

	// Setup a future to delete the image
	ctx, cancel := context.WithCancel(context.Background())
	d.deleteFuture[imageID] = cancel
	go d.removeImageImpl(imageID, ctx)

	// Delete the key from the reference count
	delete(d.imageRefCount, imageID)
}

// removeImageImpl is used to remove an image. It wil wait the specified remove
// delay to remove the image. If the context is cancalled before that the image
// removal will be cancelled.
func (d *dockerCoordinator) removeImageImpl(id string, ctx context.Context) {
	// Wait for the delay or a cancellation event
	select {
	case <-ctx.Done():
		// We have been cancelled
		return
	case <-time.After(d.removeDelay):
	}

	// Ensure we are suppose to delete. Do a short check while holding the lock
	// so there can't be interleaving. There is still the smallest chance that
	// the delete occurs after the image has been pulled but before it has been
	// incremented. For handling that we just treat it as a recoverable error in
	// the docker driver.
	d.imageLock.Lock()
	select {
	case <-ctx.Done():
		d.imageLock.Unlock()
		return
	default:
	}
	d.imageLock.Unlock()

	for i := 0; i < 3; i++ {
		err := d.client.RemoveImage(id)
		if err == nil {
			break
		}

		if err == docker.ErrNoSuchImage {
			d.logger.Printf("[DEBUG] driver.docker: unable to cleanup image %q: does not exist", id)
			return
		}
		if derr, ok := err.(*docker.Error); ok && derr.Status == 409 {
			d.logger.Printf("[DEBUG] driver.docker: unable to cleanup image %q: still in use", id)
			return
		}

		// Retry on unknown errors
		d.logger.Printf("[DEBUG] driver.docker: failed to remove image %q (attempt %d): %v", id, i+1, err)

		select {
		case <-ctx.Done():
			// We have been cancelled
			return
		case <-time.After(3 * time.Second):
		}
	}

	d.logger.Printf("[DEBUG] driver.docker: cleanup removed downloaded image: %q", id)

	// Cleanup the future from the map and free the context by cancelling it
	d.imageLock.Lock()
	if cancel, ok := d.deleteFuture[id]; ok {
		delete(d.deleteFuture, id)
		cancel()
	}
	d.imageLock.Unlock()
}

// recoverablePullError wraps the error gotten when trying to pull and image if
// the error is recoverable.
func recoverablePullError(err error, image string) error {
	recoverable := true
	if imageNotFoundMatcher.MatchString(err.Error()) {
		recoverable = false
	}
	return structs.NewRecoverableError(fmt.Errorf("Failed to pull `%s`: %s", image, err), recoverable)
}
<commit_msg>spelling: cancelable<commit_after>package driver

import (
	"context"
	"fmt"
	"log"
	"regexp"
	"sync"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/hashicorp/nomad/nomad/structs"
)

var (
	// createCoordinator allows us to only create a single coordinator
	createCoordinator sync.Once

	// globalCoordinator is the shared coordinator and should only be retreived
	// using the GetDockerCoordinator() method.
	globalCoordinator *dockerCoordinator

	// imageNotFoundMatcher is a regex expression that matches the image not
	// found error Docker returns.
	imageNotFoundMatcher = regexp.MustCompile(`Error: image .+ not found`)
)

// pullFuture is a sharable future for retrieving a pulled images ID and any
// error that may have occurred during the pull.
type pullFuture struct {
	waitCh chan struct{}

	err     error
	imageID string
}

// newPullFuture returns a new pull future
func newPullFuture() *pullFuture {
	return &pullFuture{
		waitCh: make(chan struct{}),
	}
}

// wait waits till the future has a result
func (p *pullFuture) wait() *pullFuture {
	<-p.waitCh
	return p
}

// result returns the results of the future and should only ever be called after
// wait returns.
func (p *pullFuture) result() (imageID string, err error) {
	return p.imageID, p.err
}

// set is used to set the results and unblock any waiter. This may only be
// called once.
func (p *pullFuture) set(imageID string, err error) {
	p.imageID = imageID
	p.err = err
	close(p.waitCh)
}

// DockerImageClient provides the methods required to do CRUD operations on the
// Docker images
type DockerImageClient interface {
	PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) error
	InspectImage(id string) (*docker.Image, error)
	RemoveImage(id string) error
}

// dockerCoordinatorConfig is used to configure the Docker coordinator.
type dockerCoordinatorConfig struct {
	// logger is the logger the coordinator should use
	logger *log.Logger

	// cleanup marks whether images should be deleting when the reference count
	// is zero
	cleanup bool

	// client is the Docker client to use for communicating with Docker
	client DockerImageClient

	// removeDelay is the delay between an image's reference count going to
	// zero and the image actually being deleted.
	removeDelay time.Duration
}

// dockerCoordinator is used to coordinate actions against images to prevent
// racy deletions. It can be thought of as a reference counter on images.
type dockerCoordinator struct {
	*dockerCoordinatorConfig

	// imageLock is used to lock access to all images
	imageLock sync.Mutex

	// pullFutures is used to allow multiple callers to pull the same image but
	// only have one request be sent to Docker
	pullFutures map[string]*pullFuture

	// imageRefCount is the reference count of image IDs
	imageRefCount map[string]map[string]struct{}

	// deleteFuture is indexed by image ID and has a cancelable delete future
	deleteFuture map[string]context.CancelFunc
}

// NewDockerCoordinator returns a new Docker coordinator
func NewDockerCoordinator(config *dockerCoordinatorConfig) *dockerCoordinator {
	if config.client == nil {
		return nil
	}

	return &dockerCoordinator{
		dockerCoordinatorConfig: config,
		pullFutures:             make(map[string]*pullFuture),
		imageRefCount:           make(map[string]map[string]struct{}),
		deleteFuture:            make(map[string]context.CancelFunc),
	}
}

// GetDockerCoordinator returns the shared dockerCoordinator instance
func GetDockerCoordinator(config *dockerCoordinatorConfig) *dockerCoordinator {
	createCoordinator.Do(func() {
		globalCoordinator = NewDockerCoordinator(config)
	})

	return globalCoordinator
}

// PullImage is used to pull an image. It returns the pulled imaged ID or an
// error that occurred during the pull
func (d *dockerCoordinator) PullImage(image string, authOptions *docker.AuthConfiguration, callerID string) (imageID string, err error) {
	// Get the future
	d.imageLock.Lock()
	future, ok := d.pullFutures[image]
	if !ok {
		// Make the future
		future = newPullFuture()
		d.pullFutures[image] = future
		go d.pullImageImpl(image, authOptions, future)
	}
	d.imageLock.Unlock()

	// We unlock while we wait since this can take a while
	id, err := future.wait().result()

	d.imageLock.Lock()
	defer d.imageLock.Unlock()

	// Delete the future since we don't need it and we don't want to cache an
	// image being there if it has possibly been manually deleted (outside of
	// Nomad).
	if _, ok := d.pullFutures[image]; ok {
		delete(d.pullFutures, image)
	}

	// If we are cleaning up, we increment the reference count on the image
	if err == nil && d.cleanup {
		d.incrementImageReferenceImpl(id, image, callerID)
	}

	return id, err
}

// pullImageImpl is the implementation of pulling an image. The results are
// returned via the passed future
func (d *dockerCoordinator) pullImageImpl(image string, authOptions *docker.AuthConfiguration, future *pullFuture) {
	// Parse the repo and tag
	repo, tag := docker.ParseRepositoryTag(image)
	if tag == "" {
		tag = "latest"
	}
	pullOptions := docker.PullImageOptions{
		Repository: repo,
		Tag:        tag,
	}

	// Attempt to pull the image
	var auth docker.AuthConfiguration
	if authOptions != nil {
		auth = *authOptions
	}
	err := d.client.PullImage(pullOptions, auth)
	if err != nil {
		d.logger.Printf("[ERR] driver.docker: failed pulling container %s:%s: %s", repo, tag, err)
		future.set("", recoverablePullError(err, image))
		return
	}

	d.logger.Printf("[DEBUG] driver.docker: docker pull %s:%s succeeded", repo, tag)

	dockerImage, err := d.client.InspectImage(image)
	if err != nil {
		d.logger.Printf("[ERR] driver.docker: failed getting image id for %q: %v", image, err)
		future.set("", recoverableErrTimeouts(err))
		return
	}

	future.set(dockerImage.ID, nil)
	return
}

// IncrementImageReference is used to increment an image reference count
func (d *dockerCoordinator) IncrementImageReference(imageID, imageName, callerID string) {
	d.imageLock.Lock()
	defer d.imageLock.Unlock()
	if d.cleanup {
		d.incrementImageReferenceImpl(imageID, imageName, callerID)
	}
}

// incrementImageReferenceImpl assumes the lock is held
func (d *dockerCoordinator) incrementImageReferenceImpl(imageID, imageName, callerID string) {
	// Cancel any pending delete
	if cancel, ok := d.deleteFuture[imageID]; ok {
		d.logger.Printf("[DEBUG] driver.docker: cancelling removal of image %q", imageName)
		cancel()
		delete(d.deleteFuture, imageID)
	}

	// Increment the reference
	references, ok := d.imageRefCount[imageID]
	if !ok {
		references = make(map[string]struct{})
		d.imageRefCount[imageID] = references
	}

	if _, ok := references[callerID]; !ok {
		references[callerID] = struct{}{}
		d.logger.Printf("[DEBUG] driver.docker: image %q (%v) reference count incremented: %d", imageName, imageID, len(references))
	}
}

// RemoveImage removes the given image. If there are any errors removing the
// image, the remove is retried internally.
func (d *dockerCoordinator) RemoveImage(imageID, callerID string) {
	d.imageLock.Lock()
	defer d.imageLock.Unlock()

	if !d.cleanup {
		return
	}

	references, ok := d.imageRefCount[imageID]
	if !ok {
		d.logger.Printf("[WARN] driver.docker: RemoveImage on non-referenced counted image id %q", imageID)
		return
	}

	// Decrement the reference count
	delete(references, callerID)
	count := len(references)
	d.logger.Printf("[DEBUG] driver.docker: image id %q reference count decremented: %d", imageID, count)

	// Nothing to do
	if count != 0 {
		return
	}

	// This should never be the case but we safefty guard so we don't leak a
	// cancel.
	if cancel, ok := d.deleteFuture[imageID]; ok {
		d.logger.Printf("[ERR] driver.docker: image id %q has lingering delete future", imageID)
		cancel()
	}

	// Setup a future to delete the image
	ctx, cancel := context.WithCancel(context.Background())
	d.deleteFuture[imageID] = cancel
	go d.removeImageImpl(imageID, ctx)

	// Delete the key from the reference count
	delete(d.imageRefCount, imageID)
}

// removeImageImpl is used to remove an image. It wil wait the specified remove
// delay to remove the image. If the context is cancalled before that the image
// removal will be cancelled.
func (d *dockerCoordinator) removeImageImpl(id string, ctx context.Context) {
	// Wait for the delay or a cancellation event
	select {
	case <-ctx.Done():
		// We have been cancelled
		return
	case <-time.After(d.removeDelay):
	}

	// Ensure we are suppose to delete. Do a short check while holding the lock
	// so there can't be interleaving. There is still the smallest chance that
	// the delete occurs after the image has been pulled but before it has been
	// incremented. For handling that we just treat it as a recoverable error in
	// the docker driver.
	d.imageLock.Lock()
	select {
	case <-ctx.Done():
		d.imageLock.Unlock()
		return
	default:
	}
	d.imageLock.Unlock()

	for i := 0; i < 3; i++ {
		err := d.client.RemoveImage(id)
		if err == nil {
			break
		}

		if err == docker.ErrNoSuchImage {
			d.logger.Printf("[DEBUG] driver.docker: unable to cleanup image %q: does not exist", id)
			return
		}
		if derr, ok := err.(*docker.Error); ok && derr.Status == 409 {
			d.logger.Printf("[DEBUG] driver.docker: unable to cleanup image %q: still in use", id)
			return
		}

		// Retry on unknown errors
		d.logger.Printf("[DEBUG] driver.docker: failed to remove image %q (attempt %d): %v", id, i+1, err)

		select {
		case <-ctx.Done():
			// We have been cancelled
			return
		case <-time.After(3 * time.Second):
		}
	}

	d.logger.Printf("[DEBUG] driver.docker: cleanup removed downloaded image: %q", id)

	// Cleanup the future from the map and free the context by cancelling it
	d.imageLock.Lock()
	if cancel, ok := d.deleteFuture[id]; ok {
		delete(d.deleteFuture, id)
		cancel()
	}
	d.imageLock.Unlock()
}

// recoverablePullError wraps the error gotten when trying to pull and image if
// the error is recoverable.
func recoverablePullError(err error, image string) error {
	recoverable := true
	if imageNotFoundMatcher.MatchString(err.Error()) {
		recoverable = false
	}
	return structs.NewRecoverableError(fmt.Errorf("Failed to pull `%s`: %s", image, err), recoverable)
}
<|endoftext|>


<commit_before>package main

import "github.com/lightningnetwork/lnd/lnwire"

// globalFeatures feature vector which affects HTLCs and thus are also
// advertised to other nodes.
var globalFeatures = lnwire.NewFeatureVector([]lnwire.Feature{})

// localFeatures is an feature vector which represent the features which
// only affect the protocol between these two nodes.
var localFeatures = lnwire.NewFeatureVector([]lnwire.Feature{
	{
		Name: "new-ping-and-funding",
		Flag: lnwire.RequiredFlag,
	},
	{
		Name: "node-ann-feature-addr-swap",
		Flag: lnwire.RequiredFlag,
	},
	{
		Name: "dynamic-fees",
		Flag: lnwire.RequiredFlag,
	},
	{
		Name: "shutdown-close-flow",
		Flag: lnwire.RequiredFlag,
	},
	{
		Name: "sphinx-payload",
		Flag: lnwire.RequiredFlag,
	},
	{
		Name: "htlc-dust-accounting",
		Flag: lnwire.RequiredFlag,
	},
	{
		Name: "encrypted-errors",
		Flag: lnwire.RequiredFlag,
	},
	{
		Name: "new-funding-and-commitment",
		Flag: lnwire.RequiredFlag,
	},
})
<commit_msg>lnd: reset features as we no longer need to partition dated lnd nodes<commit_after>package main

import "github.com/lightningnetwork/lnd/lnwire"

// globalFeatures feature vector which affects HTLCs and thus are also
// advertised to other nodes.
var globalFeatures = lnwire.NewFeatureVector([]lnwire.Feature{})

// localFeatures is an feature vector which represent the features which
// only affect the protocol between these two nodes.
//
// TODO(roasbeef): update to only have one, add a dummy vector?
var localFeatures = lnwire.NewFeatureVector([]lnwire.Feature{
	{
		Name: "filler",
		Flag: lnwire.OptionalFlag,
	},
	{
		Name: "announce-graph",
		Flag: lnwire.OptionalFlag,
	},
})
<|endoftext|>


<commit_before>package ei

import (
	"net/http"
	"bytes"
	"github.com/nightrune/wrench/logging"
)

const EI_URL = "https://build.electricimp.com/v4/"

type Device struct {
  Id string `json:"id"`
  Name string `json:"name"`
  ModelId string `json:"model_id"`
  PowerState string `json:"powerstate"`
  Rssi int `json:"rssi"`
  AgentId string `json:"agent_id"`
  AgentStatus string `json:"agent_status"`
}

type Model struct {
  Id string `json:"id"`
  Name string `json:"name"`
  Devices []string `json:"devices"`
}

func Concat(a string, b string) string {
	var buffer bytes.Buffer
	buffer.WriteString(a)
	buffer.WriteString(b)
    return buffer.String()
}

func ListModels() []Model {
  data := make([]byte, 100)
  url := Concat(EI_URL, "model")
  resp, err := http.Get(url)
  if err == nil {
  	resp.Body.Read(data)
    logging.Debug(string(data))
    return nil
  } else {
  	logging.Debug("An error happened, %s", err.Error())
  	return nil
  }
}

/*
func SearchModels(creds) []Model {

}
*/<commit_msg>Small changes to support authorization<commit_after>package ei

import (
	"net/http"
	"bytes"
	"github.com/nightrune/wrench/logging"
  "encoding/base64"
)

const EI_URL = "https://build.electricimp.com/v4/"

type Device struct {
  Id string `json:"id"`
  Name string `json:"name"`
  ModelId string `json:"model_id"`
  PowerState string `json:"powerstate"`
  Rssi int `json:"rssi"`
  AgentId string `json:"agent_id"`
  AgentStatus string `json:"agent_status"`
}

type Model struct {
  Id string `json:"id"`
  Name string `json:"name"`
  Devices []string `json:"devices"`
}

func Concat(a string, b string) string {
	var buffer bytes.Buffer
	buffer.WriteString(a)
	buffer.WriteString(b)
    return buffer.String()
}

func ListModels() []Model {
  data := make([]byte, 100)
  url := Concat(EI_URL, "model")
  client := &http.Client{}
  req, _ := http.NewRequest("GET", url, nil)
  cred_data := []byte("1d2da2b7e4e35667283af41ba2458527")
  creds := base64.StdEncoding.EncodeToString(cred_data)
  req.Header.Set("Authorization", "Basic " + creds)
  resp, err := client.Do(req)
  if err == nil {
  	resp.Body.Read(data)
    logging.Debug(string(data))
    return nil
  } else {
  	logging.Debug("An error happened, %s", err.Error())
  	return nil
  }
}

/*
func SearchModels(creds) []Model {

}
*/<|endoftext|>