<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	ApplicationName = "prometheus_varnish_exporter"
	Version         string
	VersionHash     string
	VersionDate     string

	PrometheusExporter = NewPrometheusExporter()
	VarnishVersion     = NewVarnishVersion()
	ExitHandler        = &exitHandler{}

	StartParams = &startParams{
		ListenAddress:  ":9131", // Reserved and publicly announced at https://github.com/prometheus/prometheus/wiki/Default-port-allocations
		Path:           "/metrics",
		VarnishstatExe: "varnishstat",
		Params:         &varnishstatParams{},
	}
	logger *log.Logger
)

type startParams struct {
	ListenAddress          string
	Path                   string
	HealthPath             string
	VarnishstatExe         string
	VarnishDockerContainer string
	Params                 *varnishstatParams

	Verbose       bool
	NoExit        bool
	Test          bool
	Raw           bool
	WithGoMetrics bool
}

type varnishstatParams struct {
	Instance string
	VSM      string
}

func (p *varnishstatParams) isEmpty() bool {
	return p.Instance == "" && p.VSM == ""
}

func (p *varnishstatParams) make() (params []string) {
	// -n
	if p.Instance != "" {
		params = append(params, "-n", p.Instance)
	}
	// -N is not supported by 3.x
	if p.VSM != "" && VarnishVersion.EqualsOrGreater(4, 0) {
		params = append(params, "-N", p.VSM)
	}
	return params
}

func init() {
	// prometheus conventions
	flag.StringVar(&StartParams.ListenAddress, "web.listen-address", StartParams.ListenAddress, "Address on which to expose metrics and web interface.")
	flag.StringVar(&StartParams.Path, "web.telemetry-path", StartParams.Path, "Path under which to expose metrics.")
	flag.StringVar(&StartParams.HealthPath, "web.health-path", StartParams.HealthPath, "Path under which to expose healthcheck. Disabled unless configured.")

	// varnish
	flag.StringVar(&StartParams.VarnishstatExe, "varnishstat-path", StartParams.VarnishstatExe, "Path to varnishstat.")
	flag.StringVar(&StartParams.Params.Instance, "n", StartParams.Params.Instance, "varnishstat -n value.")
	flag.StringVar(&StartParams.Params.VSM, "N", StartParams.Params.VSM, "varnishstat -N value.")

	// docker
	flag.StringVar(&StartParams.VarnishDockerContainer, "docker-container-name", StartParams.VarnishDockerContainer, "Docker container name to exec varnishstat in.")

	// modes
	version := false
	flag.BoolVar(&version, "version", version, "Print version and exit")
	flag.BoolVar(&StartParams.NoExit, "no-exit", StartParams.NoExit, "Do not exit server on Varnish scrape errors.")
	flag.BoolVar(&StartParams.Verbose, "verbose", StartParams.Verbose, "Verbose logging.")
	flag.BoolVar(&StartParams.Test, "test", StartParams.Test, "Test varnishstat availability, prints available metrics and exits.")
	flag.BoolVar(&StartParams.Raw, "raw", StartParams.Test, "Raw stdout logging without timestamps.")
	flag.BoolVar(&StartParams.WithGoMetrics, "with-go-metrics", StartParams.WithGoMetrics, "Export go runtime and http handler metrics")

	flag.Parse()

	if version {
		fmt.Printf("%s %s\n", ApplicationName, getVersion(true))
		os.Exit(0)
	}

	logger = log.New(os.Stdout, "", log.Ldate|log.Ltime)

	if len(StartParams.Path) == 0 || StartParams.Path[0] != '/' {
		logFatal("-web.telemetry-path cannot be empty and must start with a slash '/', given %q", StartParams.Path)
	}
	if len(StartParams.HealthPath) != 0 && StartParams.HealthPath[0] != '/' {
		logFatal("-web.health-path must start with a slash '/' if configured, given %q", StartParams.HealthPath)
	}
	if StartParams.Path == StartParams.HealthPath {
		logFatal("-web.telemetry-path and -web.health-path cannot have same value")
	}

	// This looks weird, but we want the start param to have default value
	// of the old behavior, not flip it with e.g. -exit. You have to
	// explicitly turn on the changed behavior.
	ExitHandler.exitOnError = StartParams.Test == true || StartParams.NoExit == false
}

func main() {
	if b, err := json.MarshalIndent(StartParams, "", "  "); err == nil {
		logInfo("%s %s %s", ApplicationName, getVersion(false), b)
	} else {
		logFatal(err.Error())
	}

	// Initialize
	if err := VarnishVersion.Initialize(); err != nil {
		ExitHandler.Errorf("Varnish version initialize failed: %s", err.Error())
	}
	if VarnishVersion.Valid() {
		logInfo("Found varnishstat %s", VarnishVersion)
		if err := PrometheusExporter.Initialize(); err != nil {
			logFatal("Prometheus exporter initialize failed: %s", err.Error())
		}
	}

	// Test to verify everything is ok before starting the server
	{
		metrics := make(chan prometheus.Metric)
		go func() {
			for m := range metrics {
				if StartParams.Test {
					logInfo("%s", m.Desc())
				}
			}
		}()
		tStart := time.Now()
		buf, err := ScrapeVarnish(metrics)
		close(metrics)
		if err == nil {
			logInfo("Test scrape done in %s", time.Now().Sub(tStart))
			logRaw("")
		} else {
			if len(buf) > 0 {
				logRaw("\n%s", buf)
			}
			ExitHandler.Errorf("Startup test: %s", err.Error())
		}
	}
	if StartParams.Test {
		return
	}

	// Start serving
	logInfo("Server starting on %s with metrics path %s", StartParams.ListenAddress, StartParams.Path)

	if !StartParams.WithGoMetrics {
		registry := prometheus.NewRegistry()
		if err := registry.Register(PrometheusExporter); err != nil {
			logFatal("registry.Register failed: %s", err.Error())
		}
		handler := promhttp.HandlerFor(registry, promhttp.HandlerOpts{
			ErrorLog: logger,
		})
		http.Handle(StartParams.Path, handler)
	} else {
		prometheus.MustRegister(PrometheusExporter)
		http.Handle(StartParams.Path, prometheus.Handler())
	}

	if StartParams.Path != "/" {
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			w.Write([]byte(`
    <title>Varnish Exporter</title>

Varnish Exporter

<a href="` + StartParams.Path + `">Metrics

`))
		})
	}
	if StartParams.HealthPath != "" {
		http.HandleFunc(StartParams.HealthPath, func(w http.ResponseWriter, r *http.Request) {
			// As noted in the "up" metric, needs some way to determine if everything is actually Ok.
			// For now, this just lets us check that we're accepting connections
			w.WriteHeader(http.StatusOK)
			fmt.Fprintln(w, "Ok")
		})
	}
	logFatalError(http.ListenAndServe(StartParams.ListenAddress, nil))
}

type exitHandler struct {
	sync.RWMutex
	exitOnError bool
	err         error
}

func (ex *exitHandler) Errorf(format string, a ...interface{}) error {
	return ex.Set(fmt.Errorf(format, a...))
}

func (ex *exitHandler) HasError() bool {
	ex.RLock()
	hasError := ex.err != nil
	ex.RUnlock()
	return hasError
}

func (ex *exitHandler) Set(err error) error {
	ex.Lock()
	defer ex.Unlock()

	if err == nil {
		ex.err = nil
		return nil
	}

	errDiffers := ex.err == nil || ex.err.Error() != err.Error()
	ex.err = err

	if ex.exitOnError {
		logFatal("%s", err.Error())
	} else if errDiffers {
		logError("%s", err.Error())
	}
	return err
}

func getVersion(date bool) (version string) {
	if Version == "" {
		return "dev"
	}
	version = fmt.Sprintf("v%s (%s)", Version, VersionHash)
	if date {
		version += " " + VersionDate
	}
	return version
}
<commit_msg>Fix prometheus/golang_client usage (#45) (#46)<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	ApplicationName = "prometheus_varnish_exporter"
	Version         string
	VersionHash     string
	VersionDate     string

	PrometheusExporter = NewPrometheusExporter()
	VarnishVersion     = NewVarnishVersion()
	ExitHandler        = &exitHandler{}

	StartParams = &startParams{
		ListenAddress:  ":9131", // Reserved and publicly announced at https://github.com/prometheus/prometheus/wiki/Default-port-allocations
		Path:           "/metrics",
		VarnishstatExe: "varnishstat",
		Params:         &varnishstatParams{},
	}
	logger *log.Logger
)

type startParams struct {
	ListenAddress          string
	Path                   string
	HealthPath             string
	VarnishstatExe         string
	VarnishDockerContainer string
	Params                 *varnishstatParams

	Verbose       bool
	NoExit        bool
	Test          bool
	Raw           bool
	WithGoMetrics bool
}

type varnishstatParams struct {
	Instance string
	VSM      string
}

func (p *varnishstatParams) isEmpty() bool {
	return p.Instance == "" && p.VSM == ""
}

func (p *varnishstatParams) make() (params []string) {
	// -n
	if p.Instance != "" {
		params = append(params, "-n", p.Instance)
	}
	// -N is not supported by 3.x
	if p.VSM != "" && VarnishVersion.EqualsOrGreater(4, 0) {
		params = append(params, "-N", p.VSM)
	}
	return params
}

func init() {
	// prometheus conventions
	flag.StringVar(&StartParams.ListenAddress, "web.listen-address", StartParams.ListenAddress, "Address on which to expose metrics and web interface.")
	flag.StringVar(&StartParams.Path, "web.telemetry-path", StartParams.Path, "Path under which to expose metrics.")
	flag.StringVar(&StartParams.HealthPath, "web.health-path", StartParams.HealthPath, "Path under which to expose healthcheck. Disabled unless configured.")

	// varnish
	flag.StringVar(&StartParams.VarnishstatExe, "varnishstat-path", StartParams.VarnishstatExe, "Path to varnishstat.")
	flag.StringVar(&StartParams.Params.Instance, "n", StartParams.Params.Instance, "varnishstat -n value.")
	flag.StringVar(&StartParams.Params.VSM, "N", StartParams.Params.VSM, "varnishstat -N value.")

	// docker
	flag.StringVar(&StartParams.VarnishDockerContainer, "docker-container-name", StartParams.VarnishDockerContainer, "Docker container name to exec varnishstat in.")

	// modes
	version := false
	flag.BoolVar(&version, "version", version, "Print version and exit")
	flag.BoolVar(&StartParams.NoExit, "no-exit", StartParams.NoExit, "Do not exit server on Varnish scrape errors.")
	flag.BoolVar(&StartParams.Verbose, "verbose", StartParams.Verbose, "Verbose logging.")
	flag.BoolVar(&StartParams.Test, "test", StartParams.Test, "Test varnishstat availability, prints available metrics and exits.")
	flag.BoolVar(&StartParams.Raw, "raw", StartParams.Test, "Raw stdout logging without timestamps.")
	flag.BoolVar(&StartParams.WithGoMetrics, "with-go-metrics", StartParams.WithGoMetrics, "Export go runtime and http handler metrics")

	flag.Parse()

	if version {
		fmt.Printf("%s %s\n", ApplicationName, getVersion(true))
		os.Exit(0)
	}

	logger = log.New(os.Stdout, "", log.Ldate|log.Ltime)

	if len(StartParams.Path) == 0 || StartParams.Path[0] != '/' {
		logFatal("-web.telemetry-path cannot be empty and must start with a slash '/', given %q", StartParams.Path)
	}
	if len(StartParams.HealthPath) != 0 && StartParams.HealthPath[0] != '/' {
		logFatal("-web.health-path must start with a slash '/' if configured, given %q", StartParams.HealthPath)
	}
	if StartParams.Path == StartParams.HealthPath {
		logFatal("-web.telemetry-path and -web.health-path cannot have same value")
	}

	// This looks weird, but we want the start param to have default value
	// of the old behavior, not flip it with e.g. -exit. You have to
	// explicitly turn on the changed behavior.
	ExitHandler.exitOnError = StartParams.Test == true || StartParams.NoExit == false
}

func main() {
	if b, err := json.MarshalIndent(StartParams, "", "  "); err == nil {
		logInfo("%s %s %s", ApplicationName, getVersion(false), b)
	} else {
		logFatal(err.Error())
	}

	// Initialize
	if err := VarnishVersion.Initialize(); err != nil {
		ExitHandler.Errorf("Varnish version initialize failed: %s", err.Error())
	}
	if VarnishVersion.Valid() {
		logInfo("Found varnishstat %s", VarnishVersion)
		if err := PrometheusExporter.Initialize(); err != nil {
			logFatal("Prometheus exporter initialize failed: %s", err.Error())
		}
	}

	// Test to verify everything is ok before starting the server
	{
		metrics := make(chan prometheus.Metric)
		go func() {
			for m := range metrics {
				if StartParams.Test {
					logInfo("%s", m.Desc())
				}
			}
		}()
		tStart := time.Now()
		buf, err := ScrapeVarnish(metrics)
		close(metrics)
		if err == nil {
			logInfo("Test scrape done in %s", time.Now().Sub(tStart))
			logRaw("")
		} else {
			if len(buf) > 0 {
				logRaw("\n%s", buf)
			}
			ExitHandler.Errorf("Startup test: %s", err.Error())
		}
	}
	if StartParams.Test {
		return
	}

	// Start serving
	logInfo("Server starting on %s with metrics path %s", StartParams.ListenAddress, StartParams.Path)

	if !StartParams.WithGoMetrics {
		registry := prometheus.NewRegistry()
		if err := registry.Register(PrometheusExporter); err != nil {
			logFatal("registry.Register failed: %s", err.Error())
		}
		handler := promhttp.HandlerFor(registry, promhttp.HandlerOpts{
			ErrorLog: logger,
		})
		http.Handle(StartParams.Path, handler)
	} else {
		prometheus.MustRegister(PrometheusExporter)
		http.Handle(StartParams.Path, promhttp.Handler())
	}

	if StartParams.Path != "/" {
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			w.Write([]byte(`
    <title>Varnish Exporter</title>

Varnish Exporter

<a href="` + StartParams.Path + `">Metrics

`))
		})
	}
	if StartParams.HealthPath != "" {
		http.HandleFunc(StartParams.HealthPath, func(w http.ResponseWriter, r *http.Request) {
			// As noted in the "up" metric, needs some way to determine if everything is actually Ok.
			// For now, this just lets us check that we're accepting connections
			w.WriteHeader(http.StatusOK)
			fmt.Fprintln(w, "Ok")
		})
	}
	logFatalError(http.ListenAndServe(StartParams.ListenAddress, nil))
}

type exitHandler struct {
	sync.RWMutex
	exitOnError bool
	err         error
}

func (ex *exitHandler) Errorf(format string, a ...interface{}) error {
	return ex.Set(fmt.Errorf(format, a...))
}

func (ex *exitHandler) HasError() bool {
	ex.RLock()
	hasError := ex.err != nil
	ex.RUnlock()
	return hasError
}

func (ex *exitHandler) Set(err error) error {
	ex.Lock()
	defer ex.Unlock()

	if err == nil {
		ex.err = nil
		return nil
	}

	errDiffers := ex.err == nil || ex.err.Error() != err.Error()
	ex.err = err

	if ex.exitOnError {
		logFatal("%s", err.Error())
	} else if errDiffers {
		logError("%s", err.Error())
	}
	return err
}

func getVersion(date bool) (version string) {
	if Version == "" {
		return "dev"
	}
	version = fmt.Sprintf("v%s (%s)", Version, VersionHash)
	if date {
		version += " " + VersionDate
	}
	return version
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/BurntSushi/toml"
	"io"
	"math"
	"os"
	"regexp"
	"runtime"
	"runtime/pprof"
	"sort"
	"strconv"
	"strings"
	"sync"
)

const (
	useProfile = false
)

type tomlConfig struct {
	RankingCount   int `toml:"ranking_count"`
	SlowCount      int `toml:"slow_count"`
	Scale          int
	EffectiveDigit int    `toml:"effective_digit"`
	LogFormat      string `toml:"log_format"`
	RequestIndex   int    `toml:"request_index"`
	StatusIndex    int    `toml:"status_index"`
	DurationIndex  int    `toml:"duration_index"`
	Bundles        map[string]bundleConfig
}

type bundleConfig struct {
	Name   string
	Regexp string
}

type Measure struct {
	Url    string
	Count  int
	Total  float64
	Mean   float64
	Stddev float64
	Min    float64
	P50    float64
	P90    float64
	P95    float64
	P99    float64
	Max    float64
	S2xx   int
	S3xx   int
	S4xx   int
	S5xx   int
}

type By func(a, b *Measure) bool

func (by By) Sort(measures []*Measure) {
	ms := &measureSorter{
		measures: measures,
		by:       by,
	}
	sort.Sort(ms)
}

type measureSorter struct {
	measures []*Measure
	by       func(a, b *Measure) bool
}

func (s *measureSorter) Len() int {
	return len(s.measures)
}

func (s *measureSorter) Swap(i, j int) {
	s.measures[i], s.measures[j] = s.measures[j], s.measures[i]
}

func (s *measureSorter) Less(i, j int) bool {
	return s.by(s.measures[i], s.measures[j])
}

type Column struct {
	Name    string
	Summary string
	Sort    By
}

var (
	columns = []*Column{
		&Column{Name: "Count", Summary: "Count", Sort: func(a, b *Measure) bool { return a.Count > b.Count }},
		&Column{Name: "Total", Summary: "Total", Sort: func(a, b *Measure) bool { return a.Total > b.Total }},
		&Column{Name: "Mean", Summary: "Mean", Sort: func(a, b *Measure) bool { return a.Mean > b.Mean }},
		&Column{Name: "Stddev", Summary: "Standard Deviation", Sort: func(a, b *Measure) bool { return a.Stddev > b.Stddev }},
		&Column{Name: "Min"},
		&Column{Name: "P50"},
		&Column{Name: "P90"},
		&Column{Name: "P95"},
		&Column{Name: "P99"},
		&Column{Name: "Max", Summary: "Maximum(100 Percentile)", Sort: func(a, b *Measure) bool { return a.Max > b.Max }},
		&Column{Name: "2xx"},
		&Column{Name: "3xx"},
		&Column{Name: "4xx"},
		&Column{Name: "5xx"},
	}
)

type ByTime []*Time

type Time struct {
	Url        string
	Time       float64
	StatusCode int
}

func (a ByTime) Len() int           { return len(a) }
func (a ByTime) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByTime) Less(i, j int) bool { return a[i].Time > a[j].Time }

func getIntegerDigitWidth(f float64) int {
	var w int
	switch {
	case f < 0:
		w++
		fallthrough
	case math.Abs(f) < 1:
		w++
	default:
		w += int(math.Log10(math.Abs(f)) + 1)
	}
	return w
}

func showMeasures(measures []*Measure) {
	MIN_COUNT_WIDTH := 5 // for title
	MIN_TOTAL_WIDTH := 2 + config.EffectiveDigit
	MIN_MEAN_WIDTH := 2 + config.EffectiveDigit*2
	MIN_MAX_WIDTH := 2 + config.EffectiveDigit
	MIN_STATUS_WIDTH := 3 // for title

	countWidth := MIN_COUNT_WIDTH // for title
	totalWidth := MIN_TOTAL_WIDTH
	meanWidth := MIN_MEAN_WIDTH
	maxWidth := MIN_MAX_WIDTH
	s2xxWidth := MIN_STATUS_WIDTH
	s3xxWidth := MIN_STATUS_WIDTH
	s4xxWidth := MIN_STATUS_WIDTH
	s5xxWidth := MIN_STATUS_WIDTH

	rankingCount := config.RankingCount
	if len(measures) < rankingCount {
		rankingCount = len(measures)
	}
	for i := 0; i < rankingCount; i++ {
		var w int
		w = getIntegerDigitWidth(float64(measures[i].Count))
		if countWidth < w {
			countWidth = w
		}
		w = getIntegerDigitWidth(measures[i].Total) + 1 + config.EffectiveDigit
		if totalWidth < w {
			totalWidth = w
		}
		w = getIntegerDigitWidth(measures[i].Mean) + 1 + config.EffectiveDigit*2
		if meanWidth < w {
			meanWidth = w
		}
		w = getIntegerDigitWidth(measures[i].Max) + 1 + config.EffectiveDigit
		if maxWidth < w {
			maxWidth = w
		}
		w = getIntegerDigitWidth(float64(measures[i].S2xx))
		if s2xxWidth < w {
			s2xxWidth = w
		}
		w = getIntegerDigitWidth(float64(measures[i].S3xx))
		if s3xxWidth < w {
			s3xxWidth = w
		}
		w = getIntegerDigitWidth(float64(measures[i].S4xx))
		if s4xxWidth < w {
			s4xxWidth = w
		}
		w = getIntegerDigitWidth(float64(measures[i].S5xx))
		if s5xxWidth < w {
			s5xxWidth = w
		}
	}

	var format string
	for _, column := range columns {
		switch column.Name {
		case "Count":
			fmt.Printf(fmt.Sprintf("%%%ds  ", countWidth), column.Name)
			format += fmt.Sprintf("%%%dd  ", countWidth)
		case "Total":
			fmt.Printf(fmt.Sprintf("%%%ds  ", totalWidth), column.Name)
			format += fmt.Sprintf("%%%d.%df  ", totalWidth, config.EffectiveDigit)
		case "Mean":
			fmt.Printf(fmt.Sprintf("%%%ds  ", meanWidth), column.Name)
			format += fmt.Sprintf("%%%d.%df  ", meanWidth, config.EffectiveDigit*2)
		case "Stddev":
			fmt.Printf(fmt.Sprintf("%%%ds  ", meanWidth), column.Name)
			format += fmt.Sprintf("%%%d.%df  ", meanWidth, config.EffectiveDigit*2)
		case "2xx":
			fmt.Printf(fmt.Sprintf("%%%ds  ", s2xxWidth), column.Name)
			format += fmt.Sprintf("%%%dd  ", s2xxWidth)
		case "3xx":
			fmt.Printf(fmt.Sprintf("%%%ds  ", s3xxWidth), column.Name)
			format += fmt.Sprintf("%%%dd  ", s3xxWidth)
		case "4xx":
			fmt.Printf(fmt.Sprintf("%%%ds  ", s4xxWidth), column.Name)
			format += fmt.Sprintf("%%%dd  ", s4xxWidth)
		case "5xx":
			fmt.Printf(fmt.Sprintf("%%%ds  ", s5xxWidth), column.Name)
			format += fmt.Sprintf("%%%dd  ", s5xxWidth)
		default:
			fmt.Printf(fmt.Sprintf("%%%ds  ", maxWidth), column.Name)
			format += fmt.Sprintf("%%%d.%df  ", maxWidth, config.EffectiveDigit)
		}
	}
	fmt.Printf("Request\n")
	format += "%s\n"

	for i := 0; i < rankingCount; i++ {
		m := measures[i]
		fmt.Printf(format, m.Count, m.Total, m.Mean, m.Stddev, m.Min, m.P50, m.P90, m.P95, m.P99, m.Max, m.S2xx, m.S3xx, m.S4xx, m.S5xx, m.Url)
	}
}

func showTop(allTimes []*Time) {
	sort.Sort(ByTime(allTimes))
	slowCount := config.SlowCount
	if len(allTimes) < slowCount {
		slowCount = len(allTimes)
	}
	fmt.Printf("TOP %d Slow Requests\n", slowCount)

	iWidth := getIntegerDigitWidth(float64(slowCount))
	topWidth := getIntegerDigitWidth(allTimes[0].Time) + 1 + config.EffectiveDigit
	f := fmt.Sprintf("%%%dd  %%%d.%df  %%s\n", iWidth, topWidth, config.EffectiveDigit)
	for i := 0; i < slowCount; i++ {
		fmt.Printf(f, i+1, allTimes[i].Time, allTimes[i].Url)
	}
}

var configFile string
var config tomlConfig

func init() {
	const (
		defaultConfigFile = "kataribe.toml"
		usage             = "configuration file"
	)
	flag.StringVar(&configFile, "conf", defaultConfigFile, usage)
	flag.StringVar(&configFile, "f", defaultConfigFile, usage+" (shorthand)")
	flag.Parse()
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	if useProfile {
		f, err := os.Create("/tmp/parse_access_log.prof")
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if _, err := toml.DecodeFile(configFile, &config); err != nil {
		fmt.Println(err)
		return
	}

	reader := bufio.NewReaderSize(os.Stdin, 4096)
	scale := math.Pow10(config.Scale)

	urlNormalizeRegexps := make(map[string]*regexp.Regexp)
	for _, bundle := range config.Bundles {
		urlNormalizeRegexps[bundle.Name] = regexp.MustCompile(bundle.Regexp)
	}

	ch := make(chan *Time)
	totals := make(map[string]float64)
	stddevs := make(map[string]float64)
	times := make(map[string][]float64)
	statusCode := make(map[string][]int)
	var allTimes []*Time

	var stddevWg sync.WaitGroup
	stddevWg.Add(1)
	go func() {
		defer stddevWg.Done()
		for time := range ch {
			totals[time.Url] += time.Time
			times[time.Url] = append(times[time.Url], time.Time)
			allTimes = append(allTimes, time)
			if statusCode[time.Url] == nil {
				statusCode[time.Url] = make([]int, 6)
			}
			statusCode[time.Url][time.StatusCode]++
		}
		for url, total := range totals {
			mean := total / float64(len(times[url]))
			for _, t := range times[url] {
				stddevs[url] += math.Pow(t-mean, 2)
			}
		}
	}()

	logParser := regexp.MustCompile(config.LogFormat)

	var wg sync.WaitGroup
	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF {
			break
		} else if err != nil {
			panic(err)
		}
		wg.Add(1)
		go func(line string) {
			defer wg.Done()
			submatch := logParser.FindAllStringSubmatch(strings.TrimSpace(line), -1)
			if len(submatch) > 0 {
				s := submatch[0]
				url := s[config.RequestIndex]
				for name, re := range urlNormalizeRegexps {
					if re.MatchString(url) {
						url = name
						break
					}
				}
				time, err := strconv.ParseFloat(s[config.DurationIndex], 10)
				if err == nil {
					time = time * scale
				} else {
					time = 0.000
				}
				statusCode, err := strconv.Atoi(string(s[config.StatusIndex][0]))
				if err != nil {
					statusCode = 0
				}
				ch <- &Time{Url: url, Time: time, StatusCode: statusCode}
			}
		}(line)
	}
	wg.Wait()
	close(ch)
	stddevWg.Wait()

	var measures []*Measure
	for url, total := range totals {
		sorted := times[url]
		sort.Float64s(sorted)
		count := len(sorted)
		measure := &Measure{
			Url:    url,
			Count:  count,
			Total:  total,
			Mean:   totals[url] / float64(count),
			Stddev: math.Sqrt(stddevs[url] / float64(count)),
			Min:    sorted[0],
			P50:    sorted[int(count*50/100)],
			P90:    sorted[int(count*90/100)],
			P95:    sorted[int(count*95/100)],
			P99:    sorted[int(count*99/100)],
			Max:    sorted[count-1],
			S2xx:   statusCode[url][2],
			S3xx:   statusCode[url][3],
			S4xx:   statusCode[url][4],
			S5xx:   statusCode[url][5],
		}
		measures = append(measures, measure)
	}

	for _, column := range columns {
		if column.Sort != nil {
			fmt.Printf("Sort By %s\n", column.Summary)
			By(column.Sort).Sort(measures)
			showMeasures(measures)
			fmt.Println()
		}
	}

	showTop(allTimes)
}
<commit_msg>Replace WaitGroup with chan<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/BurntSushi/toml"
	"io"
	"math"
	"os"
	"regexp"
	"runtime"
	"runtime/pprof"
	"sort"
	"strconv"
	"strings"
	"sync"
)

const (
	useProfile = false
)

type tomlConfig struct {
	RankingCount   int `toml:"ranking_count"`
	SlowCount      int `toml:"slow_count"`
	Scale          int
	EffectiveDigit int    `toml:"effective_digit"`
	LogFormat      string `toml:"log_format"`
	RequestIndex   int    `toml:"request_index"`
	StatusIndex    int    `toml:"status_index"`
	DurationIndex  int    `toml:"duration_index"`
	Bundles        map[string]bundleConfig
}

type bundleConfig struct {
	Name   string
	Regexp string
}

type Measure struct {
	Url    string
	Count  int
	Total  float64
	Mean   float64
	Stddev float64
	Min    float64
	P50    float64
	P90    float64
	P95    float64
	P99    float64
	Max    float64
	S2xx   int
	S3xx   int
	S4xx   int
	S5xx   int
}

type By func(a, b *Measure) bool

func (by By) Sort(measures []*Measure) {
	ms := &measureSorter{
		measures: measures,
		by:       by,
	}
	sort.Sort(ms)
}

type measureSorter struct {
	measures []*Measure
	by       func(a, b *Measure) bool
}

func (s *measureSorter) Len() int {
	return len(s.measures)
}

func (s *measureSorter) Swap(i, j int) {
	s.measures[i], s.measures[j] = s.measures[j], s.measures[i]
}

func (s *measureSorter) Less(i, j int) bool {
	return s.by(s.measures[i], s.measures[j])
}

type Column struct {
	Name    string
	Summary string
	Sort    By
}

var (
	columns = []*Column{
		&Column{Name: "Count", Summary: "Count", Sort: func(a, b *Measure) bool { return a.Count > b.Count }},
		&Column{Name: "Total", Summary: "Total", Sort: func(a, b *Measure) bool { return a.Total > b.Total }},
		&Column{Name: "Mean", Summary: "Mean", Sort: func(a, b *Measure) bool { return a.Mean > b.Mean }},
		&Column{Name: "Stddev", Summary: "Standard Deviation", Sort: func(a, b *Measure) bool { return a.Stddev > b.Stddev }},
		&Column{Name: "Min"},
		&Column{Name: "P50"},
		&Column{Name: "P90"},
		&Column{Name: "P95"},
		&Column{Name: "P99"},
		&Column{Name: "Max", Summary: "Maximum(100 Percentile)", Sort: func(a, b *Measure) bool { return a.Max > b.Max }},
		&Column{Name: "2xx"},
		&Column{Name: "3xx"},
		&Column{Name: "4xx"},
		&Column{Name: "5xx"},
	}
)

type ByTime []*Time

type Time struct {
	Url        string
	Time       float64
	StatusCode int
}

func (a ByTime) Len() int           { return len(a) }
func (a ByTime) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByTime) Less(i, j int) bool { return a[i].Time > a[j].Time }

func getIntegerDigitWidth(f float64) int {
	var w int
	switch {
	case f < 0:
		w++
		fallthrough
	case math.Abs(f) < 1:
		w++
	default:
		w += int(math.Log10(math.Abs(f)) + 1)
	}
	return w
}

func showMeasures(measures []*Measure) {
	MIN_COUNT_WIDTH := 5 // for title
	MIN_TOTAL_WIDTH := 2 + config.EffectiveDigit
	MIN_MEAN_WIDTH := 2 + config.EffectiveDigit*2
	MIN_MAX_WIDTH := 2 + config.EffectiveDigit
	MIN_STATUS_WIDTH := 3 // for title

	countWidth := MIN_COUNT_WIDTH // for title
	totalWidth := MIN_TOTAL_WIDTH
	meanWidth := MIN_MEAN_WIDTH
	maxWidth := MIN_MAX_WIDTH
	s2xxWidth := MIN_STATUS_WIDTH
	s3xxWidth := MIN_STATUS_WIDTH
	s4xxWidth := MIN_STATUS_WIDTH
	s5xxWidth := MIN_STATUS_WIDTH

	rankingCount := config.RankingCount
	if len(measures) < rankingCount {
		rankingCount = len(measures)
	}
	for i := 0; i < rankingCount; i++ {
		var w int
		w = getIntegerDigitWidth(float64(measures[i].Count))
		if countWidth < w {
			countWidth = w
		}
		w = getIntegerDigitWidth(measures[i].Total) + 1 + config.EffectiveDigit
		if totalWidth < w {
			totalWidth = w
		}
		w = getIntegerDigitWidth(measures[i].Mean) + 1 + config.EffectiveDigit*2
		if meanWidth < w {
			meanWidth = w
		}
		w = getIntegerDigitWidth(measures[i].Max) + 1 + config.EffectiveDigit
		if maxWidth < w {
			maxWidth = w
		}
		w = getIntegerDigitWidth(float64(measures[i].S2xx))
		if s2xxWidth < w {
			s2xxWidth = w
		}
		w = getIntegerDigitWidth(float64(measures[i].S3xx))
		if s3xxWidth < w {
			s3xxWidth = w
		}
		w = getIntegerDigitWidth(float64(measures[i].S4xx))
		if s4xxWidth < w {
			s4xxWidth = w
		}
		w = getIntegerDigitWidth(float64(measures[i].S5xx))
		if s5xxWidth < w {
			s5xxWidth = w
		}
	}

	var format string
	for _, column := range columns {
		switch column.Name {
		case "Count":
			fmt.Printf(fmt.Sprintf("%%%ds  ", countWidth), column.Name)
			format += fmt.Sprintf("%%%dd  ", countWidth)
		case "Total":
			fmt.Printf(fmt.Sprintf("%%%ds  ", totalWidth), column.Name)
			format += fmt.Sprintf("%%%d.%df  ", totalWidth, config.EffectiveDigit)
		case "Mean":
			fmt.Printf(fmt.Sprintf("%%%ds  ", meanWidth), column.Name)
			format += fmt.Sprintf("%%%d.%df  ", meanWidth, config.EffectiveDigit*2)
		case "Stddev":
			fmt.Printf(fmt.Sprintf("%%%ds  ", meanWidth), column.Name)
			format += fmt.Sprintf("%%%d.%df  ", meanWidth, config.EffectiveDigit*2)
		case "2xx":
			fmt.Printf(fmt.Sprintf("%%%ds  ", s2xxWidth), column.Name)
			format += fmt.Sprintf("%%%dd  ", s2xxWidth)
		case "3xx":
			fmt.Printf(fmt.Sprintf("%%%ds  ", s3xxWidth), column.Name)
			format += fmt.Sprintf("%%%dd  ", s3xxWidth)
		case "4xx":
			fmt.Printf(fmt.Sprintf("%%%ds  ", s4xxWidth), column.Name)
			format += fmt.Sprintf("%%%dd  ", s4xxWidth)
		case "5xx":
			fmt.Printf(fmt.Sprintf("%%%ds  ", s5xxWidth), column.Name)
			format += fmt.Sprintf("%%%dd  ", s5xxWidth)
		default:
			fmt.Printf(fmt.Sprintf("%%%ds  ", maxWidth), column.Name)
			format += fmt.Sprintf("%%%d.%df  ", maxWidth, config.EffectiveDigit)
		}
	}
	fmt.Printf("Request\n")
	format += "%s\n"

	for i := 0; i < rankingCount; i++ {
		m := measures[i]
		fmt.Printf(format, m.Count, m.Total, m.Mean, m.Stddev, m.Min, m.P50, m.P90, m.P95, m.P99, m.Max, m.S2xx, m.S3xx, m.S4xx, m.S5xx, m.Url)
	}
}

func showTop(allTimes []*Time) {
	sort.Sort(ByTime(allTimes))
	slowCount := config.SlowCount
	if len(allTimes) < slowCount {
		slowCount = len(allTimes)
	}
	fmt.Printf("TOP %d Slow Requests\n", slowCount)

	iWidth := getIntegerDigitWidth(float64(slowCount))
	topWidth := getIntegerDigitWidth(allTimes[0].Time) + 1 + config.EffectiveDigit
	f := fmt.Sprintf("%%%dd  %%%d.%df  %%s\n", iWidth, topWidth, config.EffectiveDigit)
	for i := 0; i < slowCount; i++ {
		fmt.Printf(f, i+1, allTimes[i].Time, allTimes[i].Url)
	}
}

var configFile string
var config tomlConfig

func init() {
	const (
		defaultConfigFile = "kataribe.toml"
		usage             = "configuration file"
	)
	flag.StringVar(&configFile, "conf", defaultConfigFile, usage)
	flag.StringVar(&configFile, "f", defaultConfigFile, usage+" (shorthand)")
	flag.Parse()
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	if useProfile {
		f, err := os.Create("/tmp/parse_access_log.prof")
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if _, err := toml.DecodeFile(configFile, &config); err != nil {
		fmt.Println(err)
		return
	}

	reader := bufio.NewReaderSize(os.Stdin, 4096)
	scale := math.Pow10(config.Scale)

	urlNormalizeRegexps := make(map[string]*regexp.Regexp)
	for _, bundle := range config.Bundles {
		urlNormalizeRegexps[bundle.Name] = regexp.MustCompile(bundle.Regexp)
	}

	ch := make(chan *Time)
	totals := make(map[string]float64)
	stddevs := make(map[string]float64)
	times := make(map[string][]float64)
	statusCode := make(map[string][]int)
	var allTimes []*Time

	done := make(chan struct{})
	go func() {
		for time := range ch {
			totals[time.Url] += time.Time
			times[time.Url] = append(times[time.Url], time.Time)
			allTimes = append(allTimes, time)
			if statusCode[time.Url] == nil {
				statusCode[time.Url] = make([]int, 6)
			}
			statusCode[time.Url][time.StatusCode]++
		}
		for url, total := range totals {
			mean := total / float64(len(times[url]))
			for _, t := range times[url] {
				stddevs[url] += math.Pow(t-mean, 2)
			}
		}
		done <- struct{}{}
	}()

	logParser := regexp.MustCompile(config.LogFormat)

	var wg sync.WaitGroup
	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF {
			break
		} else if err != nil {
			panic(err)
		}
		wg.Add(1)
		go func(line string) {
			defer wg.Done()
			submatch := logParser.FindAllStringSubmatch(strings.TrimSpace(line), -1)
			if len(submatch) > 0 {
				s := submatch[0]
				url := s[config.RequestIndex]
				for name, re := range urlNormalizeRegexps {
					if re.MatchString(url) {
						url = name
						break
					}
				}
				time, err := strconv.ParseFloat(s[config.DurationIndex], 10)
				if err == nil {
					time = time * scale
				} else {
					time = 0.000
				}
				statusCode, err := strconv.Atoi(string(s[config.StatusIndex][0]))
				if err != nil {
					statusCode = 0
				}
				ch <- &Time{Url: url, Time: time, StatusCode: statusCode}
			}
		}(line)
	}
	wg.Wait()
	close(ch)
	<-done

	var measures []*Measure
	for url, total := range totals {
		sorted := times[url]
		sort.Float64s(sorted)
		count := len(sorted)
		measure := &Measure{
			Url:    url,
			Count:  count,
			Total:  total,
			Mean:   totals[url] / float64(count),
			Stddev: math.Sqrt(stddevs[url] / float64(count)),
			Min:    sorted[0],
			P50:    sorted[int(count*50/100)],
			P90:    sorted[int(count*90/100)],
			P95:    sorted[int(count*95/100)],
			P99:    sorted[int(count*99/100)],
			Max:    sorted[count-1],
			S2xx:   statusCode[url][2],
			S3xx:   statusCode[url][3],
			S4xx:   statusCode[url][4],
			S5xx:   statusCode[url][5],
		}
		measures = append(measures, measure)
	}

	for _, column := range columns {
		if column.Sort != nil {
			fmt.Printf("Sort By %s\n", column.Summary)
			By(column.Sort).Sort(measures)
			showMeasures(measures)
			fmt.Println()
		}
	}

	showTop(allTimes)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
)

const (
	ChunkSeconds = 20
	BufferChunks = 2
)

var (
	logger log.Logger
)

// Configure and return the radio
func configure() *Radio {
	// Parse flags
	var (
		driver    string
		dbhost    string
		dbport    int
		record    bool
		broadcast bool
		addr      string
		loglevel  string
	)

	flag.StringVar(&driver, "driver", "redis", "Database driver: etcd|redis|ssdb")
	flag.StringVar(&dbhost, "dbhost", "localhost", "Database host")
	flag.IntVar(&dbport, "dbport", 6379, "Database port")
	flag.BoolVar(&record, "record", false, "Record presets")
	flag.BoolVar(&broadcast, "broadcast", false, "Broadcast to users")
	flag.StringVar(&addr, "addr", ":8080", "Broadcast address")
	flag.StringVar(&loglevel, "loglevel", "info", "Logging level: debug|info|warn|error")

	flag.Parse()

	// Initialize logging
	logger = log.NewLogfmtLogger(os.Stdout)
	switch loglevel {
	case "debug":
		logger = level.NewFilter(logger, level.AllowDebug())
	case "info":
		logger = level.NewFilter(logger, level.AllowInfo())
	case "warn":
		logger = level.NewFilter(logger, level.AllowWarn())
	case "error":
		logger = level.NewFilter(logger, level.AllowError())
	default:
		level.Error(logger).Log("msg", fmt.Sprintf("No %q loglevel found", loglevel))
		os.Exit(1)
	}
	logger = log.With(logger, "caller", log.DefaultCaller)
	logger = log.With(logger, "ts", log.DefaultTimestampUTC)

	// Initialize the backend
	var backend Backend
	switch driver {
	case "etcd":
		backend = &EtcdBackend{}
	case "redis":
		backend = &RedisBackend{}
	case "ssdb":
		backend = &RedisBackend{ssdb: true}
	default:
		level.Error(logger).Log("msg", fmt.Sprintf("No %q driver found", driver))
		os.Exit(1)
	}

	if err := backend.Init(dbhost, dbport); err != nil {
		level.Error(logger).Log(
			"msg", fmt.Sprintf("Cannot init backend with driver %s", driver),
			"err", err)
		os.Exit(1)
	}
	level.Info(logger).Log("msg", "Backend initialized",
		"driver", driver,
		"dbaddr", fmt.Sprintf("%s:%d", dbhost, dbport))

	// Construct the radio
	return &Radio{
		Server: &http.Server{Addr: addr},
		TapeDeck: &TapeDeck{
			backend: backend,
		},
		Presets: &Presets{
			backend: backend,
		},
		Options: RadioOptions{
			Broadcast: broadcast,
			Record:    record,
		},
		//RecordingEngineer: RecordingEngineer{
		//	ch: make(chan StatusMessage, 1),
		//	s:  make(map[string]Status),
		//},
	}
}

func main() {
	radio := configure()
	radio.On()

	sigs := make(chan os.Signal, 1)
	done := make(chan bool, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		sig := <-sigs
		level.Info(logger).Log("msg", fmt.Sprintf("Received signal %s", sig))
		radio.Off()
		done <- true
	}()

	<-done
	level.Info(logger).Log("msg", "done")
}
<commit_msg>Enable recording and broadcasting by default<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
)

const (
	ChunkSeconds = 20
	BufferChunks = 2
)

var (
	logger log.Logger
)

// Configure and return the radio
func configure() *Radio {
	// Parse flags
	var (
		driver    string
		dbhost    string
		dbport    int
		record    bool
		broadcast bool
		addr      string
		loglevel  string
	)

	flag.StringVar(&driver, "driver", "redis", "Database driver: etcd|redis|ssdb")
	flag.StringVar(&dbhost, "dbhost", "localhost", "Database host")
	flag.IntVar(&dbport, "dbport", 6379, "Database port")
	flag.BoolVar(&record, "record", true, "Record presets")
	flag.BoolVar(&broadcast, "broadcast", true, "Broadcast to users")
	flag.StringVar(&addr, "addr", ":8080", "Broadcast address")
	flag.StringVar(&loglevel, "loglevel", "info", "Logging level: debug|info|warn|error")

	flag.Parse()

	// Initialize logging
	logger = log.NewLogfmtLogger(os.Stdout)
	switch loglevel {
	case "debug":
		logger = level.NewFilter(logger, level.AllowDebug())
	case "info":
		logger = level.NewFilter(logger, level.AllowInfo())
	case "warn":
		logger = level.NewFilter(logger, level.AllowWarn())
	case "error":
		logger = level.NewFilter(logger, level.AllowError())
	default:
		level.Error(logger).Log("msg", fmt.Sprintf("No %q loglevel found", loglevel))
		os.Exit(1)
	}
	logger = log.With(logger, "caller", log.DefaultCaller)
	logger = log.With(logger, "ts", log.DefaultTimestampUTC)

	// Initialize the backend
	var backend Backend
	switch driver {
	case "etcd":
		backend = &EtcdBackend{}
	case "redis":
		backend = &RedisBackend{}
	case "ssdb":
		backend = &RedisBackend{ssdb: true}
	default:
		level.Error(logger).Log("msg", fmt.Sprintf("No %q driver found", driver))
		os.Exit(1)
	}

	if err := backend.Init(dbhost, dbport); err != nil {
		level.Error(logger).Log(
			"msg", fmt.Sprintf("Cannot init backend with driver %s", driver),
			"err", err)
		os.Exit(1)
	}
	level.Info(logger).Log("msg", "Backend initialized",
		"driver", driver,
		"dbaddr", fmt.Sprintf("%s:%d", dbhost, dbport))

	// Construct the radio
	return &Radio{
		Server: &http.Server{Addr: addr},
		TapeDeck: &TapeDeck{
			backend: backend,
		},
		Presets: &Presets{
			backend: backend,
		},
		Options: RadioOptions{
			Broadcast: broadcast,
			Record:    record,
		},
		//RecordingEngineer: RecordingEngineer{
		//	ch: make(chan StatusMessage, 1),
		//	s:  make(map[string]Status),
		//},
	}
}

func main() {
	radio := configure()
	radio.On()

	sigs := make(chan os.Signal, 1)
	done := make(chan bool, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		sig := <-sigs
		level.Info(logger).Log("msg", fmt.Sprintf("Received signal %s", sig))
		radio.Off()
		done <- true
	}()

	<-done
	level.Info(logger).Log("msg", "done")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/push"
	"log"
	"strconv"
)

const (
	version = "0.01"
	jobName = "tesla_collector"
)

var (
	configPath      = flag.String("configPath", "config.json", "Path to configuration file.")
	pushgatewayAddr = flag.String("pushgateway_address", "", "URL of pushgateway.")
	odometerMetric  = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "odometric_metric",
		Help: "Current odometer value per vehicle.",
	}, []string{"vehicle_id", "vehicle_name"})
)

func main() {
	flag.Parse()
	log.Printf("Tesla Go version %v\n", version)

	config := LoadConfig(*configPath)
	client := NewTeslaClient()

	if err := client.Authenticate(config); err != nil {
		log.Fatalf("Failed to authenticate: %s", err)
	} else {
		log.Println("Authentication complete.")
	}

	vehicles, err := client.ListVehicles()
	if err != nil {
		log.Fatalf("ListVehicles operation failed: %s", err)
	}

	for _, v := range vehicles {
		resp, err := client.GetVehicleState(v)
		if err != nil {
			log.Fatalf("Error while getting drive state of v %s: %s", v.ID, err)
		}
		odometerMetric.WithLabelValues(strconv.FormatInt(v.ID, 10), v.DisplayName).Set(resp.Odometer)
	}

	if err := push.Collectors(jobName, nil, *pushgatewayAddr, odometerMetric); err != nil {
		log.Fatalf("Failed to push metrics: %s", err)
	}
}
<commit_msg>Add logging support with file rotation.<commit_after>package main

import (
	"flag"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/push"
	"gopkg.in/natefinch/lumberjack.v2"
	"log"
	"path"
	"strconv"
)

const (
	version = "0.01"
	jobName = "tesla_collector"
)

var (
	configPath      = flag.String("config_path", "config.json", "Path to configuration file.")
	logDir          = flag.String("log_dir", "/var/log/tesla-go", "Directory of logging destination.")
	logFilename     = flag.String("log_filename", "tesla-go.INFO", "Name of logging file.")
	pushgatewayAddr = flag.String("pushgateway_address", "", "URL of pushgateway.")
)

var (
	odometerMetric = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "odometric_metric",
		Help: "Current odometer value per vehicle.",
	}, []string{"vehicle_id", "vehicle_name"})
)

func main() {
	flag.Parse()
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile)
	log.SetOutput(&lumberjack.Logger{
		Filename:   path.Join(*logDir, *logFilename),
		MaxSize:    10, // MiB
		MaxBackups: 5,
		LocalTime:  true,
	})

	log.Printf("Tesla Go version %v\n", version)

	config := LoadConfig(*configPath)
	client := NewTeslaClient()

	if err := client.Authenticate(config); err != nil {
		log.Fatalf("Failed to authenticate: %s", err)
	} else {
		log.Println("Authentication complete.")
	}

	vehicles, err := client.ListVehicles()
	if err != nil {
		log.Fatalf("ListVehicles operation failed: %s", err)
	}

	for _, v := range vehicles {
		resp, err := client.GetVehicleState(v)
		if err != nil {
			log.Fatalf("Error while getting drive state of v %s: %s", v.ID, err)
		}
		odometerMetric.WithLabelValues(strconv.FormatInt(v.ID, 10), v.DisplayName).Set(resp.Odometer)
	}

	if err := push.Collectors(jobName, nil, *pushgatewayAddr, odometerMetric); err != nil {
		log.Fatalf("Failed to push metrics: %s", err)
	} else {
		log.Println("Sucessfully pushed metrics.")
	}
}
<|endoftext|>


<commit_before>package main

import(
  "github.com/TykTechnologies/tykcommon"
  "github.com/TykTechnologies/goverify"

  "encoding/base64"
  "encoding/json"
  "fmt"
  "flag"
  "errors"
  "io/ioutil"
  "bytes"
  "archive/zip"
  "crypto/md5"
  "strings"
  "os"
  "io"
)

// tyk-cli <module> <submodule> <command> [--options] args...

var module, submodule, command string

var bundleOutput, privKey string

const(
  defaultBundleOutput = "bundle.zip"
)

func init() {
  if len(os.Args) == 1 {
    fmt.Println("No module specified!")
    os.Exit(1)
  }
  if len(os.Args) == 2 {
    fmt.Println("No command specified!")
    os.Exit(1)
  }

  module = os.Args[1]
  command = os.Args[2]

  os.Args = os.Args[2:]

  flag.StringVar(&bundleOutput, "output", "", "Bundle output")
  flag.StringVar(&privKey, "key", "", "Key for bundle signature")

  flag.Parse()
}

// main is the entrypoint.
func main() {
  fmt.Println("tyk-cli:", flag.CommandLine, os.Args)

  fmt.Println("module =", module)
  fmt.Println("command =", command)

  var err error

  switch module {
  case "bundle":
    fmt.Println("Using bundle module.")
    err = bundle(command)
  default:
    err = errors.New("Invalid module")
  }

  if err != nil {
    fmt.Println("Error:", err)
    os.Exit(1)
  }
}

// bundle will handle the bundle command calls.
func bundle(command string) (err error) {
  switch command {
  case "build":
    var manifestPath = "./manifest.json"
    if _, err := os.Stat(manifestPath); err == nil {
      var manifestData []byte
      manifestData, err = ioutil.ReadFile(manifestPath)

      var manifest tykcommon.BundleManifest
      err = json.Unmarshal(manifestData, &manifest)

      if err != nil {
        fmt.Println("Couldn't parse manifest file!")
        break
      }

      err = bundleValidateManifest(&manifest)

      if err != nil {
        fmt.Println("Bundle validation error:")
        fmt.Println(err)
        break
      }

      // The manifest is valid, we should do the checksum and sign step at this point.
      bundleBuild(&manifest)

    } else {
      err = errors.New("Manifest file doesn't exist.")
    }
  default:
    err = errors.New("Invalid command.")
  }
  return err
}

// bundleValidateManifest will validate the manifest file before building a bundle.
func bundleValidateManifest(manifest *tykcommon.BundleManifest) (err error) {
  // Validate manifest file list:
  for _, file := range manifest.FileList {
    if _, statErr := os.Stat(file); statErr != nil {
      err = errors.New("Referencing a nonexistent file: " + file)
      break
    }
  }

  // The custom middleware block must specify at least one hook:
  var definedHooks int
  definedHooks = len(manifest.CustomMiddleware.Pre) + len(manifest.CustomMiddleware.Post) + len(manifest.CustomMiddleware.PostKeyAuth)

  if manifest.CustomMiddleware.AuthCheck.Name != "" {
    definedHooks++
  }

  if definedHooks == 0 {
    err = errors.New("No hooks defined!")
    return err
  }

  // The custom middleware block must specify a driver:
  if manifest.CustomMiddleware.Driver == "" {
    err = errors.New("No driver specified!")
    return err
  }

  return err
}

// bundleBuild will build and generate a bundle file.
func bundleBuild(manifest *tykcommon.BundleManifest) (err error) {
  var useSignature bool

  if bundleOutput == "" {
    fmt.Println("No output specified, using bundle.zip")
    bundleOutput = defaultBundleOutput
  }

  if privKey == "" {
    // Warning?
    fmt.Println("The bundle won't be signed.")
  } else {
    fmt.Println("The bundle will be signed.")
    useSignature = true
  }

  var signer goverify.Signer

  if useSignature {
    signer, err = goverify.LoadPrivateKeyFromFile(privKey)
    if err != nil {
      return err
    }
  }

  // Checksum and signature:

  var bundleChecksums []string
  var bundleSignatures []string

  for _, file := range manifest.FileList {
    var data []byte
    data, err = ioutil.ReadFile(file)
    if err != nil {
      fmt.Println("*** Error: ", err)
      return err
    }
    hash := fmt.Sprintf("%x", md5.Sum(data))
    bundleChecksums = append(bundleChecksums, hash)

    if useSignature {
      var signed []byte
      signed, err = signer.Sign(data)

      sig := base64.StdEncoding.EncodeToString(signed)
      bundleSignatures = append(bundleSignatures, sig)
      fmt.Printf("Signature: %v %s\n", sig, file)
    }
  }

  mergedChecksums := strings.Join(bundleChecksums, "")
  mergedSignatures := strings.Join(bundleSignatures, "")

  // Update the manifest file:

  manifest.Checksum = fmt.Sprintf("%x", md5.Sum([]byte(mergedChecksums)))
  manifest.Signature = mergedSignatures

  var newManifestData []byte
  newManifestData, err = json.Marshal(&manifest)

  // Write the bundle file:
  buf := new(bytes.Buffer)
  bundleWriter := zip.NewWriter(buf)

  for _, file := range manifest.FileList {
    var outputFile io.Writer
    outputFile, err = bundleWriter.Create(file)
    if err != nil {
      return err
    }
    var data []byte
    data, err = ioutil.ReadFile(file)

    _, err = outputFile.Write(data)

    if err != nil {
      return err
    }
  }

  // Write manifest file:
  var newManifest io.Writer
  newManifest, err = bundleWriter.Create("manifest.json")
  _, err = newManifest.Write(newManifestData)

  err = bundleWriter.Close()
  err = ioutil.WriteFile(bundleOutput, buf.Bytes(), 0755)

  return err
}
<commit_msg>Add a comment about the auth check middleware.<commit_after>package main

import(
  "github.com/TykTechnologies/tykcommon"
  "github.com/TykTechnologies/goverify"

  "encoding/base64"
  "encoding/json"
  "fmt"
  "flag"
  "errors"
  "io/ioutil"
  "bytes"
  "archive/zip"
  "crypto/md5"
  "strings"
  "os"
  "io"
)

// tyk-cli <module> <submodule> <command> [--options] args...

var module, submodule, command string

var bundleOutput, privKey string

const(
  defaultBundleOutput = "bundle.zip"
)

func init() {
  if len(os.Args) == 1 {
    fmt.Println("No module specified!")
    os.Exit(1)
  }
  if len(os.Args) == 2 {
    fmt.Println("No command specified!")
    os.Exit(1)
  }

  module = os.Args[1]
  command = os.Args[2]

  os.Args = os.Args[2:]

  flag.StringVar(&bundleOutput, "output", "", "Bundle output")
  flag.StringVar(&privKey, "key", "", "Key for bundle signature")

  flag.Parse()
}

// main is the entrypoint.
func main() {
  fmt.Println("tyk-cli:", flag.CommandLine, os.Args)

  fmt.Println("module =", module)
  fmt.Println("command =", command)

  var err error

  switch module {
  case "bundle":
    fmt.Println("Using bundle module.")
    err = bundle(command)
  default:
    err = errors.New("Invalid module")
  }

  if err != nil {
    fmt.Println("Error:", err)
    os.Exit(1)
  }
}

// bundle will handle the bundle command calls.
func bundle(command string) (err error) {
  switch command {
  case "build":
    var manifestPath = "./manifest.json"
    if _, err := os.Stat(manifestPath); err == nil {
      var manifestData []byte
      manifestData, err = ioutil.ReadFile(manifestPath)

      var manifest tykcommon.BundleManifest
      err = json.Unmarshal(manifestData, &manifest)

      if err != nil {
        fmt.Println("Couldn't parse manifest file!")
        break
      }

      err = bundleValidateManifest(&manifest)

      if err != nil {
        fmt.Println("Bundle validation error:")
        fmt.Println(err)
        break
      }

      // The manifest is valid, we should do the checksum and sign step at this point.
      bundleBuild(&manifest)

    } else {
      err = errors.New("Manifest file doesn't exist.")
    }
  default:
    err = errors.New("Invalid command.")
  }
  return err
}

// bundleValidateManifest will validate the manifest file before building a bundle.
func bundleValidateManifest(manifest *tykcommon.BundleManifest) (err error) {
  // Validate manifest file list:
  for _, file := range manifest.FileList {
    if _, statErr := os.Stat(file); statErr != nil {
      err = errors.New("Referencing a nonexistent file: " + file)
      break
    }
  }

  // The custom middleware block must specify at least one hook:
  var definedHooks int
  definedHooks = len(manifest.CustomMiddleware.Pre) + len(manifest.CustomMiddleware.Post) + len(manifest.CustomMiddleware.PostKeyAuth)

  // We should count the auth check middleware (single), if it's present:
  if manifest.CustomMiddleware.AuthCheck.Name != "" {
    definedHooks++
  }

  if definedHooks == 0 {
    err = errors.New("No hooks defined!")
    return err
  }

  // The custom middleware block must specify a driver:
  if manifest.CustomMiddleware.Driver == "" {
    err = errors.New("No driver specified!")
    return err
  }

  return err
}

// bundleBuild will build and generate a bundle file.
func bundleBuild(manifest *tykcommon.BundleManifest) (err error) {
  var useSignature bool

  if bundleOutput == "" {
    fmt.Println("No output specified, using bundle.zip")
    bundleOutput = defaultBundleOutput
  }

  if privKey == "" {
    // Warning?
    fmt.Println("The bundle won't be signed.")
  } else {
    fmt.Println("The bundle will be signed.")
    useSignature = true
  }

  var signer goverify.Signer

  if useSignature {
    signer, err = goverify.LoadPrivateKeyFromFile(privKey)
    if err != nil {
      return err
    }
  }

  // Checksum and signature:

  var bundleChecksums []string
  var bundleSignatures []string

  for _, file := range manifest.FileList {
    var data []byte
    data, err = ioutil.ReadFile(file)
    if err != nil {
      fmt.Println("*** Error: ", err)
      return err
    }
    hash := fmt.Sprintf("%x", md5.Sum(data))
    bundleChecksums = append(bundleChecksums, hash)

    if useSignature {
      var signed []byte
      signed, err = signer.Sign(data)

      sig := base64.StdEncoding.EncodeToString(signed)
      bundleSignatures = append(bundleSignatures, sig)
      fmt.Printf("Signature: %v %s\n", sig, file)
    }
  }

  mergedChecksums := strings.Join(bundleChecksums, "")
  mergedSignatures := strings.Join(bundleSignatures, "")

  // Update the manifest file:

  manifest.Checksum = fmt.Sprintf("%x", md5.Sum([]byte(mergedChecksums)))
  manifest.Signature = mergedSignatures

  var newManifestData []byte
  newManifestData, err = json.Marshal(&manifest)

  // Write the bundle file:
  buf := new(bytes.Buffer)
  bundleWriter := zip.NewWriter(buf)

  for _, file := range manifest.FileList {
    var outputFile io.Writer
    outputFile, err = bundleWriter.Create(file)
    if err != nil {
      return err
    }
    var data []byte
    data, err = ioutil.ReadFile(file)

    _, err = outputFile.Write(data)

    if err != nil {
      return err
    }
  }

  // Write manifest file:
  var newManifest io.Writer
  newManifest, err = bundleWriter.Create("manifest.json")
  _, err = newManifest.Write(newManifestData)

  err = bundleWriter.Close()
  err = ioutil.WriteFile(bundleOutput, buf.Bytes(), 0755)

  return err
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/die-net/http-tarpit/tarpit"
	"log"
	"net"
	"net/http"
	"runtime"
	"time"
)

var listenAddr = flag.String("listen", ":8080", "The [IP]:port to listen for incoming connections on.")
var workers = flag.Int("workers", runtime.NumCPU(), "The number of worker threads to execute.")
var period = flag.Duration("period", 16*time.Second, "Time between each byte sent on a connection.")
var timeslice = flag.Duration("timeslice", 50*time.Millisecond, "How often each thread should wake up to send.")
var contentType = flag.String("content_type", "text/html", "The content-type to send with the response.")
var minResponseLen = flag.Int64("min_response_len", 1048576, "The minimum number of bytes to send total per connection.")
var maxResponseLen = flag.Int64("max_response_len", 10485760, "The maximum number of bytes to send total per connection.")
var rcvBuf = flag.Int("rcvbuf", 2048, "Kernel receive buffer size (0=default).")
var sndBuf = flag.Int("sndbuf", 2048, "Kernel send buffer size (0=default).")

func main() {
	flag.Parse()

	setRlimitFromFlags()

	runtime.GOMAXPROCS(*workers)

	tarpit := tarpit.New(*workers, *contentType, *period, *timeslice, *minResponseLen, *maxResponseLen)

	http.HandleFunc("/", tarpit.Handler)
	http.HandleFunc("/robots.txt", robotsDisallowHandler)

	log.Fatal(listenAndServe(*listenAddr))
}

func listenAndServe(addr string) error {
	srv := &http.Server{Addr: addr}
	ln, err := net.Listen("tcp", addr)
	if err != nil {
		return err
	}
	return srv.Serve(NewBufSizeListener(*rcvBuf, *sndBuf, ln.(*net.TCPListener)))
}
<commit_msg>Add check for invalid argument.<commit_after>package main

import (
	"flag"
	"github.com/die-net/http-tarpit/tarpit"
	"log"
	"net"
	"net/http"
	"runtime"
	"time"
)

var listenAddr = flag.String("listen", ":8080", "The [IP]:port to listen for incoming connections on.")
var workers = flag.Int("workers", runtime.NumCPU(), "The number of worker threads to execute.")
var period = flag.Duration("period", 16*time.Second, "Time between each byte sent on a connection.")
var timeslice = flag.Duration("timeslice", 50*time.Millisecond, "How often each thread should wake up to send.")
var contentType = flag.String("content_type", "text/html", "The content-type to send with the response.")
var minResponseLen = flag.Int64("min_response_len", 1048576, "The minimum number of bytes to send total per connection.")
var maxResponseLen = flag.Int64("max_response_len", 10485760, "The maximum number of bytes to send total per connection.")
var rcvBuf = flag.Int("rcvbuf", 2048, "Kernel receive buffer size (0=default).")
var sndBuf = flag.Int("sndbuf", 2048, "Kernel send buffer size (0=default).")

func main() {
	flag.Parse()

	setRlimitFromFlags()

	runtime.GOMAXPROCS(*workers)

	tarpit := tarpit.New(*workers, *contentType, *period, *timeslice, *minResponseLen, *maxResponseLen)
	if tarpit == nil {
		log.Fatal("Invalid argument")
	}

	http.HandleFunc("/", tarpit.Handler)
	http.HandleFunc("/robots.txt", robotsDisallowHandler)

	log.Fatal(listenAndServe(*listenAddr))
}

func listenAndServe(addr string) error {
	srv := &http.Server{Addr: addr}
	ln, err := net.Listen("tcp", addr)
	if err != nil {
		return err
	}
	return srv.Serve(NewBufSizeListener(*rcvBuf, *sndBuf, ln.(*net.TCPListener)))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/midnightfreddie/McpeTool/api"
	"github.com/midnightfreddie/McpeTool/world"
	"github.com/urfave/cli"
)

func main() {
	var path string
	app := cli.NewApp()
	app.Name = "MCPE Tool"
	app.Version = "0.1.3"
	app.Compiled = time.Now()
	app.Authors = []cli.Author{
		cli.Author{
			Name:  "Jim Nelson",
			Email: "jim@jimnelson.us",
		},
	}
	app.Copyright = "(c) 2018 Jim Nelson"
	app.Usage = "Reads and writes a Minecraft Pocket Edition world directory."
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "path, p",
			Value:       ".",
			Usage:       "`FILEPATH` of world",
			EnvVar:      "MCPETOOL_WORLD",
			Destination: &path,
		},
	}

	app.Commands = []cli.Command{
		{
			Name:    "api",
			Aliases: []string{"www"},
			Usage:   "Open world, start API at http://127.0.0.1:8080 . Control-c to exit.",
			Action: func(c *cli.Context) error {
				world, err := world.OpenWorld(path)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				defer world.Close()
				err = api.Server(&world)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				return nil
			},
		},
		{
			Name:    "keys",
			Aliases: []string{"k"},
			Usage:   "Lists all keys in the database.",
			Action: func(c *cli.Context) error {
				world, err := world.OpenWorld(path)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				defer world.Close()
				keys, err := world.GetKeys()
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				for i := 0; i < len(keys); i++ {
					fmt.Println(hex.EncodeToString(keys[i]))
				}
				return nil
			},
		},
		{
			Name:      "get",
			ArgsUsage: "<key>",
			Usage:     "Retruns a key's value in base64 format.",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "dump, d",
					Usage: "Display value as hexdump",
				},
			},
			Action: func(c *cli.Context) error {
				world, err := world.OpenWorld(path)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				defer world.Close()
				key, err := hex.DecodeString(c.Args().Get(0))
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				value, err := world.Get(key)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				if c.String("dump") == "true" {
					fmt.Println(hex.Dump(value))
				} else {
					fmt.Println(base64.StdEncoding.EncodeToString(value))
				}
				return nil
			},
		},
		{
			Name:      "put",
			ArgsUsage: "<key>",
			Usage:     "Put a key/value into the DB. The base64-encoded value read from stdin.",
			Action: func(c *cli.Context) error {
				world, err := world.OpenWorld(path)
				key, err := hex.DecodeString(c.Args().Get(0))
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				defer world.Close()
				base64Data, err := ioutil.ReadAll(os.Stdin)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				value, err := base64.StdEncoding.DecodeString(string(base64Data[:]))
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				err = world.Put(key, value)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				return nil
			},
		},
		{
			Name:      "delete",
			ArgsUsage: "<key>",
			Usage:     "Deletes a key and its value.",
			Action: func(c *cli.Context) error {
				world, err := world.OpenWorld(path)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				defer world.Close()
				key, err := hex.DecodeString(c.Args().Get(0))
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				err = world.Delete(key)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				return nil
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>Added json flag for cli get<commit_after>package main

import (
	"encoding/base64"
	"encoding/hex"
	"encoding/binary"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/midnightfreddie/McpeTool/api"
	"github.com/midnightfreddie/McpeTool/world"
	"github.com/midnightfreddie/nbt2json"
	"github.com/urfave/cli"
)

func main() {
	var path string
	app := cli.NewApp()
	app.Name = "MCPE Tool"
	app.Version = "0.1.3"
	app.Compiled = time.Now()
	app.Authors = []cli.Author{
		cli.Author{
			Name:  "Jim Nelson",
			Email: "jim@jimnelson.us",
		},
	}
	app.Copyright = "(c) 2018 Jim Nelson"
	app.Usage = "Reads and writes a Minecraft Pocket Edition world directory."
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "path, p",
			Value:       ".",
			Usage:       "`FILEPATH` of world",
			EnvVar:      "MCPETOOL_WORLD",
			Destination: &path,
		},
	}

	app.Commands = []cli.Command{
		{
			Name:    "api",
			Aliases: []string{"www"},
			Usage:   "Open world, start API at http://127.0.0.1:8080 . Control-c to exit.",
			Action: func(c *cli.Context) error {
				world, err := world.OpenWorld(path)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				defer world.Close()
				err = api.Server(&world)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				return nil
			},
		},
		{
			Name:    "keys",
			Aliases: []string{"k"},
			Usage:   "Lists all keys in the database.",
			Action: func(c *cli.Context) error {
				world, err := world.OpenWorld(path)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				defer world.Close()
				keys, err := world.GetKeys()
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				for i := 0; i < len(keys); i++ {
					fmt.Println(hex.EncodeToString(keys[i]))
				}
				return nil
			},
		},
		{
			Name:      "get",
			ArgsUsage: "<key>",
			Usage:     "Retruns a key's value in base64 format.",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "dump, d",
					Usage: "Display value as hexdump",
				},
				cli.BoolFlag{
					Name:  "json, j",
					Usage: "Display value as JSON. Only valid if value is NBT.",
				},
			},
			Action: func(c *cli.Context) error {
				world, err := world.OpenWorld(path)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				defer world.Close()
				key, err := hex.DecodeString(c.Args().Get(0))
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				value, err := world.Get(key)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				if c.String("dump") == "true" {
					fmt.Println(hex.Dump(value))
				} else if c.String("json") == "true" {
					out, err := nbt2json.Nbt2Json(value, binary.LittleEndian)
					if err != nil {
						return cli.NewExitError(err, 1);
					}
					fmt.Println(string(out[:]))
				} else {
					fmt.Println(base64.StdEncoding.EncodeToString(value))
				}
				return nil
			},
		},
		{
			Name:      "put",
			ArgsUsage: "<key>",
			Usage:     "Put a key/value into the DB. The base64-encoded value read from stdin.",
			Action: func(c *cli.Context) error {
				world, err := world.OpenWorld(path)
				key, err := hex.DecodeString(c.Args().Get(0))
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				defer world.Close()
				base64Data, err := ioutil.ReadAll(os.Stdin)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				value, err := base64.StdEncoding.DecodeString(string(base64Data[:]))
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				err = world.Put(key, value)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				return nil
			},
		},
		{
			Name:      "delete",
			ArgsUsage: "<key>",
			Usage:     "Deletes a key and its value.",
			Action: func(c *cli.Context) error {
				world, err := world.OpenWorld(path)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				defer world.Close()
				key, err := hex.DecodeString(c.Args().Get(0))
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				err = world.Delete(key)
				if err != nil {
					return cli.NewExitError(err, 1);
				}
				return nil
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package gosingleton

import (
	"fmt"
	"io/ioutil"
	"os"
	"strconv"

	"gopkg.in/fatih/set.v0"
)

// Check if we're the only process running with this path.
func UniquePath(path string) bool {
	return false // not implemented.
}

// Check if we're the only process running with this name.
func UniqueName(path string) bool {
	return false // not implemented
}

// Check if an instance of the same binary is running.
func UniqueBinary(pid int64) (bool, error) {
	// Resolve the exe symlink for this PID.
	path, err := resolveExeSymlink(pid)
	if err != nil {
		return false, err
	}

	// Build an inverted index of all /proc/PID/exe symlinks.
	index, err := buildInvertedIndex()
	if err != nil {
		return false, err
	}

	// Query the index to see if there are multiple instances.
	bin, ok := index[path]
	if !ok {
		return false, fmt.Errorf("Unknown binary path %s", path)
	}

	return bin.Has(pid), nil
}

// Resole the exe symlink for a PID.
func resolveExeSymlink(pid int64) (string, error) {
	path, err := os.Readlink(fmt.Sprintf("/proc/%d/exe", pid))
	if err != nil {
		return "", err
	}

	return path, nil
}

// Build an inverted index of paths as keys with sets of PIDs as values.
func buildInvertedIndex() (map[string]*set.Set, error) {
	index := make(map[string]*set.Set)

	// Find all entries under /proc.
	entries, err := ioutil.ReadDir("/proc")
	if err != nil {
		return nil, err
	}

	// Filter for PIDs and resolve their exe symlinks.
	for _, e := range entries {
		pid, err := strconv.ParseInt(e.Name(), 10, 64)
		if err != nil {
			continue
		}

		path, err := resolveExeSymlink(pid)
		if err != nil {
			return nil, err
		}

		if bin, ok := index[path]; ok {
			bin.Add(pid)
		} else {
			index[path] = set.New(pid)
		}
	}

	return index, nil
}
<commit_msg>* Simplified the UniqueBinary() API to only ever return nil or and error. * Switched to representing PIDs as int instead of int64.<commit_after>package gosingleton

import (
	"fmt"
	"io/ioutil"
	"os"
	"strconv"

	"gopkg.in/fatih/set.v0"
)

// Check if we're the only process running with this path.
func UniquePath(path string) bool {
	return false // not implemented.
}

// Check if we're the only process running with this name.
func UniqueName(path string) bool {
	return false // not implemented
}

// Check if an instance of the same binary is running.
func UniqueBinary(pid int) error {
	// Resolve the exe symlink for this PID.
	path, err := resolveExeSymlink(pid)
	if err != nil {
		return err
	}

	// Build an inverted index of all /proc/PID/exe symlinks.
	index, err := buildInvertedIndex()
	if err != nil {
		return err
	}

	// Query the index to see if there are multiple instances.
	bin, ok := index[path]
	if !ok {
		return fmt.Errorf("Unknown binary path %s", path)
	}

	if bin.Size() > 1 {
		return fmt.Errorf("Existing binary %s detected!", path)
	}

	return nil
}

// Resole the exe symlink for a PID.
func resolveExeSymlink(pid int) (string, error) {
	path, err := os.Readlink(fmt.Sprintf("/proc/%d/exe", pid))
	if err != nil {
		return "", err
	}

	return path, nil
}

// Build an inverted index of paths as keys with sets of PIDs as values.
func buildInvertedIndex() (map[string]*set.Set, error) {
	index := make(map[string]*set.Set)

	// Find all entries under /proc.
	entries, err := ioutil.ReadDir("/proc")
	if err != nil {
		return nil, err
	}

	// Filter for PIDs and resolve their exe symlinks.
	for _, e := range entries {
		pid, err := strconv.Atoi(e.Name())
		if err != nil {
			continue
		}

		path, err := resolveExeSymlink(pid)
		if err != nil {
			return nil, err
		}

		if bin, ok := index[path]; ok {
			bin.Add(pid)
		} else {
			index[path] = set.New(pid)
		}
	}

	return index, nil
}
<|endoftext|>


<commit_before>// Package main is the entry point for the ghbackup binary.
// Here is where you can find argument parsing, usage information and the actual execution.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"runtime"

	"qvl.io/ghbackup/ghbackup"
)

// Can be set in build step using -ldflags
var version string

const (
	// Printed for -help, -h or with wrong number of arguments
	usage = `Embarrassing simple GitHub backup tool

Usage: %s [flags] directory

  directory  path to save the repositories to

At least one of -account or -secret must be specified.

Flags:
`
	more         = "\nFor more visit https://qvl.io/ghbackup."
	accountUsage = `GitHub user or organization name to get repositories from.
	If not specified, all repositories the authenticated user has access to will be loaded.`
	secretUsage = `Authentication secret for GitHub API.
	Can use the users password or a personal access token (https://github.com/settings/tokens).
	Authentication increases rate limiting (https://developer.github.com/v3/#rate-limiting) and enables backup of private repositories.`
)

// Get command line arguments and start updating repositories
func main() {
	// Flags
	account := flag.String("account", "", accountUsage)
	secret := flag.String("secret", "", secretUsage)
	versionFlag := flag.Bool("version", false, "Print binary version")
	silent := flag.Bool("silent", false, "Suppress all output")

	// Parse args
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usage, os.Args[0])
		flag.PrintDefaults()
		fmt.Fprintln(os.Stderr, more)
	}
	flag.Parse()

	if *versionFlag {
		fmt.Printf("ghbackup %s %s %s\n", version, runtime.GOOS, runtime.GOARCH)
		os.Exit(0)
	}

	args := flag.Args()
	if len(args) != 1 || (*account == "" && *secret == "") {
		flag.Usage()
		os.Exit(1)
	}

	errs := log.New(os.Stderr, "", 0)
	logs := errs
	if *silent {
		logs = log.New(ioutil.Discard, "", 0)
	}

	err := ghbackup.Run(ghbackup.Config{
		Account: *account,
		Dir:     args[0],
		Secret:  *secret,
		Log:     logs,
		Err:     errs,
	})

	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<commit_msg>Output logs to stdout instead of stderr<commit_after>// Package main is the entry point for the ghbackup binary.
// Here is where you can find argument parsing, usage information and the actual execution.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"runtime"

	"qvl.io/ghbackup/ghbackup"
)

// Can be set in build step using -ldflags
var version string

const (
	// Printed for -help, -h or with wrong number of arguments
	usage = `Embarrassing simple GitHub backup tool

Usage: %s [flags] directory

  directory  path to save the repositories to

At least one of -account or -secret must be specified.

Flags:
`
	more         = "\nFor more visit https://qvl.io/ghbackup."
	accountUsage = `GitHub user or organization name to get repositories from.
	If not specified, all repositories the authenticated user has access to will be loaded.`
	secretUsage = `Authentication secret for GitHub API.
	Can use the users password or a personal access token (https://github.com/settings/tokens).
	Authentication increases rate limiting (https://developer.github.com/v3/#rate-limiting) and enables backup of private repositories.`
)

// Get command line arguments and start updating repositories
func main() {
	// Flags
	account := flag.String("account", "", accountUsage)
	secret := flag.String("secret", "", secretUsage)
	versionFlag := flag.Bool("version", false, "Print binary version")
	silent := flag.Bool("silent", false, "Suppress all output")

	// Parse args
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usage, os.Args[0])
		flag.PrintDefaults()
		fmt.Fprintln(os.Stderr, more)
	}
	flag.Parse()

	if *versionFlag {
		fmt.Printf("ghbackup %s %s %s\n", version, runtime.GOOS, runtime.GOARCH)
		os.Exit(0)
	}

	args := flag.Args()
	if len(args) != 1 || (*account == "" && *secret == "") {
		flag.Usage()
		os.Exit(1)
	}

	errs := log.New(os.Stderr, "", 0)
	logs := log.New(os.Stdout, "", 0)
	if *silent {
		logs = log.New(ioutil.Discard, "", 0)
	}

	err := ghbackup.Run(ghbackup.Config{
		Account: *account,
		Dir:     args[0],
		Secret:  *secret,
		Log:     logs,
		Err:     errs,
	})

	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package mpb

import (
	"unicode/utf8"

	"github.com/vbauerster/mpb/v4/decor"
)

// BarOption is a function option which changes the default behavior of a bar.
type BarOption func(*bState)

// AppendDecorators let you inject decorators to the bar's right side.
func AppendDecorators(appenders ...decor.Decorator) BarOption {
	return func(s *bState) {
		for _, decorator := range appenders {
			if ar, ok := decorator.(decor.AmountReceiver); ok {
				s.amountReceivers = append(s.amountReceivers, ar)
			}
			if sl, ok := decorator.(decor.ShutdownListener); ok {
				s.shutdownListeners = append(s.shutdownListeners, sl)
			}
			s.aDecorators = append(s.aDecorators, decorator)
		}
	}
}

// PrependDecorators let you inject decorators to the bar's left side.
func PrependDecorators(prependers ...decor.Decorator) BarOption {
	return func(s *bState) {
		for _, decorator := range prependers {
			if ar, ok := decorator.(decor.AmountReceiver); ok {
				s.amountReceivers = append(s.amountReceivers, ar)
			}
			if sl, ok := decorator.(decor.ShutdownListener); ok {
				s.shutdownListeners = append(s.shutdownListeners, sl)
			}
			s.pDecorators = append(s.pDecorators, decorator)
		}
	}
}

// BarID sets bar id.
func BarID(id int) BarOption {
	return func(s *bState) {
		s.id = id
	}
}

// BarWidth sets bar width independent of the container.
func BarWidth(width int) BarOption {
	return func(s *bState) {
		s.width = width
	}
}

// BarRemoveOnComplete is a flag, if set whole bar line will be removed
// on complete event. If both BarRemoveOnComplete and BarClearOnComplete
// are set, first bar section gets cleared and then whole bar line
// gets removed completely.
func BarRemoveOnComplete() BarOption {
	return func(s *bState) {
		s.removeOnComplete = true
	}
}

// BarReplaceOnComplete is indicator for delayed bar start, after the
// `runningBar` is complete. To achieve bar replacement effect,
// `runningBar` should has its `BarRemoveOnComplete` option set.
func BarReplaceOnComplete(runningBar *Bar) BarOption {
	return func(s *bState) {
		s.runningBar = runningBar
	}
}

// BarClearOnComplete is a flag, if set will clear bar section on
// complete event. If you need to remove a whole bar line, refer to
// BarRemoveOnComplete.
func BarClearOnComplete() BarOption {
	return func(s *bState) {
		s.barClearOnComplete = true
	}
}

// BarPriority sets bar's priority. Zero is highest priority, i.e. bar
// will be on top. If `BarReplaceOnComplete` option is supplied, this
// option is ignored.
func BarPriority(priority int) BarOption {
	return func(s *bState) {
		s.priority = priority
	}
}

// BarNewLineExtend takes user defined efn, which gets called each
// render cycle. Any write to provided writer of efn, will appear on
// new line of respective bar.
func BarExtender(extender Filler) BarOption {
	return func(s *bState) {
		s.extender = extender
	}
}

// TrimSpace trims bar's edge spaces.
func TrimSpace() BarOption {
	return func(s *bState) {
		s.trimSpace = true
	}
}

// BarStyle sets custom bar style.
// Effective when Filler type is bar.
func BarStyle(style string) BarOption {
	chk := func(filler Filler) (interface{}, bool) {
		if style == "" {
			return nil, false
		}
		t, ok := filler.(*barFiller)
		return t, ok
	}
	cb := func(t interface{}) {
		if !utf8.ValidString(style) {
			return
		}
		copy(t.(*barFiller).format, []rune(style))
	}
	return MakeFillerTypeSpecificBarOption(chk, cb)
}

// SpinnerStyle sets custom spinner style.
// Effective when Filler type is spinner.
func SpinnerStyle(frames []string) BarOption {
	chk := func(filler Filler) (interface{}, bool) {
		if len(frames) == 0 {
			return nil, false
		}
		t, ok := filler.(*spinnerFiller)
		return t, ok
	}
	cb := func(t interface{}) {
		t.(*spinnerFiller).frames = frames
	}
	return MakeFillerTypeSpecificBarOption(chk, cb)
}

// MakeFillerTypeSpecificBarOption makes BarOption specific to Filler's
// actual type. If you implement your own Filler, so most probably
// you'll need this. See BarStyle or SpinnerStyle for example.
func MakeFillerTypeSpecificBarOption(
	typeChecker func(Filler) (interface{}, bool),
	cb func(interface{}),
) BarOption {
	return func(s *bState) {
		if t, ok := typeChecker(s.filler); ok {
			cb(t)
		}
	}
}

// OptionOnCondition returns option when condition evaluates to true.
func OptionOnCondition(option BarOption, condition func() bool) BarOption {
	if condition() {
		return option
	}
	return nil
}
<commit_msg>BarExtender godoc<commit_after>package mpb

import (
	"unicode/utf8"

	"github.com/vbauerster/mpb/v4/decor"
)

// BarOption is a function option which changes the default behavior of a bar.
type BarOption func(*bState)

// AppendDecorators let you inject decorators to the bar's right side.
func AppendDecorators(appenders ...decor.Decorator) BarOption {
	return func(s *bState) {
		for _, decorator := range appenders {
			if ar, ok := decorator.(decor.AmountReceiver); ok {
				s.amountReceivers = append(s.amountReceivers, ar)
			}
			if sl, ok := decorator.(decor.ShutdownListener); ok {
				s.shutdownListeners = append(s.shutdownListeners, sl)
			}
			s.aDecorators = append(s.aDecorators, decorator)
		}
	}
}

// PrependDecorators let you inject decorators to the bar's left side.
func PrependDecorators(prependers ...decor.Decorator) BarOption {
	return func(s *bState) {
		for _, decorator := range prependers {
			if ar, ok := decorator.(decor.AmountReceiver); ok {
				s.amountReceivers = append(s.amountReceivers, ar)
			}
			if sl, ok := decorator.(decor.ShutdownListener); ok {
				s.shutdownListeners = append(s.shutdownListeners, sl)
			}
			s.pDecorators = append(s.pDecorators, decorator)
		}
	}
}

// BarID sets bar id.
func BarID(id int) BarOption {
	return func(s *bState) {
		s.id = id
	}
}

// BarWidth sets bar width independent of the container.
func BarWidth(width int) BarOption {
	return func(s *bState) {
		s.width = width
	}
}

// BarRemoveOnComplete is a flag, if set whole bar line will be removed
// on complete event. If both BarRemoveOnComplete and BarClearOnComplete
// are set, first bar section gets cleared and then whole bar line
// gets removed completely.
func BarRemoveOnComplete() BarOption {
	return func(s *bState) {
		s.removeOnComplete = true
	}
}

// BarReplaceOnComplete is indicator for delayed bar start, after the
// `runningBar` is complete. To achieve bar replacement effect,
// `runningBar` should has its `BarRemoveOnComplete` option set.
func BarReplaceOnComplete(runningBar *Bar) BarOption {
	return func(s *bState) {
		s.runningBar = runningBar
	}
}

// BarClearOnComplete is a flag, if set will clear bar section on
// complete event. If you need to remove a whole bar line, refer to
// BarRemoveOnComplete.
func BarClearOnComplete() BarOption {
	return func(s *bState) {
		s.barClearOnComplete = true
	}
}

// BarPriority sets bar's priority. Zero is highest priority, i.e. bar
// will be on top. If `BarReplaceOnComplete` option is supplied, this
// option is ignored.
func BarPriority(priority int) BarOption {
	return func(s *bState) {
		s.priority = priority
	}
}

// BarExtender is an option to extend bar to the next new line, with
// arbitrary output.
func BarExtender(extender Filler) BarOption {
	return func(s *bState) {
		s.extender = extender
	}
}

// TrimSpace trims bar's edge spaces.
func TrimSpace() BarOption {
	return func(s *bState) {
		s.trimSpace = true
	}
}

// BarStyle sets custom bar style.
// Effective when Filler type is bar.
func BarStyle(style string) BarOption {
	chk := func(filler Filler) (interface{}, bool) {
		if style == "" {
			return nil, false
		}
		t, ok := filler.(*barFiller)
		return t, ok
	}
	cb := func(t interface{}) {
		if !utf8.ValidString(style) {
			return
		}
		copy(t.(*barFiller).format, []rune(style))
	}
	return MakeFillerTypeSpecificBarOption(chk, cb)
}

// SpinnerStyle sets custom spinner style.
// Effective when Filler type is spinner.
func SpinnerStyle(frames []string) BarOption {
	chk := func(filler Filler) (interface{}, bool) {
		if len(frames) == 0 {
			return nil, false
		}
		t, ok := filler.(*spinnerFiller)
		return t, ok
	}
	cb := func(t interface{}) {
		t.(*spinnerFiller).frames = frames
	}
	return MakeFillerTypeSpecificBarOption(chk, cb)
}

// MakeFillerTypeSpecificBarOption makes BarOption specific to Filler's
// actual type. If you implement your own Filler, so most probably
// you'll need this. See BarStyle or SpinnerStyle for example.
func MakeFillerTypeSpecificBarOption(
	typeChecker func(Filler) (interface{}, bool),
	cb func(interface{}),
) BarOption {
	return func(s *bState) {
		if t, ok := typeChecker(s.filler); ok {
			cb(t)
		}
	}
}

// OptionOnCondition returns option when condition evaluates to true.
func OptionOnCondition(option BarOption, condition func() bool) BarOption {
	if condition() {
		return option
	}
	return nil
}
<|endoftext|>


<commit_before>package foursquarego

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

type FoursquareApi struct {
	clientID     string
	clientSecret string
	oauthToken   string
	queryQueue   chan query
	HttpClient   *http.Client
}

type query struct {
	url         string
	form        url.Values
	data        *foursquareResponse
	method      int
	response_ch chan response
}

type response struct {
	data interface{}
	err  error
}

type Omit struct{}

const (
	API_URL = "https://api.foursquare.com/v2/"
	VERSION = "20150813"
	MODE    = "foursquare"
	_GET    = iota
	_POST   = iota
)

func NewFoursquareApi(clientID string, clientSecret string) *FoursquareApi {
	queue := make(chan query)
	a := &FoursquareApi{
		clientID:     clientID,
		clientSecret: clientSecret,
		queryQueue:   queue,
		HttpClient:   http.DefaultClient,
	}
	go a.throttledQuery()
	return a
}

func (a *FoursquareApi) SetOauthToken(oauthToken string) {
	a.oauthToken = oauthToken
}

func (a *FoursquareApi) apiGet(urlStr string, form url.Values, data *foursquareResponse) error {
	req, err := http.NewRequest("GET", urlStr, nil)
	if err != nil {
		return err
	}
	req.URL.RawQuery = form.Encode()
	resp, err := a.HttpClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	contents, _ := ioutil.ReadAll(resp.Body)
	var apiResp apiResponse
	json.Unmarshal(contents, &apiResp)
	if resp.StatusCode != 200 || apiResp.Meta.Code != 200 {
		return newApiError(resp, apiResp.Meta)
	}
	*data = apiResp.Response
	return nil
}

func cleanValues(v url.Values) url.Values {
	if v == nil {
		return url.Values{}
	}
	return v
}

func (a *FoursquareApi) execQuery(urlStr string, form url.Values, data *foursquareResponse, method int) error {
	form.Set("v", VERSION)
	form.Set("m", MODE)
	if a.oauthToken != "" {
		form.Set("oauth_token", a.oauthToken)
	} else {
		form.Set("client_id", a.clientID)
		form.Set("client_secret", a.clientSecret)
	}
	switch method {
	case _GET:
		return a.apiGet(urlStr, form, data)
	default:
		return fmt.Errorf("HTTP method not supported")
	}
	return errors.New("ack")
}

func (a *FoursquareApi) throttledQuery() {
	for q := range a.queryQueue {
		url := q.url
		form := q.form
		data := q.data
		method := q.method

		response_ch := q.response_ch

		err := a.execQuery(url, form, data, method)
		response_ch <- response{data, err}
	}
}

func (a *FoursquareApi) Close() {
	close(a.queryQueue)
}
<commit_msg>if the ioutil read fails send error<commit_after>package foursquarego

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

type FoursquareApi struct {
	clientID     string
	clientSecret string
	oauthToken   string
	queryQueue   chan query
	HttpClient   *http.Client
}

type query struct {
	url         string
	form        url.Values
	data        *foursquareResponse
	method      int
	response_ch chan response
}

type response struct {
	data interface{}
	err  error
}

type Omit struct{}

const (
	API_URL = "https://api.foursquare.com/v2/"
	VERSION = "20150813"
	MODE    = "foursquare"
	_GET    = iota
	_POST   = iota
)

func NewFoursquareApi(clientID string, clientSecret string) *FoursquareApi {
	queue := make(chan query)
	a := &FoursquareApi{
		clientID:     clientID,
		clientSecret: clientSecret,
		queryQueue:   queue,
		HttpClient:   http.DefaultClient,
	}
	go a.throttledQuery()
	return a
}

func (a *FoursquareApi) SetOauthToken(oauthToken string) {
	a.oauthToken = oauthToken
}

func (a *FoursquareApi) apiGet(urlStr string, form url.Values, data *foursquareResponse) error {
	req, err := http.NewRequest("GET", urlStr, nil)
	if err != nil {
		return err
	}

	req.URL.RawQuery = form.Encode()
	resp, err := a.HttpClient.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()

	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	var apiResp apiResponse
	json.Unmarshal(contents, &apiResp)
	if resp.StatusCode != 200 || apiResp.Meta.Code != 200 {
		return newApiError(resp, apiResp.Meta)
	}

	*data = apiResp.Response
	return nil
}

func cleanValues(v url.Values) url.Values {
	if v == nil {
		return url.Values{}
	}
	return v
}

func (a *FoursquareApi) execQuery(urlStr string, form url.Values, data *foursquareResponse, method int) error {
	form.Set("v", VERSION)
	form.Set("m", MODE)
	if a.oauthToken != "" {
		form.Set("oauth_token", a.oauthToken)
	} else {
		form.Set("client_id", a.clientID)
		form.Set("client_secret", a.clientSecret)
	}
	switch method {
	case _GET:
		return a.apiGet(urlStr, form, data)
	default:
		return fmt.Errorf("HTTP method not supported")
	}
	return errors.New("ack")
}

func (a *FoursquareApi) throttledQuery() {
	for q := range a.queryQueue {
		url := q.url
		form := q.form
		data := q.data
		method := q.method

		response_ch := q.response_ch

		err := a.execQuery(url, form, data, method)
		response_ch <- response{data, err}
	}
}

func (a *FoursquareApi) Close() {
	close(a.queryQueue)
}
<|endoftext|>


<commit_before>// This is the main package for the `packer` application.
package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"

	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/packer/plugin"
	"github.com/mitchellh/panicwrap"
)

func main() {
	// Call realMain instead of doing the work here so we can use
	// `defer` statements within the function and have them work properly.
	// (defers aren't called with os.Exit)
	os.Exit(realMain())
}

// realMain is executed from main and returns the exit status to exit with.
func realMain() int {
	// If there is no explicit number of Go threads to use, then set it
	if os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(runtime.NumCPU())
	}

	// Determine where logs should go in general (requested by the user)
	logWriter, err := logOutput()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't setup log output: %s", err)
		return 1
	}

	// We also always send logs to a temporary file that we use in case
	// there is a panic. Otherwise, we delete it.
	logTempFile, err := ioutil.TempFile("", "packer-log")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't setup logging tempfile: %s", err)
		return 1
	}
	defer os.Remove(logTempFile.Name())
	defer logTempFile.Close()

	// Reset the log variables to minimize work in the subprocess
	os.Setenv("PACKER_LOG", "")
	os.Setenv("PACKER_LOG_FILE", "")

	// Create the configuration for panicwrap and wrap our executable
	wrapConfig := &panicwrap.WrapConfig{
		Handler: panicHandler(logTempFile),
		Writer:  io.MultiWriter(logTempFile, logWriter),
	}

	exitStatus, err := panicwrap.Wrap(wrapConfig)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't start Packer: %s", err)
		return 1
	}

	if exitStatus >= 0 {
		return exitStatus
	}

	// We're the child, so just close the tempfile we made in order to
	// save file handles since the tempfile is only used by the parent.
	logTempFile.Close()

	return wrappedMain()
}

// wrappedMain is called only when we're wrapped by panicwrap and
// returns the exit status to exit with.
func wrappedMain() int {
	log.SetOutput(os.Stderr)

	log.Printf(
		"Packer Version: %s %s %s",
		packer.Version, packer.VersionPrerelease, packer.GitCommit)
	log.Printf("Packer Target OS/Arch: %s %s", runtime.GOOS, runtime.GOARCH)
	log.Printf("Built with Go Version: %s", runtime.Version())

	// Prepare stdin for plugin usage by switching it to a pipe
	setupStdin()

	config, err := loadConfig()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error loading configuration: \n\n%s\n", err)
		return 1
	}

	log.Printf("Packer config: %+v", config)

	cacheDir := os.Getenv("PACKER_CACHE_DIR")
	if cacheDir == "" {
		cacheDir = "packer_cache"
	}

	cacheDir, err = filepath.Abs(cacheDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error preparing cache directory: \n\n%s\n", err)
		return 1
	}

	log.Printf("Setting cache directory: %s", cacheDir)
	cache := &packer.FileCache{CacheDir: cacheDir}

	// Determine if we're in machine-readable mode by mucking around with
	// the arguments...
	args, machineReadable := extractMachineReadable(os.Args[1:])

	defer plugin.CleanupClients()

	// Create the environment configuration
	envConfig := packer.DefaultEnvironmentConfig()
	envConfig.Cache = cache
	envConfig.Commands = config.CommandNames()
	envConfig.Components.Builder = config.LoadBuilder
	envConfig.Components.Command = config.LoadCommand
	envConfig.Components.Hook = config.LoadHook
	envConfig.Components.PostProcessor = config.LoadPostProcessor
	envConfig.Components.Provisioner = config.LoadProvisioner
	if machineReadable {
		envConfig.Ui = &packer.MachineReadableUi{
			Writer: os.Stdout,
		}

		// Set this so that we don't get colored output in our machine-
		// readable UI.
		if err := os.Setenv("PACKER_NO_COLOR", "1"); err != nil {
			fmt.Fprintf(os.Stderr, "Packer failed to initialize UI: %s\n", err)
			return 1
		}
	}

	env, err := packer.NewEnvironment(envConfig)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Packer initialization error: \n\n%s\n", err)
		return 1
	}

	setupSignalHandlers(env)

	exitCode, err := env.Cli(args)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error executing CLI: %s\n", err.Error())
		return 1
	}

	return exitCode
}

// extractMachineReadable checks the args for the machine readable
// flag and returns whether or not it is on. It modifies the args
// to remove this flag.
func extractMachineReadable(args []string) ([]string, bool) {
	for i, arg := range args {
		if arg == "-machine-readable" {
			// We found it. Slice it out.
			result := make([]string, len(args)-1)
			copy(result, args[:i])
			copy(result[i:], args[i+1:])
			return result, true
		}
	}

	return args, false
}

func loadConfig() (*config, error) {
	var config config
	config.PluginMinPort = 10000
	config.PluginMaxPort = 25000
	if err := config.Discover(); err != nil {
		return nil, err
	}

	mustExist := true
	configFilePath := os.Getenv("PACKER_CONFIG")
	if configFilePath == "" {
		var err error
		configFilePath, err = configFile()
		mustExist = false

		if err != nil {
			log.Printf("Error detecting default config file path: %s", err)
		}
	}

	if configFilePath == "" {
		return &config, nil
	}

	log.Printf("Attempting to open config file: %s", configFilePath)
	f, err := os.Open(configFilePath)
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, err
		}

		if mustExist {
			return nil, err
		}

		log.Println("File doesn't exist, but doesn't need to. Ignoring.")
		return &config, nil
	}
	defer f.Close()

	if err := decodeConfig(f, &config); err != nil {
		return nil, err
	}

	return &config, nil
}

// logOutput determines where we should send logs (if anywhere).
func logOutput() (logOutput io.Writer, err error) {
	logOutput = ioutil.Discard
	if os.Getenv("PACKER_LOG") != "" {
		logOutput = os.Stderr

		if logPath := os.Getenv("PACKER_LOG_PATH"); logPath != "" {
			var err error
			logOutput, err = os.Create(logPath)
			if err != nil {
				return nil, err
			}
		}
	}

	return
}
<commit_msg>Checkpoint<commit_after>// This is the main package for the `packer` application.
package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"

	"github.com/hashicorp/go-checkpoint"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/packer/plugin"
	"github.com/mitchellh/panicwrap"
)

func main() {
	// Call realMain instead of doing the work here so we can use
	// `defer` statements within the function and have them work properly.
	// (defers aren't called with os.Exit)
	os.Exit(realMain())
}

// realMain is executed from main and returns the exit status to exit with.
func realMain() int {
	// If there is no explicit number of Go threads to use, then set it
	if os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(runtime.NumCPU())
	}

	// Determine where logs should go in general (requested by the user)
	logWriter, err := logOutput()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't setup log output: %s", err)
		return 1
	}

	// We also always send logs to a temporary file that we use in case
	// there is a panic. Otherwise, we delete it.
	logTempFile, err := ioutil.TempFile("", "packer-log")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't setup logging tempfile: %s", err)
		return 1
	}
	defer os.Remove(logTempFile.Name())
	defer logTempFile.Close()

	// Reset the log variables to minimize work in the subprocess
	os.Setenv("PACKER_LOG", "")
	os.Setenv("PACKER_LOG_FILE", "")

	// Create the configuration for panicwrap and wrap our executable
	wrapConfig := &panicwrap.WrapConfig{
		Handler: panicHandler(logTempFile),
		Writer:  io.MultiWriter(logTempFile, logWriter),
	}

	exitStatus, err := panicwrap.Wrap(wrapConfig)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't start Packer: %s", err)
		return 1
	}

	if exitStatus >= 0 {
		return exitStatus
	}

	// We're the child, so just close the tempfile we made in order to
	// save file handles since the tempfile is only used by the parent.
	logTempFile.Close()

	return wrappedMain()
}

// wrappedMain is called only when we're wrapped by panicwrap and
// returns the exit status to exit with.
func wrappedMain() int {
	log.SetOutput(os.Stderr)

	log.Printf(
		"Packer Version: %s %s %s",
		packer.Version, packer.VersionPrerelease, packer.GitCommit)
	log.Printf("Packer Target OS/Arch: %s %s", runtime.GOOS, runtime.GOARCH)
	log.Printf("Built with Go Version: %s", runtime.Version())

	// Prepare stdin for plugin usage by switching it to a pipe
	setupStdin()

	config, err := loadConfig()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error loading configuration: \n\n%s\n", err)
		return 1
	}
	log.Printf("Packer config: %+v", config)

	configDir, err := ConfigDir()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error loading config dir: \n\n%s\n", err)
		return 1
	}

	// Fire off the checkpoint.
	version := packer.Version
	if packer.VersionPrerelease != "" {
		version += fmt.Sprintf(".%s", packer.VersionPrerelease)
	}
	go checkpoint.Check(&checkpoint.CheckParams{
		Product:       "packer",
		Version:       version,
		SignatureFile: filepath.Join(configDir, "checkpoint_signature"),
		CacheFile:     filepath.Join(configDir, "checkpoint_cache"),
	})

	cacheDir := os.Getenv("PACKER_CACHE_DIR")
	if cacheDir == "" {
		cacheDir = "packer_cache"
	}

	cacheDir, err = filepath.Abs(cacheDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error preparing cache directory: \n\n%s\n", err)
		return 1
	}

	log.Printf("Setting cache directory: %s", cacheDir)
	cache := &packer.FileCache{CacheDir: cacheDir}

	// Determine if we're in machine-readable mode by mucking around with
	// the arguments...
	args, machineReadable := extractMachineReadable(os.Args[1:])

	defer plugin.CleanupClients()

	// Create the environment configuration
	envConfig := packer.DefaultEnvironmentConfig()
	envConfig.Cache = cache
	envConfig.Commands = config.CommandNames()
	envConfig.Components.Builder = config.LoadBuilder
	envConfig.Components.Command = config.LoadCommand
	envConfig.Components.Hook = config.LoadHook
	envConfig.Components.PostProcessor = config.LoadPostProcessor
	envConfig.Components.Provisioner = config.LoadProvisioner
	if machineReadable {
		envConfig.Ui = &packer.MachineReadableUi{
			Writer: os.Stdout,
		}

		// Set this so that we don't get colored output in our machine-
		// readable UI.
		if err := os.Setenv("PACKER_NO_COLOR", "1"); err != nil {
			fmt.Fprintf(os.Stderr, "Packer failed to initialize UI: %s\n", err)
			return 1
		}
	}

	env, err := packer.NewEnvironment(envConfig)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Packer initialization error: \n\n%s\n", err)
		return 1
	}

	setupSignalHandlers(env)

	exitCode, err := env.Cli(args)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error executing CLI: %s\n", err.Error())
		return 1
	}

	return exitCode
}

// extractMachineReadable checks the args for the machine readable
// flag and returns whether or not it is on. It modifies the args
// to remove this flag.
func extractMachineReadable(args []string) ([]string, bool) {
	for i, arg := range args {
		if arg == "-machine-readable" {
			// We found it. Slice it out.
			result := make([]string, len(args)-1)
			copy(result, args[:i])
			copy(result[i:], args[i+1:])
			return result, true
		}
	}

	return args, false
}

func loadConfig() (*config, error) {
	var config config
	config.PluginMinPort = 10000
	config.PluginMaxPort = 25000
	if err := config.Discover(); err != nil {
		return nil, err
	}

	mustExist := true
	configFilePath := os.Getenv("PACKER_CONFIG")
	if configFilePath == "" {
		var err error
		configFilePath, err = configFile()
		mustExist = false

		if err != nil {
			log.Printf("Error detecting default config file path: %s", err)
		}
	}

	if configFilePath == "" {
		return &config, nil
	}

	log.Printf("Attempting to open config file: %s", configFilePath)
	f, err := os.Open(configFilePath)
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, err
		}

		if mustExist {
			return nil, err
		}

		log.Println("File doesn't exist, but doesn't need to. Ignoring.")
		return &config, nil
	}
	defer f.Close()

	if err := decodeConfig(f, &config); err != nil {
		return nil, err
	}

	return &config, nil
}

// logOutput determines where we should send logs (if anywhere).
func logOutput() (logOutput io.Writer, err error) {
	logOutput = ioutil.Discard
	if os.Getenv("PACKER_LOG") != "" {
		logOutput = os.Stderr

		if logPath := os.Getenv("PACKER_LOG_PATH"); logPath != "" {
			var err error
			logOutput, err = os.Create(logPath)
			if err != nil {
				return nil, err
			}
		}
	}

	return
}
<|endoftext|>